Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .talismanrc
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ fileignoreconfig:
- filename: packages/contentstack-query-export/.env-example
checksum: 922c7aa9c788ab60b987de2b0a2aee6d90843c463a8bbc29201e4efe31081187
- filename: pnpm-lock.yaml
checksum: bb5303f2fe64f90ae95d2738363267fb0bfcfeb71f025c2110d4cec87ff84d95
checksum: 3d2eaabf1df366efee1759156465c6aefa68f30d372717de2cdc3e41946aa3d8
- filename: packages/contentstack-import/src/utils/build-import-spaces-options.ts
checksum: fe0cb6cb5903515982af1e3642f2a19233207d35f13dc205cebeda0aa399f8b5
- filename: packages/contentstack-export/src/export/modules/stack.ts
Expand Down
49 changes: 0 additions & 49 deletions packages/contentstack-asset-management/README.md

This file was deleted.

9 changes: 5 additions & 4 deletions packages/contentstack-asset-management/package.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "@contentstack/cli-asset-management",
"version": "1.0.0-beta.0",
"description": "Asset Management 2.0 API adapter for export and import",
"description": "Contentstack Assets API adapter for export and import",
"main": "lib/index.js",
"types": "lib/index.d.ts",
"files": [
Expand All @@ -24,7 +24,8 @@
},
"keywords": [
"contentstack",
"asset-management",
"cs-assets",
"contentstack-assets",
"cli"
],
"license": "MIT",
Expand All @@ -37,7 +38,7 @@
"devPlugins": [
"@oclif/plugin-help"
],
"repositoryPrefix": "<%- repo %>/blob/main/packages/contentstack-asset-management/<%- commandPath %>"
"repositoryPrefix": "<%- repo %>/blob/main/packages/contentstack-cs-assets/<%- commandPath %>"
},
"devDependencies": {
"@types/chai": "^4.3.11",
Expand All @@ -55,4 +56,4 @@
"ts-node": "^10.9.2",
"typescript": "^5.8.3"
}
}
}
50 changes: 47 additions & 3 deletions packages/contentstack-asset-management/src/constants/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,25 @@ export const FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS = [
export const CHUNK_FILE_SIZE_MB = FALLBACK_AM_CHUNK_FILE_SIZE_MB;

/**
* Main process name for Asset Management 2.0 export (single progress bar).
* Main process name for Contentstack Assets export (single progress bar).
* Use this when adding/starting the process and for all ticks.
*/
export const AM_MAIN_PROCESS_NAME = 'Asset Management 2.0';
export const CS_ASSETS_MAIN_PROCESS_NAME = 'Contentstack Assets';
/** @deprecated Use CS_ASSETS_MAIN_PROCESS_NAME */
export const AM_MAIN_PROCESS_NAME = CS_ASSETS_MAIN_PROCESS_NAME;

/**
* Process names for Asset Management 2.0 export progress (for tick labels).
* Process names for Contentstack Assets export/import progress.
*
* In the new per-space layout each entry below corresponds to a single row in
* the multibar:
* - {@link AM_FIELDS} / {@link AM_ASSET_TYPES} are the shared bootstrap rows
* (one execution per org, ahead of per-space work).
* - {@link AM_IMPORT_FIELDS} / {@link AM_IMPORT_ASSET_TYPES} are the import
* equivalents.
* - One additional row per space is added dynamically via
* {@link getSpaceProcessName} and ticks include folders + metadata + asset
* transfer for that space.
*/
export const PROCESS_NAMES = {
AM_SPACE_METADATA: 'Space metadata',
Expand All @@ -51,6 +63,38 @@ export const PROCESS_NAMES = {
AM_IMPORT_ASSETS: 'Import assets',
} as const;

/**
* Maximum visual length of a per-space process row label. The CLIProgressManager
* truncates anything over 20 characters; reserve 6 chars for the `Space ` prefix
* so the trailing space uid keeps 14 chars before truncation.
*/
const SPACE_PROCESS_NAME_PREFIX = 'Space ';
const SPACE_PROCESS_NAME_MAX_UID_LEN = 14;

/**
* Returns the multibar row label for a single CS Assets space.
* The label is bounded so CLIProgressManager.formatProcessName doesn't truncate
* it mid-string; the full uid is still used for tick item labels and structured
* logs, only the row label itself is shortened for display.
*/
export function getSpaceProcessName(spaceUid: string): string {
const safeUid = spaceUid ?? '';
const trimmed =
safeUid.length > SPACE_PROCESS_NAME_MAX_UID_LEN
? safeUid.substring(0, SPACE_PROCESS_NAME_MAX_UID_LEN)
: safeUid;
return `${SPACE_PROCESS_NAME_PREFIX}${trimmed}`;
}

/**
* Detects whether a process name belongs to a per-space progress row, used by
* the export/import strategy registries to aggregate counts for the final
* summary across all spaces.
*/
export function isSpaceProcessName(processName: string): boolean {
return typeof processName === 'string' && processName.startsWith(SPACE_PROCESS_NAME_PREFIX);
}

/**
* Status messages for each process (exporting, fetching, importing, failed).
*/
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
import { log } from '@contentstack/cli-utilities';

import type { AssetManagementAPIConfig } from '../types/asset-management-api';
import type { CSAssetsAPIConfig } from '../types/cs-assets-api';
import type { ExportContext } from '../types/export-types';
import { AssetManagementExportAdapter } from './base';
import { CSAssetsExportAdapter } from './base';
import { getArrayFromResponse } from '../utils/export-helpers';
import { PROCESS_NAMES } from '../constants/index';

export default class ExportAssetTypes extends AssetManagementExportAdapter {
constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) {
export default class ExportAssetTypes extends CSAssetsExportAdapter {
protected processName: string = PROCESS_NAMES.AM_ASSET_TYPES;

constructor(apiConfig: CSAssetsAPIConfig, exportContext: ExportContext) {
super(apiConfig, exportContext);
}

Expand All @@ -24,7 +26,13 @@ export default class ExportAssetTypes extends AssetManagementExportAdapter {
} else {
log.debug(`Writing ${items.length} shared asset types`, this.exportContext.context);
}
await this.writeItemsToChunkedJson(dir, 'asset-types.json', 'asset_types', ['uid', 'title', 'category', 'file_extension'], items);
this.tick(true, PROCESS_NAMES.AM_ASSET_TYPES, null);
await this.writeItemsToChunkedJson(
dir,
'asset-types.json',
'asset_types',
['uid', 'title', 'category', 'file_extension'],
items,
);
this.tick(true, `asset_types (${items.length})`, null);
}
}
31 changes: 18 additions & 13 deletions packages/contentstack-asset-management/src/export/assets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,15 @@ import { Readable } from 'node:stream';
import { mkdir, writeFile } from 'node:fs/promises';
import { configHandler, log } from '@contentstack/cli-utilities';

import type { AssetManagementAPIConfig, LinkedWorkspace } from '../types/asset-management-api';
import type { CSAssetsAPIConfig, LinkedWorkspace } from '../types/cs-assets-api';
import type { ExportContext } from '../types/export-types';
import { AssetManagementExportAdapter } from './base';
import { CSAssetsExportAdapter } from './base';
import { getAssetItems, writeStreamToFile } from '../utils/export-helpers';
import { runInBatches } from '../utils/concurrent-batch';
import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index';

export default class ExportAssets extends AssetManagementExportAdapter {
constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) {
export default class ExportAssets extends CSAssetsExportAdapter {
constructor(apiConfig: CSAssetsAPIConfig, exportContext: ExportContext) {
super(apiConfig, exportContext);
}

Expand All @@ -32,11 +32,17 @@ export default class ExportAssets extends AssetManagementExportAdapter {
this.getWorkspaceAssets(workspace.space_uid, workspace.uid),
]);

const assetItems = getAssetItems(assetsData);
const downloadableCount = assetItems.filter((asset) => Boolean(asset.url && (asset.uid ?? asset._uid))).length;
// Per-space total: 1 folder write + 1 metadata write + N per-asset downloads.
// The shared module-level total is just a placeholder before this point; update
// it now so the multibar row shows real progress as downloads tick in.
this.progressOrParent?.updateProcessTotal?.(this.processName, 2 + downloadableCount);

await writeFile(pResolve(assetsDir, 'folders.json'), JSON.stringify(folders, null, 2));
this.tick(true, `folders: ${workspace.space_uid}`, null);
log.debug(`Wrote folders.json for space ${workspace.space_uid}`, this.exportContext.context);

const assetItems = getAssetItems(assetsData);
log.debug(
assetItems.length === 0
? `No assets for space ${workspace.space_uid}, wrote empty assets.json`
Expand All @@ -60,7 +66,7 @@ export default class ExportAssets extends AssetManagementExportAdapter {
: `Wrote ${assetItems.length} asset metadata record(s) for space ${workspace.space_uid}`,
this.exportContext.context,
);
this.tick(true, `assets: ${workspace.space_uid} (${assetItems.length})`, null);
this.tick(true, `metadata: ${workspace.space_uid} (${assetItems.length})`, null);

log.debug(`Starting binary downloads for space ${workspace.space_uid}`, this.exportContext.context);
await this.downloadWorkspaceAssets(assetsData, assetsDir, workspace.space_uid);
Expand All @@ -87,8 +93,6 @@ export default class ExportAssets extends AssetManagementExportAdapter {
`Asset downloads: securedAssets=${securedAssets}, concurrency=${this.downloadAssetsBatchConcurrency}`,
this.exportContext.context,
);
let lastError: string | null = null;
let allSuccess = true;
let downloadOk = 0;
let downloadFail = 0;

Expand Down Expand Up @@ -118,24 +122,25 @@ export default class ExportAssets extends AssetManagementExportAdapter {
const filePath = pResolve(assetFolderPath, filename);
await writeStreamToFile(nodeStream, filePath);
downloadOk += 1;
// Per-asset tick so the per-space progress bar moves in real time.
this.tick(true, `asset: ${filename}`, null);
log.debug(`Downloaded asset ${uid} → ${filePath}`, this.exportContext.context);
} catch (e) {
allSuccess = false;
downloadFail += 1;
lastError = (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].FAILED;
const err = (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].FAILED;
this.tick(false, `asset: ${filename}`, err);
log.debug(`Failed to download asset ${uid}: ${e}`, this.exportContext.context);
}
});

this.tick(allSuccess, `downloads: ${spaceUid}`, lastError);
log.info(
allSuccess
downloadFail === 0
? `Finished downloading ${downloadOk} asset file(s) for space ${spaceUid}`
: `Asset downloads for space ${spaceUid} completed with errors: ${downloadOk} succeeded, ${downloadFail} failed`,
this.exportContext.context,
);
log.debug(
`Asset downloads finished for space ${spaceUid}: ok=${downloadOk}, failed=${downloadFail}, allSuccess=${allSuccess}`,
`Asset downloads finished for space ${spaceUid}: ok=${downloadOk}, failed=${downloadFail}`,
this.exportContext.context,
);
}
Expand Down
23 changes: 16 additions & 7 deletions packages/contentstack-asset-management/src/export/base.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,25 @@ import { resolve as pResolve } from 'node:path';
import { writeFile } from 'node:fs/promises';
import { FsUtility, log, CLIProgressManager, configHandler } from '@contentstack/cli-utilities';

import type { AssetManagementAPIConfig } from '../types/asset-management-api';
import type { CSAssetsAPIConfig } from '../types/cs-assets-api';
import type { ExportContext } from '../types/export-types';
import { AssetManagementAdapter } from '../utils/asset-management-api-adapter';
import { AM_MAIN_PROCESS_NAME, FALLBACK_AM_API_CONCURRENCY, FALLBACK_AM_CHUNK_FILE_SIZE_MB } from '../constants/index';
import { CSAssetsAdapter } from '../utils/cs-assets-api-adapter';
import { CS_ASSETS_MAIN_PROCESS_NAME, FALLBACK_AM_API_CONCURRENCY, FALLBACK_AM_CHUNK_FILE_SIZE_MB } from '../constants/index';

export type { ExportContext };

/**
* Base class for export modules. Extends the API adapter and adds export context,
* internal progress management, and shared write helpers.
*/
export class AssetManagementExportAdapter extends AssetManagementAdapter {
protected readonly apiConfig: AssetManagementAPIConfig;
export class CSAssetsExportAdapter extends CSAssetsAdapter {
protected readonly apiConfig: CSAssetsAPIConfig;
protected readonly exportContext: ExportContext;
protected progressManager: CLIProgressManager | null = null;
protected parentProgressManager: CLIProgressManager | null = null;
protected readonly processName: string = AM_MAIN_PROCESS_NAME;
protected processName: string = CS_ASSETS_MAIN_PROCESS_NAME;

constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) {
constructor(apiConfig: CSAssetsAPIConfig, exportContext: ExportContext) {
super(apiConfig);
this.apiConfig = apiConfig;
this.exportContext = exportContext;
Expand All @@ -30,6 +30,15 @@ export class AssetManagementExportAdapter extends AssetManagementAdapter {
this.parentProgressManager = parent;
}

/**
* Override the default progress process name for {@link tick}/{@link updateStatus}
* calls. Used by the per-space orchestrator so each module's ticks land on the
* row for the space currently being exported.
*/
public setProcessName(name: string): void {
this.processName = name;
}

protected get progressOrParent(): CLIProgressManager | null {
return this.parentProgressManager ?? this.progressManager;
}
Expand Down
12 changes: 7 additions & 5 deletions packages/contentstack-asset-management/src/export/fields.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
import { log } from '@contentstack/cli-utilities';

import type { AssetManagementAPIConfig } from '../types/asset-management-api';
import type { CSAssetsAPIConfig } from '../types/cs-assets-api';
import type { ExportContext } from '../types/export-types';
import { AssetManagementExportAdapter } from './base';
import { CSAssetsExportAdapter } from './base';
import { getArrayFromResponse } from '../utils/export-helpers';
import { PROCESS_NAMES } from '../constants/index';

export default class ExportFields extends AssetManagementExportAdapter {
constructor(apiConfig: AssetManagementAPIConfig, exportContext: ExportContext) {
export default class ExportFields extends CSAssetsExportAdapter {
protected processName: string = PROCESS_NAMES.AM_FIELDS;

constructor(apiConfig: CSAssetsAPIConfig, exportContext: ExportContext) {
super(apiConfig, exportContext);
}

Expand All @@ -25,6 +27,6 @@ export default class ExportFields extends AssetManagementExportAdapter {
log.debug(`Writing ${items.length} shared fields`, this.exportContext.context);
}
await this.writeItemsToChunkedJson(dir, 'fields.json', 'fields', ['uid', 'title', 'display_type'], items);
this.tick(true, PROCESS_NAMES.AM_FIELDS, null);
this.tick(true, `fields (${items.length})`, null);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@ export { default as ExportAssetTypes } from './asset-types';
export { default as ExportFields } from './fields';
export { default as ExportAssets } from './assets';
export { default as ExportWorkspace } from './workspaces';
export { AssetManagementExportAdapter } from './base';
export { CSAssetsExportAdapter } from './base';
export type { ExportContext } from './base';
Loading
Loading