From 78ffc9fec30559e210e59e5a6c3b907f32f4811e Mon Sep 17 00:00:00 2001 From: David Nuescheler Date: Sun, 19 Apr 2026 14:37:40 -0600 Subject: [PATCH 1/2] feat(content): clone HTML/JSON by default, add --media for assets - Filter listed files to .html and .json unless --media is passed - Add filterFilesForContentClone helper in content-shared - Log when total discovered count exceeds cloned count - Extend clone command and tests Made-with: Cursor --- src/content/clone.cmd.js | 20 +++++++- src/content/clone.js | 6 +++ src/content/content-shared.js | 30 +++++++++++ test/content/clone.cmd.test.js | 73 ++++++++++++++++++++++++++- test/content/content-commands.test.js | 19 +++++-- 5 files changed, 140 insertions(+), 8 deletions(-) diff --git a/src/content/clone.cmd.js b/src/content/clone.cmd.js index ccf6c9430..f3599cb78 100644 --- a/src/content/clone.cmd.js +++ b/src/content/clone.cmd.js @@ -25,6 +25,7 @@ import { GIT_AUTHOR, LARGE_CLONE_FILE_THRESHOLD, CONTENT_IO_CONCURRENCY, + filterFilesForContentClone, } from './content-shared.js'; import { writeSyncedRef, ensureGitIgnored } from './content-git.js'; @@ -67,6 +68,7 @@ export default class CloneCommand { this._force = false; this._assumeYes = false; this._rootPath = null; + this._includeMedia = false; } withDirectory(dir) { @@ -94,6 +96,11 @@ export default class CloneCommand { return this; } + withIncludeMedia(include) { + this._includeMedia = !!include; + return this; + } + async run() { const { log } = this; @@ -126,7 +133,7 @@ export default class CloneCommand { const client = new DaClient(token); log.info('Fetching file list...'); const showDiscoveryProgress = process.stdout.isTTY; - const files = await client.listAll(org, repo, this._rootPath, showDiscoveryProgress + const listedFiles = await client.listAll(org, repo, this._rootPath, showDiscoveryProgress ? (n) => { process.stdout.write(`\r ${n} file(s) discovered so far...`); } @@ -134,7 +141,16 @@ export default class CloneCommand { if (showDiscoveryProgress) { process.stdout.write('\n'); } - log.info(`Found ${files.length} file(s).`); + const listedCount = listedFiles.length; + const files = filterFilesForContentClone(listedFiles, this._includeMedia); + if (!this._includeMedia && listedCount > files.length) { + log.info( + `Found ${listedCount} file(s); cloning ${files.length} HTML/JSON file(s). ` + + 'Use --media to include images, video, PDFs, and other assets.', + ); + } else { + log.info(`Found ${files.length} file(s).`); + } await confirmLargeCloneIfNeeded(log, files.length, this._assumeYes); diff --git a/src/content/clone.js b/src/content/clone.js index 27f7e1665..d57f329f4 100644 --- a/src/content/clone.js +++ b/src/content/clone.js @@ -35,6 +35,11 @@ export default function clone() { describe: 'IMS Bearer token for da.live authentication', type: 'string', }) + .option('media', { + describe: 'Include non-HTML/JSON assets (images, video, SVG, PDF, etc.)', + type: 'boolean', + default: false, + }) .option('force', { describe: 'Overwrite existing content/ without prompting', type: 'boolean', @@ -67,6 +72,7 @@ export default function clone() { .withToken(argv.token) .withForce(argv.force) .withAssumeYes(argv.yes) + .withIncludeMedia(argv.media) .withRootPath(rootPath) .run(); }, diff --git a/src/content/content-shared.js b/src/content/content-shared.js index e34ffd84c..19a2316b9 100644 --- a/src/content/content-shared.js +++ b/src/content/content-shared.js @@ -10,6 +10,8 @@ * governing permissions and limitations under the License. */ +import path from 'path'; + export const CONTENT_DIR = 'content'; export const CONFIG_FILE = '.da-config.json'; export const GIT_AUTHOR = { name: 'aem-cli', email: 'aem-cli@adobe.com' }; @@ -22,6 +24,34 @@ export const LARGE_CLONE_FILE_THRESHOLD = 10000; */ export const CONTENT_IO_CONCURRENCY = 10; +/** File extensions cloned by default (without `--media`). Case-insensitive. */ +const CONTENT_CLONE_DEFAULT_EXTS = new Set(['html', 'json']); + +/** + * @param {{ ext?: string, name?: string, path?: string }} file + * @returns {string} lower-case extension without leading dot, or '' if unknown + */ +function contentFileExtension(file) { + if (file.ext != null && String(file.ext) !== '') { + return String(file.ext).replace(/^\./, '').toLowerCase(); + } + const base = file.name || file.path || ''; + return path.extname(base).replace(/^\./, '').toLowerCase(); +} + +/** + * When `includeMedia` is false, keeps only `.html` and `.json` entries (da.live list items). + * @param {Array<{ ext?: string, name?: string, path?: string }>} files + * @param {boolean} includeMedia + * @returns {typeof files} + */ +export function filterFilesForContentClone(files, includeMedia) { + if (includeMedia) { + return files; + } + return files.filter((f) => CONTENT_CLONE_DEFAULT_EXTS.has(contentFileExtension(f))); +} + /** * Normalizes a da.live path: leading slash, no trailing slash except root. * @param {string} input diff --git a/test/content/clone.cmd.test.js b/test/content/clone.cmd.test.js index f420829a2..c52db9289 100644 --- a/test/content/clone.cmd.test.js +++ b/test/content/clone.cmd.test.js @@ -19,7 +19,12 @@ import git from 'isomorphic-git'; import esmock from 'esmock'; import { createTestRoot } from '../utils.js'; import { makeLogger, createDaClientClass } from './content-test-utils.js'; -import { normalizeDaPath, CONTENT_DIR, LARGE_CLONE_FILE_THRESHOLD } from '../../src/content/content-shared.js'; +import { + normalizeDaPath, + CONTENT_DIR, + LARGE_CLONE_FILE_THRESHOLD, + filterFilesForContentClone, +} from '../../src/content/content-shared.js'; import { DA_SYNCED_REF } from '../../src/content/content-git.js'; async function makeCloneCommand(testRoot, DaClientClass) { @@ -83,6 +88,12 @@ describe('CloneCommand', () => { assert.strictEqual(cmd._assumeYes, true); // eslint-disable-line no-underscore-dangle }); + it('withIncludeMedia sets _includeMedia', async () => { + const cmd = await makeCloneCommand(testRoot, createDaClientClass()); + cmd.withIncludeMedia(true); + assert.strictEqual(cmd._includeMedia, true); // eslint-disable-line no-underscore-dangle + }); + it('builder methods return this for chaining', async () => { const mod = await esmock('../../src/content/clone.cmd.js', { '../../src/git-utils.js': { default: { getOriginURL: async () => null } }, @@ -96,6 +107,34 @@ describe('CloneCommand', () => { assert.strictEqual(cmd.withForce(false), cmd); assert.strictEqual(cmd.withRootPath('/'), cmd); assert.strictEqual(cmd.withAssumeYes(false), cmd); + assert.strictEqual(cmd.withIncludeMedia(false), cmd); + }); + }); + + describe('filterFilesForContentClone()', () => { + it('keeps only html and json when includeMedia is false', () => { + const files = [ + { path: '/o/r/a.html', name: 'a.html', ext: 'html' }, + { path: '/o/r/b.json', name: 'b.json', ext: 'json' }, + { path: '/o/r/c.jpg', name: 'c.jpg', ext: 'jpg' }, + ]; + const out = filterFilesForContentClone(files, false); + assert.strictEqual(out.length, 2); + assert.ok(out.every((f) => f.ext === 'html' || f.ext === 'json')); + }); + + it('is case-insensitive on extension', () => { + const files = [{ path: '/o/r/x.HTML', name: 'x.HTML', ext: 'HTML' }]; + const out = filterFilesForContentClone(files, false); + assert.strictEqual(out.length, 1); + }); + + it('returns all files when includeMedia is true', () => { + const files = [ + { path: '/o/r/a.html', name: 'a.html', ext: 'html' }, + { path: '/o/r/v.mp4', name: 'v.mp4', ext: 'mp4' }, + ]; + assert.strictEqual(filterFilesForContentClone(files, true).length, 2); }); }); @@ -176,6 +215,38 @@ describe('CloneCommand', () => { assert.strictEqual(content, 'page'); }); + it('by default skips non-HTML/JSON files', async () => { + const DaClientClass = createDaClientClass({ + files: [ + { path: '/myorg/myrepo/page.html', name: 'page.html', ext: 'html' }, + { path: '/myorg/myrepo/hero.jpg', name: 'hero.jpg', ext: 'jpg' }, + ], + sourceContent: 'x', + }); + const cmd = await makeCloneCommand(testRoot, DaClientClass); + cmd.withRootPath('/'); + await cmd.run(); + + assert.ok(await fse.pathExists(path.join(testRoot, CONTENT_DIR, 'page.html'))); + assert.ok(!await fse.pathExists(path.join(testRoot, CONTENT_DIR, 'hero.jpg'))); + }); + + it('withIncludeMedia downloads non-HTML/JSON files', async () => { + const DaClientClass = createDaClientClass({ + files: [ + { path: '/myorg/myrepo/page.html', name: 'page.html', ext: 'html' }, + { path: '/myorg/myrepo/hero.jpg', name: 'hero.jpg', ext: 'jpg' }, + ], + sourceContent: 'payload', + }); + const cmd = await makeCloneCommand(testRoot, DaClientClass); + cmd.withIncludeMedia(true).withRootPath('/'); + await cmd.run(); + + assert.ok(await fse.pathExists(path.join(testRoot, CONTENT_DIR, 'page.html'))); + assert.ok(await fse.pathExists(path.join(testRoot, CONTENT_DIR, 'hero.jpg'))); + }); + it('writes .da-config.json with org, repo, and rootPath', async () => { const cmd = await makeCloneCommand(testRoot, createDaClientClass()); cmd.withRootPath('/blog'); diff --git a/test/content/content-commands.test.js b/test/content/content-commands.test.js index 1e7171c5d..4fdf3b4c8 100644 --- a/test/content/content-commands.test.js +++ b/test/content/content-commands.test.js @@ -69,7 +69,7 @@ describe('clone()', () => { assert.ok(cmd.description && cmd.description.length > 0); }); - it('has a builder that registers path, all, token, force, and yes options', () => { + it('has a builder that registers path, all, token, media, force, and yes options', () => { const cmd = clone(); const registered = {}; const chainable = { @@ -84,6 +84,7 @@ describe('clone()', () => { assert.ok('path' in registered); assert.ok('all' in registered); assert.ok('token' in registered); + assert.ok('media' in registered); assert.ok('force' in registered); assert.ok('yes' in registered); }); @@ -94,6 +95,7 @@ describe('clone()', () => { withToken: () => mockExecutor, withForce: () => mockExecutor, withAssumeYes: () => mockExecutor, + withIncludeMedia: () => mockExecutor, withRootPath: () => mockExecutor, run: async () => {}, }; @@ -109,6 +111,7 @@ describe('clone()', () => { const cmd = clone(); let ranWith; let assumeYesArg; + let includeMediaArg; let rootPathArg; cmd.executor = { withToken: (t) => { @@ -118,9 +121,14 @@ describe('clone()', () => { withAssumeYes: (y) => { assumeYesArg = y; return { - withRootPath: (rp) => { - rootPathArg = rp; - return { run: async () => {} }; + withIncludeMedia: (m) => { + includeMediaArg = m; + return { + withRootPath: (rp) => { + rootPathArg = rp; + return { run: async () => {} }; + }, + }; }, }; }, @@ -129,10 +137,11 @@ describe('clone()', () => { }, }; await cmd.handler({ - token: 'abc', force: false, yes: true, all: false, path: '/ca/fr_ca', + token: 'abc', force: false, yes: true, media: true, all: false, path: '/ca/fr_ca', }); assert.strictEqual(ranWith, 'abc'); assert.strictEqual(assumeYesArg, true); + assert.strictEqual(includeMediaArg, true); assert.strictEqual(rootPathArg, '/ca/fr_ca'); }); }); From 6aa12e44c4fbb14079db73b62ace9facc9d5cdb1 Mon Sep 17 00:00:00 2001 From: David Nuescheler Date: Sun, 26 Apr 2026 14:00:44 -0600 Subject: [PATCH 2/2] chore: simpler extension handling --- src/content/content-shared.js | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/content/content-shared.js b/src/content/content-shared.js index 19a2316b9..c58f26c6f 100644 --- a/src/content/content-shared.js +++ b/src/content/content-shared.js @@ -25,18 +25,15 @@ export const LARGE_CLONE_FILE_THRESHOLD = 10000; export const CONTENT_IO_CONCURRENCY = 10; /** File extensions cloned by default (without `--media`). Case-insensitive. */ -const CONTENT_CLONE_DEFAULT_EXTS = new Set(['html', 'json']); +const CONTENT_CLONE_DEFAULT_EXTS = new Set(['.html', '.json']); /** * @param {{ ext?: string, name?: string, path?: string }} file - * @returns {string} lower-case extension without leading dot, or '' if unknown + * @returns {string} lower-case extension with leading dot, or '' if unknown */ function contentFileExtension(file) { - if (file.ext != null && String(file.ext) !== '') { - return String(file.ext).replace(/^\./, '').toLowerCase(); - } - const base = file.name || file.path || ''; - return path.extname(base).replace(/^\./, '').toLowerCase(); + const lowerExt = (file.ext != null && file.ext !== '' ? String(file.ext) : path.extname(file.name || file.path || '')).toLowerCase(); + return !lowerExt || lowerExt.startsWith('.') ? lowerExt : `.${lowerExt}`; } /**