feat(content): clone HTML/JSON by default, add --media for assets

src/content/clone.cmd.js

@@ -25,6 +25,7 @@ import {
   GIT_AUTHOR,
   LARGE_CLONE_FILE_THRESHOLD,
   CONTENT_IO_CONCURRENCY,
+  filterFilesForContentClone,
 } from './content-shared.js';
 import { writeSyncedRef, ensureGitIgnored } from './content-git.js';
 
@@ -67,6 +68,7 @@ export default class CloneCommand {
     this._force = false;
     this._assumeYes = false;
     this._rootPath = null;
+    this._includeMedia = false;
   }
 
   withDirectory(dir) {
@@ -94,6 +96,11 @@ export default class CloneCommand {
     return this;
   }
 
+  withIncludeMedia(include) {
+    this._includeMedia = !!include;
+    return this;
+  }
+
   async run() {
     const { log } = this;
 
@@ -126,15 +133,24 @@ export default class CloneCommand {
     const client = new DaClient(token);
     log.info('Fetching file list...');
     const showDiscoveryProgress = process.stdout.isTTY;
-    const files = await client.listAll(org, repo, this._rootPath, showDiscoveryProgress
+    const listedFiles = await client.listAll(org, repo, this._rootPath, showDiscoveryProgress
       ? (n) => {
         process.stdout.write(`\r  ${n} file(s) discovered so far...`);
       }
       : undefined);
     if (showDiscoveryProgress) {
       process.stdout.write('\n');
     }
-    log.info(`Found ${files.length} file(s).`);
+    const listedCount = listedFiles.length;
+    const files = filterFilesForContentClone(listedFiles, this._includeMedia);
+    if (!this._includeMedia && listedCount > files.length) {
+      log.info(
+        `Found ${listedCount} file(s); cloning ${files.length} HTML/JSON file(s). `
+        + 'Use --media to include images, video, PDFs, and other assets.',
+      );
+    } else {
+      log.info(`Found ${files.length} file(s).`);
+    }
 
     await confirmLargeCloneIfNeeded(log, files.length, this._assumeYes);
 

src/content/clone.js

@@ -35,6 +35,11 @@ export default function clone() {
           describe: 'IMS Bearer token for da.live authentication',
           type: 'string',
         })
+        .option('media', {
+          describe: 'Include non-HTML/JSON assets (images, video, SVG, PDF, etc.)',
+          type: 'boolean',
+          default: false,
+        })
         .option('force', {
           describe: 'Overwrite existing content/ without prompting',
           type: 'boolean',
@@ -67,6 +72,7 @@ export default function clone() {
         .withToken(argv.token)
         .withForce(argv.force)
         .withAssumeYes(argv.yes)
+        .withIncludeMedia(argv.media)
         .withRootPath(rootPath)
         .run();
     },

src/content/content-shared.js

@@ -10,6 +10,8 @@
  * governing permissions and limitations under the License.
  */
 
+import path from 'path';
+
 export const CONTENT_DIR = 'content';
 export const CONFIG_FILE = '.da-config.json';
 export const GIT_AUTHOR = { name: 'aem-cli', email: 'aem-cli@adobe.com' };
@@ -22,6 +24,31 @@ export const LARGE_CLONE_FILE_THRESHOLD = 10000;
  */
 export const CONTENT_IO_CONCURRENCY = 10;
 
+/** File extensions cloned by default (without `--media`). Case-insensitive. */
+const CONTENT_CLONE_DEFAULT_EXTS = new Set(['.html', '.json']);
+
+/**
+ * @param {{ ext?: string, name?: string, path?: string }} file
+ * @returns {string} lower-case extension with leading dot, or '' if unknown
+ */
+function contentFileExtension(file) {
+  const lowerExt = (file.ext != null && file.ext !== '' ? String(file.ext) : path.extname(file.name || file.path || '')).toLowerCase();
+  return !lowerExt || lowerExt.startsWith('.') ? lowerExt : `.${lowerExt}`;
+}
+
+/**
+ * When `includeMedia` is false, keeps only `.html` and `.json` entries (da.live list items).
+ * @param {Array<{ ext?: string, name?: string, path?: string }>} files
+ * @param {boolean} includeMedia
+ * @returns {typeof files}
+ */
+export function filterFilesForContentClone(files, includeMedia) {
+  if (includeMedia) {
+    return files;
+  }
+  return files.filter((f) => CONTENT_CLONE_DEFAULT_EXTS.has(contentFileExtension(f)));
+}
+
 /**
  * Normalizes a da.live path: leading slash, no trailing slash except root.
  * @param {string} input

test/content/clone.cmd.test.js

@@ -19,7 +19,12 @@ import git from 'isomorphic-git';
 import esmock from 'esmock';
 import { createTestRoot } from '../utils.js';
 import { makeLogger, createDaClientClass } from './content-test-utils.js';
-import { normalizeDaPath, CONTENT_DIR, LARGE_CLONE_FILE_THRESHOLD } from '../../src/content/content-shared.js';
+import {
+  normalizeDaPath,
+  CONTENT_DIR,
+  LARGE_CLONE_FILE_THRESHOLD,
+  filterFilesForContentClone,
+} from '../../src/content/content-shared.js';
 import { DA_SYNCED_REF } from '../../src/content/content-git.js';
 
 async function makeCloneCommand(testRoot, DaClientClass) {
@@ -83,6 +88,12 @@ describe('CloneCommand', () => {
       assert.strictEqual(cmd._assumeYes, true); // eslint-disable-line no-underscore-dangle
     });
 
+    it('withIncludeMedia sets _includeMedia', async () => {
+      const cmd = await makeCloneCommand(testRoot, createDaClientClass());
+      cmd.withIncludeMedia(true);
+      assert.strictEqual(cmd._includeMedia, true); // eslint-disable-line no-underscore-dangle
+    });
+
     it('builder methods return this for chaining', async () => {
       const mod = await esmock('../../src/content/clone.cmd.js', {
         '../../src/git-utils.js': { default: { getOriginURL: async () => null } },
@@ -96,6 +107,34 @@ describe('CloneCommand', () => {
       assert.strictEqual(cmd.withForce(false), cmd);
       assert.strictEqual(cmd.withRootPath('/'), cmd);
       assert.strictEqual(cmd.withAssumeYes(false), cmd);
+      assert.strictEqual(cmd.withIncludeMedia(false), cmd);
+    });
+  });
+
+  describe('filterFilesForContentClone()', () => {
+    it('keeps only html and json when includeMedia is false', () => {
+      const files = [
+        { path: '/o/r/a.html', name: 'a.html', ext: 'html' },
+        { path: '/o/r/b.json', name: 'b.json', ext: 'json' },
+        { path: '/o/r/c.jpg', name: 'c.jpg', ext: 'jpg' },
+      ];
+      const out = filterFilesForContentClone(files, false);
+      assert.strictEqual(out.length, 2);
+      assert.ok(out.every((f) => f.ext === 'html' || f.ext === 'json'));
+    });
+
+    it('is case-insensitive on extension', () => {
+      const files = [{ path: '/o/r/x.HTML', name: 'x.HTML', ext: 'HTML' }];
+      const out = filterFilesForContentClone(files, false);
+      assert.strictEqual(out.length, 1);
+    });
+
+    it('returns all files when includeMedia is true', () => {
+      const files = [
+        { path: '/o/r/a.html', name: 'a.html', ext: 'html' },
+        { path: '/o/r/v.mp4', name: 'v.mp4', ext: 'mp4' },
+      ];
+      assert.strictEqual(filterFilesForContentClone(files, true).length, 2);
     });
   });
 
@@ -176,6 +215,38 @@ describe('CloneCommand', () => {
       assert.strictEqual(content, '<html>page</html>');
     });
 
+    it('by default skips non-HTML/JSON files', async () => {
+      const DaClientClass = createDaClientClass({
+        files: [
+          { path: '/myorg/myrepo/page.html', name: 'page.html', ext: 'html' },
+          { path: '/myorg/myrepo/hero.jpg', name: 'hero.jpg', ext: 'jpg' },
+        ],
+        sourceContent: 'x',
+      });
+      const cmd = await makeCloneCommand(testRoot, DaClientClass);
+      cmd.withRootPath('/');
+      await cmd.run();
+
+      assert.ok(await fse.pathExists(path.join(testRoot, CONTENT_DIR, 'page.html')));
+      assert.ok(!await fse.pathExists(path.join(testRoot, CONTENT_DIR, 'hero.jpg')));
+    });
+
+    it('withIncludeMedia downloads non-HTML/JSON files', async () => {
+      const DaClientClass = createDaClientClass({
+        files: [
+          { path: '/myorg/myrepo/page.html', name: 'page.html', ext: 'html' },
+          { path: '/myorg/myrepo/hero.jpg', name: 'hero.jpg', ext: 'jpg' },
+        ],
+        sourceContent: 'payload',
+      });
+      const cmd = await makeCloneCommand(testRoot, DaClientClass);
+      cmd.withIncludeMedia(true).withRootPath('/');
+      await cmd.run();
+
+      assert.ok(await fse.pathExists(path.join(testRoot, CONTENT_DIR, 'page.html')));
+      assert.ok(await fse.pathExists(path.join(testRoot, CONTENT_DIR, 'hero.jpg')));
+    });
+
     it('writes .da-config.json with org, repo, and rootPath', async () => {
       const cmd = await makeCloneCommand(testRoot, createDaClientClass());
       cmd.withRootPath('/blog');

test/content/content-commands.test.js

@@ -69,7 +69,7 @@ describe('clone()', () => {
     assert.ok(cmd.description && cmd.description.length > 0);
   });
 
-  it('has a builder that registers path, all, token, force, and yes options', () => {
+  it('has a builder that registers path, all, token, media, force, and yes options', () => {
     const cmd = clone();
     const registered = {};
     const chainable = {
@@ -84,6 +84,7 @@ describe('clone()', () => {
     assert.ok('path' in registered);
     assert.ok('all' in registered);
     assert.ok('token' in registered);
+    assert.ok('media' in registered);
     assert.ok('force' in registered);
     assert.ok('yes' in registered);
   });
@@ -94,6 +95,7 @@ describe('clone()', () => {
       withToken: () => mockExecutor,
       withForce: () => mockExecutor,
       withAssumeYes: () => mockExecutor,
+      withIncludeMedia: () => mockExecutor,
       withRootPath: () => mockExecutor,
       run: async () => {},
     };
@@ -109,6 +111,7 @@ describe('clone()', () => {
     const cmd = clone();
     let ranWith;
     let assumeYesArg;
+    let includeMediaArg;
     let rootPathArg;
     cmd.executor = {
       withToken: (t) => {
@@ -118,9 +121,14 @@ describe('clone()', () => {
             withAssumeYes: (y) => {
               assumeYesArg = y;
               return {
-                withRootPath: (rp) => {
-                  rootPathArg = rp;
-                  return { run: async () => {} };
+                withIncludeMedia: (m) => {
+                  includeMediaArg = m;
+                  return {
+                    withRootPath: (rp) => {
+                      rootPathArg = rp;
+                      return { run: async () => {} };
+                    },
+                  };
                 },
               };
             },
@@ -129,10 +137,11 @@ describe('clone()', () => {
       },
     };
     await cmd.handler({
-      token: 'abc', force: false, yes: true, all: false, path: '/ca/fr_ca',
+      token: 'abc', force: false, yes: true, media: true, all: false, path: '/ca/fr_ca',
     });
     assert.strictEqual(ranWith, 'abc');
     assert.strictEqual(assumeYesArg, true);
+    assert.strictEqual(includeMediaArg, true);
     assert.strictEqual(rootPathArg, '/ca/fr_ca');
   });
 });