Skip to content

Commit ffc237c

Browse files
jakemccloskeyjakemccloskey
committed
Fix on-demand runtime expansion for published source deps
1 parent 42fdccd commit ffc237c

4 files changed

Lines changed: 252 additions & 116 deletions

File tree

‎scripts/release/prepare_npm_test.ts‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -532,6 +532,154 @@ Deno.test('prepare_npm --stdlib-only emits canonical project-transform that lowe
532532
}
533533
});
534534

535+
Deno.test('prepare_npm --stdlib-only project-transform macro-expands source-published dependency modules under Node', async () => {
536+
const { distRoot, tarballPath } = await prepareCanonicalTarball(
537+
'soundscript-prepare-project-transform-dependency-node-',
538+
);
539+
try {
540+
const projectRoot = await Deno.makeTempDir({
541+
prefix: 'soundscript-project-transform-dependency-node-',
542+
});
543+
try {
544+
await writeProjectFile(
545+
projectRoot,
546+
'package.json',
547+
JSON.stringify(
548+
{
549+
name: 'project-transform-dependency-smoke',
550+
private: true,
551+
type: 'module',
552+
},
553+
null,
554+
2,
555+
),
556+
);
557+
await writeProjectFile(
558+
projectRoot,
559+
'tsconfig.json',
560+
JSON.stringify(
561+
{
562+
compilerOptions: {
563+
strict: true,
564+
noEmit: true,
565+
target: 'ES2022',
566+
module: 'ESNext',
567+
moduleResolution: 'Bundler',
568+
},
569+
include: ['src/**/*.sts'],
570+
},
571+
null,
572+
2,
573+
),
574+
);
575+
await writeProjectFile(
576+
projectRoot,
577+
'src/main.sts',
578+
[
579+
"import { encoded } from 'example-pkg';",
580+
'export const value = encoded;',
581+
'',
582+
].join('\n'),
583+
);
584+
await installNpmTarballs(projectRoot, [tarballPath]);
585+
586+
await writeProjectFile(
587+
projectRoot,
588+
'node_modules/example-pkg/package.json',
589+
JSON.stringify(
590+
{
591+
name: 'example-pkg',
592+
version: '1.0.0',
593+
type: 'module',
594+
soundscript: {
595+
version: 1,
596+
exports: {
597+
'.': { source: './src/index.sts' },
598+
},
599+
},
600+
},
601+
null,
602+
2,
603+
),
604+
);
605+
await writeProjectFile(
606+
projectRoot,
607+
'node_modules/example-pkg/src/contracts.sts',
608+
[
609+
"import { codec } from 'sts:derive';",
610+
'',
611+
'// #[codec]',
612+
'export interface User {',
613+
' readonly id: string;',
614+
'}',
615+
'',
616+
].join('\n'),
617+
);
618+
await writeProjectFile(
619+
projectRoot,
620+
'node_modules/example-pkg/src/shared.sts',
621+
[
622+
"import { UserCodec } from './contracts.sts';",
623+
"export const encoded = UserCodec.encode({ id: 'user-1' });",
624+
'',
625+
].join('\n'),
626+
);
627+
await writeProjectFile(
628+
projectRoot,
629+
'node_modules/example-pkg/src/index.sts',
630+
"export { encoded } from './shared.sts';\n",
631+
);
632+
633+
const loadResult = await runCommand(
634+
'node',
635+
[
636+
'--input-type=module',
637+
'--eval',
638+
[
639+
"import { dirname, join } from 'node:path';",
640+
"import { createOnDemandTransformer } from '@soundscript/soundscript/project-transform';",
641+
'const projectRoot = process.cwd();',
642+
'const transformer = createOnDemandTransformer({ workingDirectory: projectRoot });',
643+
"const entryPath = join(projectRoot, 'src', 'main.sts');",
644+
"const packageEntry = transformer.resolveImportSpecifier('example-pkg', entryPath);",
645+
"const sharedPath = transformer.resolveImportSpecifier('./shared.sts', packageEntry);",
646+
"const contractsPath = transformer.resolveImportSpecifier('./contracts.sts', sharedPath);",
647+
'const transformedContracts = transformer.transformModuleSync(contractsPath);',
648+
'const transformedShared = transformer.transformModuleSync(sharedPath);',
649+
'console.log(JSON.stringify({',
650+
' packageEntry,',
651+
' sharedPath,',
652+
' contractsCode: transformedContracts.code,',
653+
' sharedCode: transformedShared.code,',
654+
'}));',
655+
].join('\n'),
656+
],
657+
{ cwd: projectRoot },
658+
);
659+
assertEquals(
660+
loadResult.success,
661+
true,
662+
`node project-transform dependency smoke failed.\nstdout:\n${loadResult.stdout}\nstderr:\n${loadResult.stderr}`,
663+
);
664+
const payload = JSON.parse(loadResult.stdout) as {
665+
contractsCode?: string;
666+
packageEntry?: string;
667+
sharedCode?: string;
668+
sharedPath?: string;
669+
};
670+
assertEquals((payload.packageEntry ?? '').endsWith('/node_modules/example-pkg/src/index.sts'), true);
671+
assertEquals((payload.sharedPath ?? '').endsWith('/node_modules/example-pkg/src/shared.sts'), true);
672+
assertStringIncludes(payload.contractsCode ?? '', 'export const UserCodec =');
673+
assertStringIncludes(payload.sharedCode ?? '', "import { UserCodec } from './contracts.sts';");
674+
assertStringIncludes(payload.sharedCode ?? '', "export const encoded = UserCodec.encode({ id: 'user-1' });");
675+
} finally {
676+
await Deno.remove(projectRoot, { recursive: true }).catch(() => undefined);
677+
}
678+
} finally {
679+
await Deno.remove(distRoot, { recursive: true }).catch(() => undefined);
680+
}
681+
});
682+
535683
Deno.test('prepare_npm --stdlib-only emits runtime stdlib JS that Node can import directly', async () => {
536684
const { distRoot, tarballPath } = await prepareCanonicalTarball(
537685
'soundscript-prepare-stdlib-runtime-node-',

‎src/cli/cli.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ import {
5858
import { runProgram, type RunProgramOptions, type RunProgramResult } from './run_program.ts';
5959
import { projectEditorFile } from '../editor/editor_projection.ts';
6060

61-
export const VERSION = '0.1.27';
61+
export const VERSION = '0.1.28';
6262
const FINDINGS_EXIT_CODE = 1;
6363
const CLI_FAILURE_EXIT_CODE = 2;
6464

‎src/runtime/on_demand.ts‎

Lines changed: 14 additions & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,9 @@ import ts from 'typescript';
22

33
import { createSoundStdlibCompilerHost } from '../bundled/sound_stdlib.ts';
44
import { loadConfig, type LoadedConfig } from '../project/config.ts';
5-
import {
6-
getAlwaysAvailableBuiltinMacroDefinitions,
7-
getAlwaysAvailableBuiltinMacroExports,
8-
getBuiltinMacroDefinitionsBySpecifier,
9-
getBuiltinMacroExportsBySpecifier,
10-
getBuiltinMacroFactoriesBySpecifier,
11-
} from '../frontend/builtin_macro_support.ts';
12-
import { SemanticMacroExpansionRequiredError } from '../frontend/macro_errors.ts';
13-
import { createProjectMacroEnvironment } from '../frontend/project_macro_support.ts';
5+
import { createBuiltinExpandedProgram } from '../frontend/builtin_macro_support.ts';
146
import { dirname, join } from '../platform/path.ts';
15-
import { createPreparedProgram, toSourceFileName } from '../frontend/project_frontend.ts';
7+
import { toSourceFileName } from '../frontend/project_frontend.ts';
168
import { resolveSoundScriptAwareModule } from '../project/soundscript_packages.ts';
179
import { type RuntimeTransformArtifact, transpileTypeScriptModuleToEsm } from './transform.ts';
1810

@@ -150,11 +142,11 @@ function getProjectContext(
150142
};
151143
}
152144

153-
function createPreparedRuntimeProgram(
145+
function createExpandedRuntimeProgram(
154146
projectContext: TransformProjectContext,
155147
rootNames: readonly string[],
156148
) {
157-
return createPreparedProgram({
149+
return createBuiltinExpandedProgram({
158150
baseHost: createSoundStdlibCompilerHost(
159151
projectContext.loadedConfig.commandLine.options,
160152
dirname(projectContext.projectPath),
@@ -167,80 +159,18 @@ function createPreparedRuntimeProgram(
167159
});
168160
}
169161

170-
function expandPreparedRuntimeProgram(
171-
preparedProgram: ReturnType<typeof createPreparedRuntimeProgram>,
172-
options: { readonly deferToSemanticExpansion: boolean },
173-
): ReadonlyMap<string, ts.SourceFile> {
174-
const macroEnvironment = createProjectMacroEnvironment(
175-
preparedProgram,
176-
getBuiltinMacroDefinitionsBySpecifier(),
177-
getBuiltinMacroExportsBySpecifier(
178-
options.deferToSemanticExpansion ? undefined : preparedProgram,
179-
{ deferToSemanticExpansion: options.deferToSemanticExpansion },
180-
),
181-
getBuiltinMacroFactoriesBySpecifier(),
182-
getAlwaysAvailableBuiltinMacroDefinitions(),
183-
getAlwaysAvailableBuiltinMacroExports(
184-
options.deferToSemanticExpansion ? undefined : preparedProgram,
185-
{ deferToSemanticExpansion: options.deferToSemanticExpansion },
186-
),
187-
{ deferToSemanticExpansion: options.deferToSemanticExpansion },
188-
);
189-
try {
190-
return macroEnvironment.expandPreparedProgram();
191-
} finally {
192-
macroEnvironment.dispose();
193-
}
194-
}
195-
196162
export function createOnDemandTransformer(
197163
options: OnDemandTransformerOptions = {},
198164
): SyncOnDemandTransformer {
199165
const soundscriptCache = new Map<string, OnDemandTransformResult>();
200166
const typeScriptCache = new Map<string, OnDemandTransformResult>();
201167

202-
function createFastSoundscriptTransform(
203-
fileName: string,
204-
projectContext: TransformProjectContext,
205-
sourceText: string,
206-
): OnDemandTransformResult | null {
207-
const sourceHash = ts.sys.createHash?.(sourceText) ?? sourceText;
208-
const cacheKey = `${projectContext.projectPath}\u0000fast\u0000${fileName}\u0000${sourceHash}`;
209-
const cached = soundscriptCache.get(cacheKey);
210-
if (cached) {
211-
return cached;
212-
}
213-
214-
const preparedProgram = createPreparedRuntimeProgram(projectContext, [fileName]);
215-
try {
216-
const preparedSource = preparedProgram.preparedHost.getPreparedSourceFile(fileName);
217-
if ((preparedSource?.rewriteResult.macrosById.size ?? 0) > 0) {
218-
return null;
219-
}
220-
const artifact = transpileTypeScriptModuleToEsm(
221-
fileName,
222-
`${fileName}.js`,
223-
preparedSource?.rewrittenText ?? sourceText,
224-
{
225-
module: ts.ModuleKind.ES2022,
226-
moduleSpecifierMode: 'preserve',
227-
target: ts.ScriptTarget.ES2022,
228-
},
229-
);
230-
const result = {
231-
...artifact,
232-
projectPath: projectContext.projectPath,
233-
};
234-
soundscriptCache.set(cacheKey, result);
235-
return result;
236-
} finally {
237-
preparedProgram.dispose();
238-
}
239-
}
240-
241168
function transformModuleSync(fileName: string): OnDemandTransformResult {
242169
const projectContext = getProjectContext(fileName, options.projectPath);
243-
if (projectContext && projectContext.loadedConfig.isSoundscriptSourceFile(fileName)) {
170+
const isSoundscriptRuntimeSource = fileName.endsWith('.sts') ||
171+
(projectContext?.loadedConfig.isSoundscriptSourceFile(fileName) ?? false);
172+
173+
if (projectContext && isSoundscriptRuntimeSource) {
244174
const sourceText = ts.sys.readFile(fileName);
245175
if (sourceText === undefined) {
246176
throw new Error(`Could not read source file ${fileName}.`);
@@ -252,24 +182,11 @@ export function createOnDemandTransformer(
252182
if (cachedFull) {
253183
return cachedFull;
254184
}
255-
const fastTransformed = createFastSoundscriptTransform(fileName, projectContext, sourceText);
256-
if (fastTransformed) {
257-
return fastTransformed;
258-
}
259-
const deferredCacheKey =
260-
`${projectContext.projectPath}\u0000deferred\u0000${fileName}\u0000${sourceHash}`;
261-
const cachedDeferred = soundscriptCache.get(deferredCacheKey);
262-
if (cachedDeferred) {
263-
return cachedDeferred;
264-
}
265-
const preparedProgram = createPreparedRuntimeProgram(projectContext, [fileName]);
185+
const expandedProgram = createExpandedRuntimeProgram(projectContext, [fileName]);
266186
try {
267-
const transpileExpanded = (
268-
expandedFiles: ReadonlyMap<string, ts.SourceFile>,
269-
): OnDemandTransformResult => {
270-
const programFileName = preparedProgram.toProgramFileName(fileName);
271-
const sourceFile = expandedFiles.get(programFileName) ??
272-
preparedProgram.program.getSourceFile(programFileName);
187+
const transpileExpanded = (): OnDemandTransformResult => {
188+
const programFileName = expandedProgram.preparedProgram.toProgramFileName(fileName);
189+
const sourceFile = expandedProgram.program.getSourceFile(programFileName);
273190
if (!sourceFile) {
274191
throw new Error(`Missing expanded source file for ${fileName}.`);
275192
}
@@ -289,29 +206,11 @@ export function createOnDemandTransformer(
289206
};
290207
};
291208

292-
try {
293-
const result = transpileExpanded(
294-
expandPreparedRuntimeProgram(preparedProgram, {
295-
deferToSemanticExpansion: true,
296-
}),
297-
);
298-
soundscriptCache.set(deferredCacheKey, result);
299-
return result;
300-
} catch (error) {
301-
if (!(error instanceof SemanticMacroExpansionRequiredError)) {
302-
throw error;
303-
}
304-
}
305-
306-
const result = transpileExpanded(
307-
expandPreparedRuntimeProgram(preparedProgram, {
308-
deferToSemanticExpansion: false,
309-
}),
310-
);
209+
const result = transpileExpanded();
311210
soundscriptCache.set(fullCacheKey, result);
312211
return result;
313212
} finally {
314-
preparedProgram.dispose();
213+
expandedProgram.dispose();
315214
}
316215
}
317216

0 commit comments

Comments
 (0)