Token leadingTrivia handling and reduce memory retention in long-lived compiler/LSP caches

src/CrossScopeValidator.ts

@@ -278,7 +278,7 @@ export class CrossScopeValidator {
             return this.providedTreeMap.get(scope.name);
         }
         const providedTree = new ProvidedNode('', this.componentsMap);
-        const duplicatesMap = new Map<string, Set<FileSymbolPair>>();
+        let duplicatesMap: Map<string, Set<FileSymbolPair>> = null;
 
         const referenceTypesMap = new Map<{ symbolName: string; file: BscFile; symbolObj: ProvidedSymbol }, Array<{ name: string; namespacedName?: string }>>();
 
@@ -289,9 +289,12 @@ export class CrossScopeValidator {
             const symbolIsNamespace = providedTree.getNamespace(symbolName);
             const isDupe = providedTree.addSymbol(symbolName, { file: file, symbol: symbolObj.symbol });
             if (symbolIsNamespace || globalSymbol || isDupe || symbolObj.duplicates.length > 0) {
-                let dupesSet = duplicatesMap.get(symbolName);
+                let dupesSet = duplicatesMap?.get(symbolName) ?? null;
                 if (!dupesSet) {
                     dupesSet = new Set<{ file: BrsFile; symbol: BscSymbol }>();
+                    if (!duplicatesMap) {
+                        duplicatesMap = new Map<string, Set<FileSymbolPair>>();
+                    }
                     duplicatesMap.set(symbolName, dupesSet);
                     const existing = providedTree.getSymbol(symbolName);
                     if (existing) {
@@ -539,7 +542,7 @@ export class CrossScopeValidator {
             });
 
             const { missingSymbols, duplicatesMap } = this.getIssuesForScope(scope);
-            if (addDuplicateSymbolDiagnostics) {
+            if (addDuplicateSymbolDiagnostics && duplicatesMap) {
                 for (const [_flag, dupeSet] of duplicatesMap.entries()) {
                     if (dupeSet.size > 1) {
 

src/Scope.spec.ts

@@ -205,6 +205,16 @@ describe('Scope', () => {
         expect(scopes.length).to.equal(2);
     });
 
+    it('clears cache when disposed', () => {
+        const scope = program.getScopeByName('source');
+        scope.getAllFiles();
+        expect(scope['cache'].size).to.be.greaterThan(0);
+
+        scope.dispose();
+
+        expect(scope['cache'].size).to.equal(0);
+    });
+
     describe('addFile', () => {
         it('detects callables from all loaded files', () => {
             const sourceScope = program.getScopeByName('source');

src/Scope.ts

@@ -528,6 +528,8 @@ export class Scope {
      * Clean up all event handles
      */
     public dispose() {
+        this.unlinkSymbolTable();
+        this.cache.clear();
         this.unsubscribeFromDependencyGraph?.();
     }
 

src/astUtils/visitors.spec.ts

@@ -1332,7 +1332,7 @@ describe('astUtils visitors', () => {
 
             file.ast.walk(createVisitor({
                 AstNode: (node) => {
-                    const endNodeComments = node.endTrivia.filter(t => t.kind === TokenKind.Comment);
+                    const endNodeComments = node.endTrivia?.filter(t => t.kind === TokenKind.Comment) ?? [];
                     comments.push(...endNodeComments);
                 }
             }), {

src/bscPlugin/completions/CompletionsProcessor.ts

@@ -188,7 +188,7 @@ export class CompletionsProcessor {
         const tokenKind = currentToken?.kind;
 
         //if cursor is after a comment, disable completions
-        if (this.isPostionInComment(file, position)) {
+        if (this.isPositionInComment(file, position)) {
             return emptyResult;
         }
 
@@ -643,7 +643,7 @@ export class CompletionsProcessor {
         ];
     }
 
-    private isPostionInComment(file: BrsFile, position: Position) {
+    private isPositionInComment(file: BrsFile, position: Position) {
         const currentToken = file.getCurrentOrNextTokenAt(position);
         const tokenKind = currentToken?.kind;
         if (!currentToken) {
@@ -654,7 +654,7 @@ export class CompletionsProcessor {
         }
 
         const nextNonComment = file.getNextTokenByPredicate(currentToken, (t: Token) => !AllowedTriviaTokens.includes(t.kind), 1);
-        const firstComment = nextNonComment?.leadingTrivia.find(t => t.kind === TokenKind.Comment);
+        const firstComment = nextNonComment?.leadingTrivia?.find(t => t.kind === TokenKind.Comment);
         if (firstComment && util.comparePosition(position, firstComment?.location?.range.start) >= 0) {
             return true;
         }

src/files/BrsFile.spec.ts

@@ -103,6 +103,18 @@ describe('BrsFile', () => {
             program.validate();
             expectZeroDiagnostics(program);
         });
+
+        it('clears cached lookups', () => {
+            const file = program.setFile('source/main.bs', `sub main(): end sub`);
+            // prime lookup cache
+            void file['_cachedLookups'].functionStatements;
+            const cache = file['_cachedLookups']['cache'];
+            expect(cache.size).to.be.greaterThan(0);
+
+            file.dispose();
+
+            expect(cache.size).to.equal(0);
+        });
     });
 
     describe('allowBrighterScriptInBrightScript', () => {
@@ -3455,7 +3467,7 @@ describe('BrsFile', () => {
                         event.file.ast.walk(createVisitor({
                             AstNode: (node) => {
                                 //delete all trivia from the node
-                                for (let i = 0; i < (node.leadingTrivia.length ?? 0); i++) {
+                                for (let i = 0; i < (node.leadingTrivia?.length ?? 0); i++) {
                                     delete node.leadingTrivia[i];
                                 }
                                 for (let i = 0; i < (node.endTrivia.length ?? 0); i++) {

src/files/BrsFile.ts

@@ -1386,6 +1386,7 @@ export class BrsFile implements BscFile {
     }
 
     public dispose() {
+        this._cachedLookups?.invalidate();
         this._parser?.dispose();
 
         //deleting these properties result in lower memory usage (garbage collection is magic!)

src/lexer/Lexer.spec.ts

@@ -1456,7 +1456,7 @@ describe('lexer', () => {
             return tokens
                 //exclude the explicit triva tokens since they'll be included in the leading/trailing arrays
                 .filter(x => !AllowedTriviaTokens.includes(x.kind))
-                .flatMap(x => [...x.leadingTrivia, x])
+                .flatMap(x => [...x.leadingTrivia ?? [], x])
                 .map(x => x.text)
                 .join('');
         }
@@ -1476,13 +1476,37 @@ describe('lexer', () => {
             ).to.eql(input);
         });
 
+        it('tokens only have leadingTrivia array when they have at least one trivia item', () => {
+            const input = `
+                function   test(  )
+                    'comment
+                    print   "alpha"
+                    x = 1 + 2
+                    '  blabla
+                    print x
+                end function`;
+            const tokens = Lexer.scan(input).tokens;
+            let numWithTrivia = 0;
+            for (const token of tokens) {
+                if (token.leadingTrivia) {
+                    expect(token.leadingTrivia.length).to.greaterThan(0);
+                    numWithTrivia++;
+                } else {
+                    expect(token.leadingTrivia).to.be.undefined;
+                }
+
+            }
+            expect(numWithTrivia).to.be.lessThan(tokens.length);
+        });
+
+
         function expectTrivia(text: string, expected: Array<{ text: string; leadingTrivia?: string[]; trailingTrivia?: string[] }>) {
             const tokens = Lexer.scan(text).tokens.filter(x => !AllowedTriviaTokens.includes(x.kind));
             expect(
                 tokens.map(x => {
                     return {
                         text: x.text,
-                        leadingTrivia: x.leadingTrivia.map(x => x.text)
+                        leadingTrivia: x.leadingTrivia?.map(x => x.text) ?? []
                     };
                 })
             ).to.eql(

src/lexer/Lexer.ts

@@ -123,7 +123,7 @@ export class Lexer {
                 ? util.createLocation(this.lineBegin, this.columnBegin, this.lineEnd, this.columnEnd + 1, this.uri)
                 : undefined,
             leadingWhitespace: this.leadingWhitespace,
-            leadingTrivia: this.leadingTrivia ?? []
+            leadingTrivia: this.leadingTrivia.length > 0 ? this.leadingTrivia : undefined
         });
         this.leadingWhitespace = '';
         return this;
@@ -1079,13 +1079,13 @@ export class Lexer {
             isReserved: ReservedWords.has(text.toLowerCase()),
             location: this.locationOf(),
             leadingWhitespace: this.leadingWhitespace,
-            leadingTrivia: []
+            leadingTrivia: undefined
         };
 
         if (this.isTrivia(token)) {
             this.pushTrivia(token);
-        } else {
-            token.leadingTrivia.push(...this.leadingTrivia);
+        } else if (this.leadingTrivia.length > 0) {
+            token.leadingTrivia = [...this.leadingTrivia];
             this.leadingTrivia = [];
         }
         this.leadingWhitespace = '';

src/lexer/Token.ts

@@ -30,7 +30,7 @@ export interface Token {
     /**
      * Any tokens starting on the next line of the previous token, up to the start of this token
      */
-    leadingTrivia: Token[];
+    leadingTrivia?: Token[];
 }
 
 /**

src/parser/Statement.ts

@@ -928,14 +928,14 @@ export class PrintStatement extends Statement {
         ] as TranspileResult;
 
         //if the first expression has no leading whitespace, add a single space between the `print` and the expression
-        if (this.expressions.length > 0 && !this.expressions[0].leadingTrivia.find(t => t?.kind === TokenKind.Whitespace)) {
+        if (this.expressions.length > 0 && !this.expressions[0].leadingTrivia?.find(t => t?.kind === TokenKind.Whitespace)) {
             result.push(' ');
         }
 
         // eslint-disable-next-line @typescript-eslint/prefer-for-of
         for (let i = 0; i < this.expressions.length; i++) {
             const expression = this.expressions[i];
-            let leadingWhitespace = expression.leadingTrivia.find(t => t?.kind === TokenKind.Whitespace)?.text;
+            let leadingWhitespace = expression.leadingTrivia?.find(t => t?.kind === TokenKind.Whitespace)?.text;
             if (leadingWhitespace) {
                 result.push(leadingWhitespace);
                 //if the previous expression was NOT a separator, and this one is not also, add a space between them
@@ -2936,7 +2936,7 @@ export class ClassStatement extends Statement implements TypedefProvider {
                 state.classStatement = this;
                 state.skipLeadingComments = true;
                 //add leading comments
-                if (statement.leadingTrivia.filter(token => token.kind === TokenKind.Comment).length > 0) {
+                if ((statement.leadingTrivia?.filter(token => token.kind === TokenKind.Comment) ?? []).length > 0) {
                     result.push(
                         ...state.transpileComments(statement.leadingTrivia),
                         state.indent()

src/parser/TranspileState.ts

@@ -113,6 +113,9 @@ export class TranspileState {
 
     public transpileLeadingCommentsForAstNode(node: { leadingTrivia?: Token[] }) {
         const leadingTrivia = node?.leadingTrivia ?? [];
+        if (!leadingTrivia || leadingTrivia.length === 0) {
+            return [];
+        }
         const leadingCommentsSourceNodes = this.transpileComments(leadingTrivia);
         if (leadingCommentsSourceNodes.length > 0) {
             // indent in preparation for next text
@@ -123,7 +126,10 @@ export class TranspileState {
     }
 
     public transpileLeadingComments(token: TranspileToken) {
-        const leadingTrivia = (token?.leadingTrivia ?? []);
+        const leadingTrivia = token?.leadingTrivia ?? [];
+        if (!leadingTrivia || leadingTrivia.length === 0) {
+            return [];
+        }
         const leadingCommentsSourceNodes = this.transpileComments(leadingTrivia);
         if (leadingCommentsSourceNodes.length > 0 && token.text) {
             // indent in preparation for next text

src/util.spec.ts

@@ -1659,13 +1659,15 @@ describe('util', () => {
     describe('standardizePath', () => {
         let isWindowsOrig = util['isWindows'];
         let isWindows = isWindowsOrig;
+        let standardizePathCacheLimitOrig = util['standardizePathCacheLimit'];
 
         beforeEach(() => {
             util['standardizePathCache'].clear();
         });
         afterEach(() => {
             util['standardizePathCache'].clear();
             util['isWindows'] = isWindowsOrig;
+            util['standardizePathCacheLimit'] = standardizePathCacheLimitOrig;
         });
 
         function test(incoming: string, expected: string) {

src/util.ts

@@ -792,10 +792,14 @@ export class Util {
     public pathToUri(filePath: string) {
         if (!filePath) {
             return filePath;
+        } else if (this.pathToURICache.has(filePath)) {
+            return this.pathToURICache.get(filePath);
         } else if (this.isUriLike(filePath)) {
             return filePath;
         } else {
-            return URI.file(filePath).toString();
+            const uri = URI.file(filePath).toString();
+            this.pathToURICache.set(filePath, uri);
+            return uri;
         }
     }
 
@@ -1077,7 +1081,7 @@ export class Util {
                 text: token.text,
                 isReserved: token.isReserved,
                 leadingWhitespace: token.leadingWhitespace,
-                leadingTrivia: token.leadingTrivia.map(x => this.cloneToken(x))
+                leadingTrivia: token.leadingTrivia ? token.leadingTrivia.map(x => this.cloneToken(x)) : undefined
             } as Token;
             //handle those tokens that have charCode
             if ('charCode' in token) {
@@ -1837,6 +1841,8 @@ export class Util {
     private isWindows = process.platform === 'win32';
     private standardizePathCache = new Map<string, string>();
 
+    private pathToURICache = new Map<string, string>();
+
     /**
      * Converts a path into a standardized format (drive letter to lower, remove extra slashes, use single slash type, resolve relative parts, etc...)
      */
@@ -2527,7 +2533,7 @@ export class Util {
 
     public hasLeadingComments(input: Token | AstNode) {
         const leadingTrivia = isToken(input) ? input?.leadingTrivia : input?.leadingTrivia ?? [];
-        return !!leadingTrivia.find(t => t?.kind === TokenKind.Comment);
+        return !!leadingTrivia?.find(t => t?.kind === TokenKind.Comment);
     }
 
     public getLeadingComments(input: Token | AstNode) {