Merge pull request #21 from microsoft/fix/tokenizer-camelcase-keyword

fix: tokenizer word_continuation case sensitivity for camelCase ident…

Author: niclasko

flowquery-py/src/tokenization/string_walker.py

@@ -155,4 +155,4 @@ def word_continuation(self, word: str) -> bool:
         if next_pos >= len(self._text):
             return False
         next_char = self._text[next_pos]
-        return next_char in StringUtils.word_valid_chars
+        return next_char.lower() in StringUtils.word_valid_chars

flowquery-py/tests/parsing/test_parser.py

@@ -325,6 +325,52 @@ def test_lookup_with_reserved_keyword_property_names(self):
         )
         assert ast.print() == expected
 
+    def test_lookup_with_from_keyword_as_property_name(self):
+        """Test lookup with from and to keywords as property names."""
+        parser = Parser()
+        ast = parser.parse("with {from: 1, to: 2} as x return x.from, x.to")
+        expected = (
+            "ASTNode\n"
+            "- With\n"
+            "-- Expression (x)\n"
+            "--- AssociativeArray\n"
+            "---- KeyValuePair\n"
+            "----- String (from)\n"
+            "----- Expression\n"
+            "------ Number (1)\n"
+            "---- KeyValuePair\n"
+            "----- String (to)\n"
+            "----- Expression\n"
+            "------ Number (2)\n"
+            "- Return\n"
+            "-- Expression\n"
+            "--- Lookup\n"
+            "---- Identifier (from)\n"
+            "---- Reference (x)\n"
+            "-- Expression\n"
+            "--- Lookup\n"
+            "---- Identifier (to)\n"
+            "---- Reference (x)"
+        )
+        assert ast.print() == expected
+
+    def test_camel_case_alias_starting_with_keyword(self):
+        """Test that camelCase identifiers starting with a keyword (e.g. fromUser) are tokenized correctly."""
+        parser = Parser()
+        ast = parser.parse("LOAD JSON FROM '/data.json' AS x RETURN x.from AS fromUser")
+        output = ast.print()
+        assert "Lookup" in output
+        assert "Identifier (from)" in output
+
+    def test_from_keyword_property_in_create_virtual_subquery(self):
+        """Test that email.from parses correctly inside a CREATE VIRTUAL subquery."""
+        parser = Parser()
+        # Should not raise - email.from should be parsed correctly even with FROM being a keyword
+        parser.parse(
+            "CREATE VIRTUAL (:Email) AS { LOAD JSON FROM '/data/emails.json' AS email "
+            "RETURN email.id AS id, email.from AS fromUser }"
+        )
+
     def test_load_with_post(self):
         """Test load with post."""
         parser = Parser()

src/tokenization/string_walker.ts

@@ -2,10 +2,10 @@ import StringUtils from "../utils/string_utils";
 
 /**
  * Utility class for walking through a string character by character during tokenization.
- * 
+ *
  * Provides methods to check for specific character patterns, move through the string,
  * and extract substrings. Used by the Tokenizer to process input text.
- * 
+ *
  * @example
  * ```typescript
  * const walker = new StringWalker("WITH x as variable");
@@ -20,7 +20,7 @@ class StringWalker {
 
     /**
      * Creates a new StringWalker for the given text.
-     * 
+     *
      * @param text - The input text to walk through
      */
     constructor(text: string) {
@@ -81,42 +81,45 @@ class StringWalker {
     }
 
     public singleLineCommentStart(): boolean {
-        return this.currentChar === '/' && this.nextChar === '/';
+        return this.currentChar === "/" && this.nextChar === "/";
     }
 
     public multiLineCommentStart(): boolean {
-        return this.currentChar === '/' && this.nextChar === '*';
+        return this.currentChar === "/" && this.nextChar === "*";
     }
 
     public multiLineCommentEnd(): boolean {
-        return this.currentChar === '*' && this.nextChar === '/';
+        return this.currentChar === "*" && this.nextChar === "/";
     }
 
     public newLine(): boolean {
-        if (this.currentChar === '\n') {
+        if (this.currentChar === "\n") {
             return true;
         }
         return false;
     }
 
     public escaped(char: string): boolean {
-        return this.currentChar === '\\' && this.nextChar === char;
+        return this.currentChar === "\\" && this.nextChar === char;
     }
 
     public escapedBrace(): boolean {
-        return (this.currentChar === '{' && this.nextChar === '{') || (this.currentChar === '}' && this.nextChar === '}');
+        return (
+            (this.currentChar === "{" && this.nextChar === "{") ||
+            (this.currentChar === "}" && this.nextChar === "}")
+        );
     }
 
     public openingBrace(): boolean {
-        return this.currentChar === '{';
+        return this.currentChar === "{";
     }
 
     public closingBrace(): boolean {
-        return this.currentChar === '}';
+        return this.currentChar === "}";
     }
 
     public checkForUnderScore(): boolean {
-        const foundUnderScore = this.currentChar === '_';
+        const foundUnderScore = this.currentChar === "_";
         if (foundUnderScore) {
             this._position++;
         }
@@ -141,7 +144,7 @@ class StringWalker {
 
     public checkForQuote(): string | null {
         const quoteChar = this.currentChar;
-        if (quoteChar === '"' || quoteChar === "'" || quoteChar === '`') {
+        if (quoteChar === '"' || quoteChar === "'" || quoteChar === "`") {
             this._position++;
             return quoteChar;
         }
@@ -163,7 +166,7 @@ class StringWalker {
     }
 
     public checkForFStringStart(): boolean {
-        return this.currentChar.toLowerCase() === 'f' && ['\'', '"', '`'].includes(this.nextChar);
+        return this.currentChar.toLowerCase() === "f" && ["'", '"', "`"].includes(this.nextChar);
     }
 
     public moveNext(): void {
@@ -187,8 +190,8 @@ class StringWalker {
 
     public word_continuation(word: string): boolean {
         const next = this.text[this.position + word.length];
-        return StringUtils.word_valid_chars.includes(next);
+        return next !== undefined && StringUtils.word_valid_chars.includes(next.toLowerCase());
     }
 }
 
-export default StringWalker;
+export default StringWalker;

tests/parsing/parser.test.ts

@@ -264,6 +264,51 @@ test("Test lookup with JSON array", () => {
     expect(_return.firstChild().value()).toBe(2);
 });
 
+test("Test lookup with from keyword as property name", () => {
+    const parser = new Parser();
+    const ast = parser.parse("with {from: 1, to: 2} as x return x.from, x.to");
+    expect(ast.print()).toBe(
+        "ASTNode\n" +
+            "- With\n" +
+            "-- Expression (x)\n" +
+            "--- AssociativeArray\n" +
+            "---- KeyValuePair\n" +
+            "----- String (from)\n" +
+            "----- Expression\n" +
+            "------ Number (1)\n" +
+            "---- KeyValuePair\n" +
+            "----- String (to)\n" +
+            "----- Expression\n" +
+            "------ Number (2)\n" +
+            "- Return\n" +
+            "-- Expression\n" +
+            "--- Lookup\n" +
+            "---- Identifier (from)\n" +
+            "---- Reference (x)\n" +
+            "-- Expression\n" +
+            "--- Lookup\n" +
+            "---- Identifier (to)\n" +
+            "---- Reference (x)"
+    );
+});
+
+test("Test camelCase alias starting with keyword", () => {
+    const parser = new Parser();
+    const ast = parser.parse("LOAD JSON FROM '/data.json' AS x RETURN x.from AS fromUser");
+    expect(ast.print()).toContain("Lookup");
+    expect(ast.print()).toContain("Identifier (from)");
+});
+
+test("Test from keyword property in create virtual subquery", () => {
+    const parser = new Parser();
+    // Should not throw - email.from should be parsed correctly even with FROM being a keyword
+    expect(() => {
+        parser.parse(
+            "CREATE VIRTUAL (:Email) AS { LOAD JSON FROM '/data/emails.json' AS email RETURN email.id AS id, email.from AS fromUser }"
+        );
+    }).not.toThrow();
+});
+
 test("Test lookup with reserved keyword property names", () => {
     const parser = new Parser();
     const ast = parser.parse("with {end: 1, null: 2, case: 3} as x return x.end, x.null, x.case");