Merge pull request #1263 from forki/clean-lex

Clean lexer

Author: KevinRansom

src/fsharp/CompileOptions.fs

@@ -891,7 +891,7 @@ let compilingFsLib20Flag (tcConfigB : TcConfigBuilder) =
 let compilingFsLib40Flag (tcConfigB : TcConfigBuilder) = 
         CompilerOption("compiling-fslib-40", tagNone, OptionUnit (fun () -> tcConfigB.compilingFslib40 <- true; ), Some(InternalCommandLineOption("--compiling-fslib-40", rangeCmdArgs)), None)
 let mlKeywordsFlag = 
-        CompilerOption("ml-keywords", tagNone, OptionUnit (fun () -> Lexhelp.Keywords.permitFsharpKeywords <- false), Some(DeprecatedCommandLineOptionNoDescription("--ml-keywords", rangeCmdArgs)), None)
+        CompilerOption("ml-keywords", tagNone, OptionUnit (fun () -> ()), Some(DeprecatedCommandLineOptionNoDescription("--ml-keywords", rangeCmdArgs)), None)
 
 let gnuStyleErrorsFlag tcConfigB = 
         CompilerOption("gnu-style-errors", tagNone, OptionUnit (fun () -> tcConfigB.errorStyle <- ErrorStyle.EmacsErrors), Some(DeprecatedCommandLineOptionNoDescription("--gnu-style-errors", rangeCmdArgs)), None)

src/fsharp/lexhelp.fs

@@ -305,48 +305,44 @@ module Keywords =
         keywordList |> List.map (fun (_, w, _) -> w) 
 
     let keywordTable = 
-        // TODO: this doesn't need to be a multi-map, a dictionary will do
         let tab = System.Collections.Generic.Dictionary<string,token>(100)
-        for (_mode,keyword,token) in keywordList do tab.Add(keyword,token)
+        for _,keyword,token in keywordList do 
+            tab.Add(keyword,token)
         tab
 
     let KeywordToken s = keywordTable.[s]
 
-    /// ++GLOBAL MUTABLE STATE. Note this is a deprecated, undocumented command line option anyway, we can ignore it.
-    let mutable permitFsharpKeywords = true
-
     let IdentifierToken args (lexbuf:UnicodeLexing.Lexbuf) (s:string) =
         if IsCompilerGeneratedName s then 
             warning(Error(FSComp.SR.lexhlpIdentifiersContainingAtSymbolReserved(), lexbuf.LexemeRange));
         args.resourceManager.InternIdentifierToken s
 
     let KeywordOrIdentifierToken args (lexbuf:UnicodeLexing.Lexbuf) s =
-        if not permitFsharpKeywords && List.contains s unreserveWords then
-            // You can assume this condition never fires - this is a deprecated, undocumented command line option anyway, we can ignore it.
-            IdentifierToken args lexbuf s
-        else
-          let mutable v = Unchecked.defaultof<_>
-          if keywordTable.TryGetValue(s, &v) then 
-              if (match v with RESERVED -> true | _ -> false) then
-                  warning(ReservedKeyword(FSComp.SR.lexhlpIdentifierReserved(s), lexbuf.LexemeRange));
-                  IdentifierToken args lexbuf s
-              else v
-          else 
+        match keywordTable.TryGetValue s with
+        | true,v ->
+            match v with 
+            | RESERVED ->
+                warning(ReservedKeyword(FSComp.SR.lexhlpIdentifierReserved(s), lexbuf.LexemeRange));
+                IdentifierToken args lexbuf s
+            | _ -> v
+        | _ ->
             match s with 
             | "__SOURCE_DIRECTORY__" ->
                 let filename = fileOfFileIndex lexbuf.StartPos.FileIndex
-                let dirname  = if filename = stdinMockFilename then
-                                   System.IO.Directory.GetCurrentDirectory()
-                               else
-                                   filename |> FileSystem.GetFullPathShim (* asserts that path is already absolute *)
-                                            |> System.IO.Path.GetDirectoryName
+                let dirname = 
+                    if filename = stdinMockFilename then
+                        System.IO.Directory.GetCurrentDirectory()
+                    else
+                        filename 
+                        |> FileSystem.GetFullPathShim (* asserts that path is already absolute *)
+                        |> System.IO.Path.GetDirectoryName
                 KEYWORD_STRING dirname
             | "__SOURCE_FILE__" -> 
-               KEYWORD_STRING (System.IO.Path.GetFileName((fileOfFileIndex lexbuf.StartPos.FileIndex))) 
+                KEYWORD_STRING (System.IO.Path.GetFileName((fileOfFileIndex lexbuf.StartPos.FileIndex))) 
             | "__LINE__" -> 
-               KEYWORD_STRING (string lexbuf.StartPos.Line)
+                KEYWORD_STRING (string lexbuf.StartPos.Line)
             | _ -> 
-               IdentifierToken args lexbuf s
+                IdentifierToken args lexbuf s
 
     /// A utility to help determine if an identifier needs to be quoted 
     let QuoteIdentifierIfNeeded (s : string) : string =

src/fsharp/lexhelp.fsi

@@ -67,5 +67,4 @@ module Keywords =
     val KeywordOrIdentifierToken : lexargs -> UnicodeLexing.Lexbuf -> string -> Parser.token
     val IdentifierToken : lexargs -> UnicodeLexing.Lexbuf -> string -> Parser.token
     val QuoteIdentifierIfNeeded : string -> string
-    val mutable permitFsharpKeywords : bool 
     val keywordNames : string list