Skip to content

Commit 1da0ea6

Browse files
author
Omar Tawfik
committed
Fix PR comments, add tokens cache per line
1 parent 30c92a6 commit 1da0ea6

File tree

4 files changed

+148
-135
lines changed

4 files changed

+148
-135
lines changed

‎vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs‎

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ open Microsoft.VisualStudio.Text.Tagging
2525
open Microsoft.FSharp.Compiler.Parser
2626
open Microsoft.FSharp.Compiler.SourceCodeServices
2727

28+
// TODO: add defines flags if available from project sites and files
29+
2830
[<ExportBraceMatcher(FSharpCommonConstants.FSharpLanguageName)>]
2931
type internal FSharpBraceMatchingService() =
3032

@@ -41,17 +43,17 @@ type internal FSharpBraceMatchingService() =
4143
ClassificationTypeNames.ExcludedCode;
4244
]
4345

44-
static let getBraceMatchingResult(sourceText: SourceText, fileName: Option<string>, position: int, cancellationToken: CancellationToken) : Option<BraceMatchingResult> =
46+
static let getBraceMatchingResult(sourceText: SourceText, fileName: Option<string>, defines: string list, position: int, cancellationToken: CancellationToken) : Option<BraceMatchingResult> =
4547
if position < 0 || position >= sourceText.Length then
4648
None
4749
else
48-
let completeTextSpan = TextSpan(0, sourceText.Length)
49-
let classificationData = FSharpColorizationService.GetColorizationData(sourceText, completeTextSpan, fileName, [], cancellationToken)
50-
5150
let shouldBeIgnored(characterPosition) =
52-
match classificationData.GetClassifiedSpan(characterPosition) with
53-
| None -> false
54-
| Some(classifiedSpan) -> ignoredClassificationTypes |> Seq.contains classifiedSpan.ClassificationType
51+
let textSpan = TextSpan(characterPosition, 1)
52+
let classifiedSpans = FSharpColorizationService.GetColorizationData(sourceText, textSpan, fileName, defines, cancellationToken)
53+
if classifiedSpans.Any() then
54+
ignoredClassificationTypes |> Seq.contains (classifiedSpans.First().ClassificationType)
55+
else
56+
false
5557

5658
if shouldBeIgnored(position) then
5759
None
@@ -69,7 +71,7 @@ type internal FSharpBraceMatchingService() =
6971
else if currentCharacter = rightBrace then Some(proceedToStartOfString, beforeStartOfString, rightBrace, leftBrace)
7072
else None
7173

72-
match supportedBraceTypes |> Seq.tryPick(pickBraceType) with
74+
match supportedBraceTypes |> List.tryPick(pickBraceType) with
7375
| None -> None
7476
| Some(proceedFunc, stoppingCondition, matchedBrace, nonMatchedBrace) ->
7577
let mutable currentPosition = proceedFunc position
@@ -91,19 +93,19 @@ type internal FSharpBraceMatchingService() =
9193

9294
interface IBraceMatcher with
9395
member this.FindBracesAsync(document: Document, position: int, cancellationToken: CancellationToken): Task<Nullable<BraceMatchingResult>> =
94-
let computation() =
95-
let sourceText = document.GetTextAsync(cancellationToken).ConfigureAwait(false).GetAwaiter().GetResult()
96-
try match getBraceMatchingResult(sourceText, Some(document.Name), position, cancellationToken) with
97-
| None -> Nullable()
98-
| Some(braceMatchingResult) -> Nullable(braceMatchingResult)
99-
with ex ->
100-
Assert.Exception(ex)
101-
reraise()
102-
Task.Run(computation, cancellationToken)
103-
96+
document.GetTextAsync(cancellationToken).ContinueWith(
97+
fun (sourceTextTask: Task<SourceText>) ->
98+
try match getBraceMatchingResult(sourceTextTask.Result, Some(document.Name), [], position, cancellationToken) with
99+
| None -> Nullable()
100+
| Some(braceMatchingResult) -> Nullable(braceMatchingResult)
101+
with ex ->
102+
Assert.Exception(ex)
103+
reraise()
104+
, TaskContinuationOptions.OnlyOnRanToCompletion)
105+
104106
// Helper function to proxy Roslyn types to tests
105-
static member FindMatchingBrace(sourceText: SourceText, fileName: Option<string>, position: int, cancellationToken: CancellationToken) : Option<int> =
106-
match getBraceMatchingResult(sourceText, fileName, position, cancellationToken) with
107+
static member FindMatchingBrace(sourceText: SourceText, fileName: Option<string>, defines: string list, position: int, cancellationToken: CancellationToken) : Option<int> =
108+
match getBraceMatchingResult(sourceText, fileName, defines, position, cancellationToken) with
107109
| None -> None
108110
| Some(braceMatchingResult) ->
109111
if braceMatchingResult.LeftSpan.Start = position then

‎vsintegration/src/FSharp.Editor/FSharpColorizationService.fs‎

Lines changed: 109 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -28,116 +28,129 @@ open Microsoft.FSharp.Compiler.SourceCodeServices
2828
// TODO: add types colorization if available from intellisense
2929
// TODO: add defines flags if available from project sites and files
3030

31-
type internal SourceTextColorizationData(classificationData: seq<ClassifiedSpan>) =
32-
member this.Tokens = classificationData |> Seq.toArray
33-
member this.GetClassifiedSpan(position: int): Option<ClassifiedSpan> =
34-
let mutable left = 0
35-
let mutable right = this.Tokens.Length - 1
36-
let mutable result = None
37-
while result.IsNone && right >= left do
38-
let middle = (left + right) / 2
39-
let middleToken = this.Tokens.[middle]
40-
if middleToken.TextSpan.End <= position then
41-
left <- middle + 1
42-
else if middleToken.TextSpan.Start > position then
43-
right <- middle - 1
44-
else
45-
result <- Some(middleToken)
46-
result
31+
type private SourceLineData(lexStateAtEndOfLine: FSharpTokenizerLexState, hashCode: int, classifiedSpans: IReadOnlyList<ClassifiedSpan>) =
32+
member val LexStateAtEndOfLine = lexStateAtEndOfLine
33+
member val HashCode = hashCode
34+
member val ClassifiedSpans = classifiedSpans
35+
36+
type private SourceTextData(lines: int) =
37+
member val Lines = Array.create<Option<SourceLineData>> lines None
4738

4839
[<ExportLanguageService(typeof<IEditorClassificationService>, FSharpCommonConstants.FSharpLanguageName)>]
4940
type internal FSharpColorizationService() =
5041

51-
static let colorizationDataCache = ConditionalWeakTable<(SourceText * TextSpan * Option<string>), SourceTextColorizationData>()
42+
static let DataCache = ConditionalWeakTable<SourceText, SourceTextData>()
43+
44+
static let compilerTokenToRoslynToken(colorKind: FSharpTokenColorKind) : string =
45+
match colorKind with
46+
| FSharpTokenColorKind.Comment -> ClassificationTypeNames.Comment
47+
| FSharpTokenColorKind.Identifier -> ClassificationTypeNames.Identifier
48+
| FSharpTokenColorKind.Keyword -> ClassificationTypeNames.Keyword
49+
| FSharpTokenColorKind.String -> ClassificationTypeNames.StringLiteral
50+
| FSharpTokenColorKind.Text -> ClassificationTypeNames.Text
51+
| FSharpTokenColorKind.UpperIdentifier -> ClassificationTypeNames.Identifier
52+
| FSharpTokenColorKind.Number -> ClassificationTypeNames.NumericLiteral
53+
| FSharpTokenColorKind.InactiveCode -> ClassificationTypeNames.ExcludedCode
54+
| FSharpTokenColorKind.PreprocessorKeyword -> ClassificationTypeNames.PreprocessorKeyword
55+
| FSharpTokenColorKind.Operator -> ClassificationTypeNames.Operator
56+
| FSharpTokenColorKind.TypeName -> ClassificationTypeNames.ClassName
57+
| FSharpTokenColorKind.Default | _ -> ClassificationTypeNames.Text
58+
59+
static let scanAndColorNextToken(lineTokenizer: FSharpLineTokenizer, colorMap: string[], lexState: Ref<FSharpTokenizerLexState>) : Option<FSharpTokenInfo> =
60+
let tokenInfoOption, nextLexState = lineTokenizer.ScanToken(lexState.Value)
61+
lexState.Value <- nextLexState
62+
if tokenInfoOption.IsSome then
63+
let classificationType = compilerTokenToRoslynToken(tokenInfoOption.Value.ColorClass)
64+
for i = tokenInfoOption.Value.LeftColumn to tokenInfoOption.Value.RightColumn do
65+
Array.set colorMap i classificationType
66+
tokenInfoOption
67+
68+
static let scanSourceLine(sourceTokenizer: FSharpSourceTokenizer, textLine: TextLine, lineContents: string, lexState: FSharpTokenizerLexState) : SourceLineData =
69+
let colorMap = Array.create textLine.Span.Length ClassificationTypeNames.Text
70+
let lineTokenizer = sourceTokenizer.CreateLineTokenizer(lineContents)
71+
72+
let previousLextState = ref(lexState)
73+
let mutable tokenInfoOption = scanAndColorNextToken(lineTokenizer, colorMap, previousLextState)
74+
while tokenInfoOption.IsSome do
75+
tokenInfoOption <- scanAndColorNextToken(lineTokenizer, colorMap, previousLextState)
76+
77+
let mutable startPosition = 0
78+
let mutable endPosition = startPosition
79+
let classifiedSpans = new List<ClassifiedSpan>()
80+
81+
while startPosition < colorMap.Length do
82+
let classificationType = colorMap.[startPosition]
83+
endPosition <- startPosition
84+
while endPosition < colorMap.Length && classificationType = colorMap.[endPosition] do
85+
endPosition <- endPosition + 1
86+
let textSpan = new TextSpan(textLine.Start + startPosition, endPosition - startPosition)
87+
classifiedSpans.Add(new ClassifiedSpan(classificationType, textSpan))
88+
startPosition <- endPosition
89+
90+
SourceLineData(previousLextState.Value, lineContents.GetHashCode(), classifiedSpans)
5291

53-
static let scanSourceText(sourceText: SourceText, textSpan: TextSpan, fileName: Option<string>, defines: string list, cancellationToken: CancellationToken): SourceTextColorizationData =
54-
let mutable runningLexState = ref(0L)
55-
let result = new List<ClassifiedSpan>()
56-
let sourceTokenizer = FSharpSourceTokenizer(defines, fileName)
57-
58-
let compilerTokenToRoslynToken(colorKind: FSharpTokenColorKind) =
59-
match colorKind with
60-
| FSharpTokenColorKind.Comment -> ClassificationTypeNames.Comment
61-
| FSharpTokenColorKind.Identifier -> ClassificationTypeNames.Identifier
62-
| FSharpTokenColorKind.Keyword -> ClassificationTypeNames.Keyword
63-
| FSharpTokenColorKind.String -> ClassificationTypeNames.StringLiteral
64-
| FSharpTokenColorKind.Text -> ClassificationTypeNames.Text
65-
| FSharpTokenColorKind.UpperIdentifier -> ClassificationTypeNames.Identifier
66-
| FSharpTokenColorKind.Number -> ClassificationTypeNames.NumericLiteral
67-
| FSharpTokenColorKind.InactiveCode -> ClassificationTypeNames.ExcludedCode
68-
| FSharpTokenColorKind.PreprocessorKeyword -> ClassificationTypeNames.PreprocessorKeyword
69-
| FSharpTokenColorKind.Operator -> ClassificationTypeNames.Operator
70-
| FSharpTokenColorKind.TypeName -> ClassificationTypeNames.ClassName
71-
| FSharpTokenColorKind.Default | _ -> ClassificationTypeNames.Text
72-
73-
let scanNextToken(lineTokenizer: FSharpLineTokenizer, colorMap: string[], lexState: Ref<int64>) =
74-
let tokenInfoOption, currentLexState = lineTokenizer.ScanToken(lexState.Value)
75-
lexState.Value <- currentLexState
76-
match tokenInfoOption with
77-
| None -> false
78-
| Some(tokenInfo) ->
79-
let classificationType = compilerTokenToRoslynToken(tokenInfo.ColorClass)
80-
for i = tokenInfo.LeftColumn to tokenInfo.RightColumn do
81-
Array.set colorMap i classificationType
82-
true
83-
84-
let scanSourceLine(textLine: TextLine, lexState: Ref<int64>) =
85-
let lineTokenizer = sourceTokenizer.CreateLineTokenizer(textLine.Text.ToString(textLine.Span))
86-
let colorMap = Array.create textLine.Span.Length ClassificationTypeNames.Text
87-
while scanNextToken(lineTokenizer, colorMap, lexState) do ()
88-
89-
let mutable startPosition = 0
90-
let mutable endPosition = startPosition
91-
while startPosition < colorMap.Length do
92-
let classificationType = colorMap.[startPosition]
93-
endPosition <- startPosition
94-
while endPosition < colorMap.Length && classificationType = colorMap.[endPosition] do
95-
endPosition <- endPosition + 1
96-
let textSpan = new TextSpan(textLine.Start + startPosition, endPosition - startPosition)
97-
result.Add(new ClassifiedSpan(classificationType, textSpan))
98-
startPosition <- endPosition
99-
100-
let scanStartLine = sourceText.Lines.GetLineFromPosition(textSpan.Start).LineNumber
101-
let scanEndLine = sourceText.Lines.GetLineFromPosition(textSpan.End).LineNumber
102-
103-
for i = scanStartLine to scanEndLine do
104-
cancellationToken.ThrowIfCancellationRequested()
105-
let currentLine = sourceText.Lines.Item(i)
106-
scanSourceLine(currentLine, runningLexState)
107-
108-
SourceTextColorizationData(result)
109-
110-
static let classifySourceTextAsync(sourceText: SourceText, fileName: Option<string>, textSpan: TextSpan, result: List<ClassifiedSpan>, cancellationToken: CancellationToken) =
111-
Task.Run(fun () ->
112-
try
113-
let classificationData = FSharpColorizationService.GetColorizationData(sourceText, textSpan, fileName, [], cancellationToken)
114-
result.AddRange(classificationData.Tokens |> Seq.filter(fun token -> textSpan.Start <= token.TextSpan.Start && token.TextSpan.End <= textSpan.End))
115-
with ex ->
116-
Assert.Exception(ex)
117-
reraise()
118-
)
119-
12092
interface IEditorClassificationService with
12193

12294
member this.AddLexicalClassifications(text: SourceText, textSpan: TextSpan, result: List<ClassifiedSpan>, cancellationToken: CancellationToken) =
123-
classifySourceTextAsync(text, None, textSpan, result, cancellationToken).Wait(cancellationToken)
95+
result.AddRange(FSharpColorizationService.GetColorizationData(text, textSpan, None, [], cancellationToken))
12496

12597
member this.AddSyntacticClassificationsAsync(document: Document, textSpan: TextSpan, result: List<ClassifiedSpan>, cancellationToken: CancellationToken) =
126-
let sourceText = document.GetTextAsync(cancellationToken).ConfigureAwait(false).GetAwaiter().GetResult()
127-
classifySourceTextAsync(sourceText, Some(document.Name), textSpan, result, cancellationToken)
98+
document.GetTextAsync(cancellationToken).ContinueWith(
99+
fun (sourceTextTask: Task<SourceText>) ->
100+
result.AddRange(FSharpColorizationService.GetColorizationData(sourceTextTask.Result, textSpan, None, [], cancellationToken))
101+
, TaskContinuationOptions.OnlyOnRanToCompletion)
128102

129103
member this.AddSemanticClassificationsAsync(document: Document, textSpan: TextSpan, result: List<ClassifiedSpan>, cancellationToken: CancellationToken) =
130-
let sourceText = document.GetTextAsync(cancellationToken).ConfigureAwait(false).GetAwaiter().GetResult()
131-
classifySourceTextAsync(sourceText, Some(document.Name), textSpan, result, cancellationToken)
104+
document.GetTextAsync(cancellationToken).ContinueWith(
105+
fun (sourceTextTask: Task<SourceText>) ->
106+
//TODO: Replace with types data when available from intellisense (behaving as AddSyntacticClassificationsAsync() for now)
107+
result.AddRange(FSharpColorizationService.GetColorizationData(sourceTextTask.Result, textSpan, None, [], cancellationToken))
108+
, TaskContinuationOptions.OnlyOnRanToCompletion)
132109

133110
member this.AdjustStaleClassification(text: SourceText, classifiedSpan: ClassifiedSpan) : ClassifiedSpan =
134-
let result = new List<ClassifiedSpan>()
135-
classifySourceTextAsync(text, None, classifiedSpan.TextSpan, result, CancellationToken.None).Wait()
136-
if result.Any() then
137-
result.First()
111+
let tokens = FSharpColorizationService.GetColorizationData(text, classifiedSpan.TextSpan, None, [], CancellationToken.None)
112+
if tokens.Any() then
113+
tokens.First()
138114
else
139115
new ClassifiedSpan(ClassificationTypeNames.WhiteSpace, classifiedSpan.TextSpan)
140116

141-
// Helper function to proxy Roslyn types to tests
142-
static member GetColorizationData(sourceText: SourceText, textSpan: TextSpan, fileName: Option<string>, defines: string list, cancellationToken: CancellationToken) : SourceTextColorizationData =
143-
colorizationDataCache.GetValue((sourceText, textSpan, fileName), fun key -> scanSourceText(sourceText, textSpan, fileName, defines, cancellationToken))
117+
static member GetColorizationData(sourceText: SourceText, textSpan: TextSpan, fileName: Option<string>, defines: string list, cancellationToken: CancellationToken) : List<ClassifiedSpan> =
118+
try
119+
let sourceTokenizer = FSharpSourceTokenizer(defines, fileName)
120+
let sourceTextData = DataCache.GetValue(sourceText, fun key -> SourceTextData(key.Lines.Count))
121+
122+
let startLine = sourceText.Lines.GetLineFromPosition(textSpan.Start).LineNumber
123+
let endLine = sourceText.Lines.GetLineFromPosition(textSpan.End).LineNumber
124+
125+
// Get the last cached scanned line
126+
let mutable scanStartLine = startLine
127+
while scanStartLine > 0 && sourceTextData.Lines.[scanStartLine - 1].IsNone do
128+
scanStartLine <- scanStartLine - 1
129+
130+
let result = new List<ClassifiedSpan>()
131+
let mutable lexState = if scanStartLine = 0 then 0L else sourceTextData.Lines.[scanStartLine - 1].Value.LexStateAtEndOfLine
132+
133+
for i = scanStartLine to sourceText.Lines.Count - 1 do
134+
cancellationToken.ThrowIfCancellationRequested()
135+
136+
let textLine = sourceText.Lines.[i]
137+
let lineContents = textLine.Text.ToString(textLine.Span)
138+
let lineHashCode = lineContents.GetHashCode()
139+
140+
let mutable lineData = sourceTextData.Lines.[i]
141+
if lineData.IsNone || lineData.Value.HashCode <> lineHashCode then
142+
lineData <- Some(scanSourceLine(sourceTokenizer, textLine, lineContents, lexState))
143+
144+
lexState <- lineData.Value.LexStateAtEndOfLine
145+
sourceTextData.Lines.[i] <- lineData
146+
147+
if startLine <= i && i<= endLine then
148+
result.AddRange(lineData.Value.ClassifiedSpans |> Seq.filter(fun token ->
149+
textSpan.Contains(token.TextSpan.Start) ||
150+
textSpan.Contains(token.TextSpan.End - 1) ||
151+
(token.TextSpan.Start <= textSpan.Start && textSpan.End <= token.TextSpan.End)))
152+
153+
result
154+
with ex ->
155+
Assert.Exception(ex)
156+
reraise()

0 commit comments

Comments
 (0)