Skip to content

Commit 56c3255

Browse files
KevinRansomKevinRansom
authored
Embedded source (#1591)
* Improve scope arrangement for portable pdbs * Git: 1) Enable implicit sequence points for portable pdbs 2) Minor rename and cleanup for portable pdb generation * Start enabling /embed * Saved * Fix manifest --- it was inadvertently updated with debuggableattribute flags. * reset debug build to full pdpbs * Fix up test baselines * Make build setup switch offable * Add testcase for portablepdb generation many let bindings
1 parent 0e240cd commit 56c3255

File tree

18 files changed

+375
-62
lines changed

18 files changed

+375
-62
lines changed

‎build.cmd‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ for %%i in (%BUILD_FSC_DEFAULT%) do ( call :SET_CONFIG %%i )
6464
setlocal disableDelayedExpansion
6565
echo.
6666

67+
rem disable setup build by setting FSC_BUILD_SETUP=0
68+
if /i '%FSC_BUILD_SETUP%' == '' (set FSC_BUILD_SETUP=1)
6769
goto :MAIN
6870

6971
:SET_CONFIG
@@ -96,7 +98,7 @@ if /i '%ARG%' == 'all' (
9698
set BUILD_CORECLR=1
9799
set BUILD_PORTABLE=1
98100
set BUILD_VS=1
99-
set BUILD_SETUP=1
101+
set BUILD_SETUP=%FSC_BUILD_SETUP%
100102

101103
set TEST_COMPILERUNIT=1
102104
set TEST_NET40_COREUNIT=1
@@ -120,7 +122,7 @@ if /i '%ARG%' == 'microbuild' (
120122
set BUILD_CORECLR=0
121123
set BUILD_PORTABLE=1
122124
set BUILD_VS=1
123-
set BUILD_SETUP=1
125+
set BUILD_SETUP=%FSC_BUILD_SETUP%
124126

125127
set TEST_COMPILERUNIT=1
126128
set TEST_NET40_COREUNIT=1
@@ -143,7 +145,7 @@ if /i '%ARG%' == 'ci' (
143145
set BUILD_CORECLR=1
144146
set BUILD_PORTABLE=1
145147
set BUILD_VS=1
146-
set BUILD_SETUP=1
148+
set BUILD_SETUP=%FSC_BUILD_SETUP%
147149

148150
set TEST_COMPILERUNIT=1
149151
set TEST_NET40_COREUNIT=1
@@ -164,7 +166,7 @@ if /i '%ARG%' == 'ci_part1' (
164166
set BUILD_CORECLR=0
165167
set BUILD_PORTABLE=1
166168
set BUILD_VS=1
167-
set BUILD_SETUP=1
169+
set BUILD_SETUP=%FSC_BUILD_SETUP%
168170

169171
set TEST_COMPILERUNIT=1
170172
set TEST_NET40_COREUNIT=0

‎src/FSharpSource.Settings.targets‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,11 @@
2626
<!-- Standard interpretations of Debug and Release configurations -->
2727
<PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
2828
<DebugType Condition=" '$(DebugType)' == '' and '$(TargetFramework)' != 'coreclr' ">full</DebugType>
29-
<DebugType Condition=" '$(DebugType)' == '' and '$(TargetFramework)' == 'coreclr' ">portable</DebugType>
29+
<DebugType Condition=" '$(DebugType)' == '' and '$(TargetFramework)' == 'coreclr' ">embedded</DebugType>
3030
<Optimize Condition=" '$(Optimize)' == '' ">false</Optimize>
3131
<ErrorReport Condition=" '$(ErrorReport)' == '' ">prompt</ErrorReport>
32-
<OtherFlags>$(OtherFlags) --no-jit-optimize</OtherFlags>
32+
<OtherFlags Condition=" '$(DebugType)' == '' and '$(TargetFramework)' != 'coreclr' ">$(OtherFlags) --no-jit-optimize</OtherFlags>
33+
<OtherFlags Condition=" '$(DebugType)' == '' and '$(TargetFramework)' == 'coreclr' ">$(OtherFlags) --no-jit-optimize --embed</OtherFlags>
3334
<DefineConstants Condition=" '$(ProjectLanguage)' != 'VisualBasic' ">DEBUG;TRACE;CODE_ANALYSIS;$(DefineConstants)</DefineConstants>
3435
<DefineConstants Condition=" '$(ProjectLanguage)' == 'VisualBasic' ">DEBUG=True,TRACE=True,CODE_ANALYSIS=True,$(DefineConstants)</DefineConstants>
3536
<SIGN_WITH_MSFT_KEY Condition=" '$(SIGN_WITH_MSFT_KEY)' == '' ">false</SIGN_WITH_MSFT_KEY>

‎src/absil/il.fs‎

100755100644
File mode changed.

‎src/absil/ilwrite.fs‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3536,7 +3536,7 @@ let writeDirectory os dict =
35363536

35373537
let writeBytes (os: BinaryWriter) (chunk:byte[]) = os.Write(chunk,0,chunk.Length)
35383538

3539-
let writeBinaryAndReportMappings (outfile, ilg, pdbfile: string option, signer: ILStrongNameSigner option, portablePDB, embeddedPDB,
3539+
let writeBinaryAndReportMappings (outfile, ilg, pdbfile: string option, signer: ILStrongNameSigner option, portablePDB, embeddedPDB, embedAllSource, embedSourceList,
35403540
fixupOverlappingSequencePoints, emitTailcalls, showTimes, dumpDebugInfo) modul noDebugData =
35413541
// Store the public key from the signer into the manifest. This means it will be written
35423542
// to the binary and also acts as an indicator to leave space for delay sign
@@ -3690,7 +3690,7 @@ let writeBinaryAndReportMappings (outfile, ilg, pdbfile: string option, signer:
36903690
let pdbOpt =
36913691
match portablePDB with
36923692
| true ->
3693-
let struct (uncompressedLength, contentId, stream) as pdbStream = generatePortablePdb fixupOverlappingSequencePoints showTimes pdbData
3693+
let struct (uncompressedLength, contentId, stream) as pdbStream = generatePortablePdb fixupOverlappingSequencePoints embedAllSource embedSourceList showTimes pdbData
36943694
if embeddedPDB then Some (compressPortablePdbStream uncompressedLength contentId stream)
36953695
else Some (pdbStream)
36963696
| _ -> None
@@ -4260,13 +4260,15 @@ type options =
42604260
pdbfile: string option
42614261
portablePDB: bool
42624262
embeddedPDB: bool
4263+
embedAllSource: bool
4264+
embedSourceList: string list
42634265
signer: ILStrongNameSigner option
42644266
fixupOverlappingSequencePoints: bool
42654267
emitTailcalls : bool
42664268
showTimes: bool
42674269
dumpDebugInfo:bool }
42684270

42694271
let WriteILBinary (outfile, (args: options), modul, noDebugData) =
4270-
ignore (writeBinaryAndReportMappings (outfile, args.ilg, args.pdbfile, args.signer, args.portablePDB, args.embeddedPDB,
4271-
args.fixupOverlappingSequencePoints, args.emitTailcalls, args.showTimes,
4272-
args.dumpDebugInfo) modul noDebugData)
4272+
ignore (writeBinaryAndReportMappings (outfile, args.ilg, args.pdbfile, args.signer, args.portablePDB, args.embeddedPDB,
4273+
args.embedAllSource, args.embedSourceList, args.fixupOverlappingSequencePoints,
4274+
args.emitTailcalls, args.showTimes, args.dumpDebugInfo) modul noDebugData)

‎src/absil/ilwrite.fsi‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ type options =
2020
pdbfile: string option
2121
portablePDB: bool
2222
embeddedPDB: bool
23+
embedAllSource: bool
24+
embedSourceList: string list
2325
signer : ILStrongNameSigner option
2426
fixupOverlappingSequencePoints : bool
2527
emitTailcalls: bool

‎src/absil/ilwritepdb.fs‎

Lines changed: 84 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,31 @@ open Microsoft.FSharp.Compiler.AbstractIL.Internal.Library
1919
open Microsoft.FSharp.Compiler.ErrorLogger
2020
open Microsoft.FSharp.Compiler.Range
2121

22+
23+
type BlobBuildingStream () =
24+
inherit Stream()
25+
26+
static let chunkSize = 32 * 1024
27+
let builder = new BlobBuilder(chunkSize)
28+
29+
override this.CanWrite with get() = true
30+
override this.CanRead with get() = false
31+
override this.CanSeek with get() = false
32+
override this.Length with get() = int64(builder.Count)
33+
34+
override this.Write(buffer:byte array, offset:int, count:int) = builder.WriteBytes(buffer, offset, count)
35+
override this.WriteByte(value:byte) = builder.WriteByte(value)
36+
member this.WriteInt32(value:int) = builder.WriteInt32(value)
37+
member this.ToImmutableArray() = builder.ToImmutableArray()
38+
member this.TryWriteBytes(stream:Stream, length:int) = builder.TryWriteBytes(stream, length)
39+
40+
override this.Flush() = ()
41+
override this.Dispose(_disposing:bool) = ()
42+
override this.Seek(_offset:int64, _origin:SeekOrigin) = raise (new NotSupportedException())
43+
override this.Read(_buffer:byte array, _offset:int, _count:int) = raise (new NotSupportedException())
44+
override this.SetLength(_value:int64) = raise (new NotSupportedException())
45+
override val Position = 0L with get, set
46+
2247
// --------------------------------------------------------------------
2348
// PDB types
2449
// --------------------------------------------------------------------
@@ -227,7 +252,7 @@ let fixupOverlappingSequencePoints fixupSPs showTimes methods =
227252
Array.sortInPlaceBy fst allSps
228253
spCounts, allSps
229254

230-
let generatePortablePdb fixupSPs showTimes (info:PdbData) =
255+
let generatePortablePdb fixupSPs (embedAllSource:bool) (embedSourceList:string list) showTimes (info:PdbData) =
231256
sortMethods showTimes info
232257
let _spCounts, _allSps = fixupOverlappingSequencePoints fixupSPs showTimes info.Methods
233258
let externalRowCounts = getRowCounts info.TableRowCounts
@@ -253,23 +278,71 @@ let generatePortablePdb fixupSPs showTimes (info:PdbData) =
253278
metadata.GetOrAddBlob(writer)
254279

255280
let corSymLanguageTypeFSharp = System.Guid(0xAB4F38C9u, 0xB6E6us, 0x43baus, 0xBEuy, 0x3Buy, 0x58uy, 0x08uy, 0x0Buy, 0x2Cuy, 0xCCuy, 0xE3uy)
281+
let embeddedSource = System.Guid(0x0e8a571bu, 0x6926us, 0x466eus, 0xb4uy, 0xaduy, 0x8auy, 0xb0uy, 0x46uy, 0x11uy, 0xf5uy, 0xfeuy)
282+
283+
/// <summary>
284+
/// The maximum number of bytes in to write out uncompressed.
285+
///
286+
/// This prevents wasting resources on compressing tiny files with little to negative gain
287+
/// in PDB file size.
288+
///
289+
/// Chosen as the point at which we start to see > 10% blob size reduction using all
290+
/// current source files in corefx and roslyn as sample data.
291+
/// </summary>
292+
let sourceCompressionThreshold = 200
293+
256294
let documentIndex =
295+
let includeSource file =
296+
let isInList =
297+
if embedSourceList |> List.length = 0 then false
298+
else
299+
match embedSourceList |> List.tryFind(fun f -> String.Compare(file, f, StringComparison.OrdinalIgnoreCase ) = 0) with
300+
| Some _ -> true
301+
| None -> false
302+
303+
if not embedAllSource && not isInList || not (File.Exists(file)) then
304+
None
305+
else
306+
let stream = File.OpenRead(file)
307+
let length64 = stream.Length
308+
if length64 > int64(Int32.MaxValue) then raise (new IOException("File is too long"))
309+
310+
let builder = new BlobBuildingStream()
311+
let length = int(length64)
312+
if length < sourceCompressionThreshold then
313+
builder.WriteInt32(0)
314+
builder.TryWriteBytes(stream, length) |> ignore
315+
else
316+
builder.WriteInt32(length) |>ignore
317+
use deflater = new DeflateStream(builder, CompressionMode.Compress, true)
318+
stream.CopyTo(deflater) |> ignore
319+
Some (builder.ToImmutableArray())
320+
257321
let mutable index = new Dictionary<string, DocumentHandle>(docs.Length)
258322
metadata.SetCapacity(TableIndex.Document, docs.Length)
259323
for doc in docs do
260324
let handle =
261325
match checkSum doc.File with
262326
| Some (hashAlg, checkSum) ->
263-
serializeDocumentName doc.File,
264-
metadata.GetOrAddGuid(hashAlg),
265-
metadata.GetOrAddBlob(checkSum.ToImmutableArray()),
266-
metadata.GetOrAddGuid(corSymLanguageTypeFSharp)
327+
let h =
328+
(serializeDocumentName doc.File,
329+
metadata.GetOrAddGuid(hashAlg),
330+
metadata.GetOrAddBlob(checkSum.ToImmutableArray()),
331+
metadata.GetOrAddGuid(corSymLanguageTypeFSharp)) |> metadata.AddDocument
332+
match includeSource doc.File with
333+
| None -> ()
334+
| Some blob ->
335+
metadata.AddCustomDebugInformation(DocumentHandle.op_Implicit(h),
336+
metadata.GetOrAddGuid(embeddedSource),
337+
metadata.GetOrAddBlob(blob)) |> ignore
338+
h
267339
| None ->
268-
serializeDocumentName doc.File,
269-
metadata.GetOrAddGuid(System.Guid.Empty),
270-
metadata.GetOrAddBlob(ImmutableArray<byte>.Empty),
271-
metadata.GetOrAddGuid(corSymLanguageTypeFSharp)
272-
|> metadata.AddDocument
340+
let h =
341+
(serializeDocumentName doc.File,
342+
metadata.GetOrAddGuid(System.Guid.Empty),
343+
metadata.GetOrAddBlob(ImmutableArray<byte>.Empty),
344+
metadata.GetOrAddGuid(corSymLanguageTypeFSharp)) |> metadata.AddDocument
345+
h
273346
index.Add(doc.File, handle)
274347
index
275348

@@ -291,7 +364,7 @@ let generatePortablePdb fixupSPs showTimes (info:PdbData) =
291364
else
292365
match documentIndex.TryGetValue(docs.[d].File) with
293366
| false, _ -> Unchecked.defaultof<DocumentHandle>
294-
| true, f -> f
367+
| true, h -> h
295368

296369
if sps.Length = 0 then
297370
Unchecked.defaultof<DocumentHandle>, Unchecked.defaultof<BlobHandle>
@@ -306,7 +379,6 @@ let generatePortablePdb fixupSPs showTimes (info:PdbData) =
306379
singleDocumentIndex
307380

308381
let builder = new BlobBuilder()
309-
310382
builder.WriteCompressedInteger(minfo.LocalSignatureToken)
311383

312384
// Initial document: When sp's spread over more than one document we put the initial document here.

‎src/absil/ilwritepdb.fsi‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ type idd =
8282
iddData: byte[];
8383
iddChunk: BinaryChunk }
8484

85-
val generatePortablePdb : fixupSPs:bool -> showTimes:bool -> info:PdbData -> struct (int64 * BlobContentId * MemoryStream)
85+
val generatePortablePdb : fixupSPs:bool -> embedAllSource:bool -> embedSourceList:string list -> showTimes:bool -> info:PdbData -> struct (int64 * BlobContentId * MemoryStream)
8686
val compressPortablePdbStream : uncompressedLength:int64 -> contentId:BlobContentId -> stream:MemoryStream -> struct (int64 * BlobContentId * MemoryStream)
8787
val embedPortablePdbInfo : uncompressedLength:int64 -> contentId:BlobContentId -> stream:MemoryStream -> showTimes:bool -> fpdb:string -> cvChunk:BinaryChunk -> pdbChunk:BinaryChunk -> idd[]
8888
val writePortablePdbInfo : contentId:BlobContentId -> stream:MemoryStream -> showTimes:bool -> fpdb:string -> cvChunk:BinaryChunk -> idd[]

‎src/fsharp/CompileOps.fs‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2070,6 +2070,9 @@ type TcConfigBuilder =
20702070
mutable jitTracking : bool
20712071
mutable portablePDB : bool
20722072
mutable embeddedPDB : bool
2073+
mutable embedAllSource : bool
2074+
mutable embedSourceList : string list
2075+
20732076
mutable ignoreSymbolStoreSequencePoints : bool
20742077
mutable internConstantStrings : bool
20752078
mutable extraOptimizationIterations : int
@@ -2237,7 +2240,9 @@ type TcConfigBuilder =
22372240
useSignatureDataFile = false
22382241
jitTracking = true
22392242
portablePDB = true
2240-
embeddedPDB = true
2243+
embeddedPDB = false
2244+
embedAllSource = false
2245+
embedSourceList = []
22412246
ignoreSymbolStoreSequencePoints = false
22422247
internConstantStrings = true
22432248
extraOptimizationIterations = 0
@@ -2367,7 +2372,7 @@ type TcConfigBuilder =
23672372
| None -> false
23682373
if ok && not (List.contains absolutePath tcConfigB.includes) then
23692374
tcConfigB.includes <- tcConfigB.includes ++ absolutePath
2370-
2375+
23712376
member tcConfigB.AddLoadedSource(m,path,pathLoadedFrom) =
23722377
if FileSystem.IsInvalidPathShim(path) then
23732378
warning(Error(FSComp.SR.buildInvalidFilename(path),m))
@@ -2380,7 +2385,9 @@ type TcConfigBuilder =
23802385
ComputeMakePathAbsolute pathLoadedFrom path
23812386
if not (List.contains path (List.map snd tcConfigB.loadedSources)) then
23822387
tcConfigB.loadedSources <- tcConfigB.loadedSources ++ (m,path)
2383-
2388+
2389+
member tcConfigB.AddEmbeddedSourceFile (file) =
2390+
tcConfigB.embedSourceList <- tcConfigB.embedSourceList ++ file
23842391

23852392
member tcConfigB.AddEmbeddedResource filename =
23862393
tcConfigB.embedResources <- tcConfigB.embedResources ++ filename
@@ -2723,6 +2730,8 @@ type TcConfig private (data : TcConfigBuilder,validate:bool) =
27232730
member x.jitTracking = data.jitTracking
27242731
member x.portablePDB = data.portablePDB
27252732
member x.embeddedPDB = data.embeddedPDB
2733+
member x.embedAllSource = data.embedAllSource
2734+
member x.embedSourceList = data.embedSourceList
27262735
member x.ignoreSymbolStoreSequencePoints = data.ignoreSymbolStoreSequencePoints
27272736
member x.internConstantStrings = data.internConstantStrings
27282737
member x.extraOptimizationIterations = data.extraOptimizationIterations

‎src/fsharp/CompileOps.fsi‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,8 @@ type TcConfigBuilder =
300300
mutable jitTracking : bool
301301
mutable portablePDB : bool
302302
mutable embeddedPDB : bool
303+
mutable embedAllSource : bool
304+
mutable embedSourceList : string list
303305
mutable ignoreSymbolStoreSequencePoints : bool
304306
mutable internConstantStrings : bool
305307
mutable extraOptimizationIterations : int
@@ -372,6 +374,7 @@ type TcConfigBuilder =
372374
member AddIncludePath : range * string * string -> unit
373375
member AddReferencedAssemblyByPath : range * string -> unit
374376
member RemoveReferencedAssemblyByPath : range * string -> unit
377+
member AddEmbeddedSourceFile : string -> unit
375378
member AddEmbeddedResource : string -> unit
376379
377380
static member SplitCommandLineResourceInfo : string -> string * string * ILResourceAccess
@@ -453,6 +456,8 @@ type TcConfig =
453456
member jitTracking : bool
454457
member portablePDB : bool
455458
member embeddedPDB : bool
459+
member embedAllSource : bool
460+
member embedSourceList : string list
456461
member ignoreSymbolStoreSequencePoints : bool
457462
member internConstantStrings : bool
458463
member extraOptimizationIterations : int

0 commit comments

Comments
 (0)