Skip to content

Commit 6685a5e

Browse files
hvitvedhvitved
authored
Merge pull request #3136 from calumgrant/cs/buildless-extraction
C#: Improvements to buildless extraction
2 parents bacb11a + adde52d commit 6685a5e

File tree

26 files changed

+404
-230
lines changed

26 files changed

+404
-230
lines changed

‎.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,6 @@
1919

2020
# It's useful (though not required) to be able to unpack codeql in the ql checkout itself
2121
/codeql/
22-
.vscode/settings.json
22+
2323
csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json
24+
.vscode

‎csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs

+13-1
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,19 @@ public AssemblyInfo ResolveReference(string id)
163163
/// </summary>
164164
/// <param name="filepath">The filename to query.</param>
165165
/// <returns>The assembly info.</returns>
166-
public AssemblyInfo GetAssemblyInfo(string filepath) => assemblyInfo[filepath];
166+
public AssemblyInfo GetAssemblyInfo(string filepath)
167+
{
168+
if(assemblyInfo.TryGetValue(filepath, out var info))
169+
{
170+
return info;
171+
}
172+
else
173+
{
174+
info = AssemblyInfo.ReadFromFile(filepath);
175+
assemblyInfo.Add(filepath, info);
176+
return info;
177+
}
178+
}
167179

168180
// List of pending DLLs to index.
169181
readonly List<string> dlls = new List<string>();

‎csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs

+147-94
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
1-
using System;
1+
using Semmle.Util;
2+
using System;
23
using System.Collections.Generic;
34
using System.IO;
45
using System.Linq;
5-
using System.Runtime.InteropServices;
6-
using Semmle.Util;
76
using Semmle.Extraction.CSharp.Standalone;
7+
using System.Threading.Tasks;
8+
using System.Collections.Concurrent;
9+
using System.Text;
10+
using System.Security.Cryptography;
811

912
namespace Semmle.BuildAnalyser
1013
{
@@ -43,19 +46,18 @@ interface IBuildAnalysis
4346
/// <summary>
4447
/// Main implementation of the build analysis.
4548
/// </summary>
46-
class BuildAnalysis : IBuildAnalysis
49+
class BuildAnalysis : IBuildAnalysis, IDisposable
4750
{
48-
readonly AssemblyCache assemblyCache;
49-
readonly NugetPackages nuget;
50-
readonly IProgressMonitor progressMonitor;
51-
HashSet<string> usedReferences = new HashSet<string>();
52-
readonly HashSet<string> usedSources = new HashSet<string>();
53-
readonly HashSet<string> missingSources = new HashSet<string>();
54-
readonly Dictionary<string, string> unresolvedReferences = new Dictionary<string, string>();
55-
readonly DirectoryInfo sourceDir;
56-
int failedProjects, succeededProjects;
57-
readonly string[] allSources;
58-
int conflictedReferences = 0;
51+
private readonly AssemblyCache assemblyCache;
52+
private readonly NugetPackages nuget;
53+
private readonly IProgressMonitor progressMonitor;
54+
private readonly IDictionary<string, bool> usedReferences = new ConcurrentDictionary<string, bool>();
55+
private readonly IDictionary<string, bool> sources = new ConcurrentDictionary<string, bool>();
56+
private readonly IDictionary<string, string> unresolvedReferences = new ConcurrentDictionary<string, string>();
57+
private readonly DirectoryInfo sourceDir;
58+
private int failedProjects, succeededProjects;
59+
private readonly string[] allSources;
60+
private int conflictedReferences = 0;
5961

6062
/// <summary>
6163
/// Performs a C# build analysis.
@@ -64,6 +66,8 @@ class BuildAnalysis : IBuildAnalysis
6466
/// <param name="progress">Display of analysis progress.</param>
6567
public BuildAnalysis(Options options, IProgressMonitor progress)
6668
{
69+
var startTime = DateTime.Now;
70+
6771
progressMonitor = progress;
6872
sourceDir = new DirectoryInfo(options.SrcDir);
6973

@@ -74,36 +78,43 @@ public BuildAnalysis(Options options, IProgressMonitor progress)
7478
Where(d => !options.ExcludesFile(d)).
7579
ToArray();
7680

77-
var dllDirNames = options.DllDirs.Select(Path.GetFullPath);
81+
var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList();
82+
PackageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
7883

7984
if (options.UseNuGet)
8085
{
81-
nuget = new NugetPackages(sourceDir.FullName);
82-
ReadNugetFiles();
83-
dllDirNames = dllDirNames.Concat(Enumerators.Singleton(nuget.PackageDirectory));
86+
try
87+
{
88+
nuget = new NugetPackages(sourceDir.FullName, PackageDirectory);
89+
ReadNugetFiles();
90+
}
91+
catch(FileNotFoundException)
92+
{
93+
progressMonitor.MissingNuGet();
94+
}
8495
}
8596

8697
// Find DLLs in the .Net Framework
8798
if (options.ScanNetFrameworkDlls)
8899
{
89-
dllDirNames = dllDirNames.Concat(Runtime.Runtimes.Take(1));
100+
dllDirNames.Add(Runtime.Runtimes.First());
90101
}
91102

92-
assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress);
93-
94-
// Analyse all .csproj files in the source tree.
95-
if (options.SolutionFile != null)
96-
{
97-
AnalyseSolution(options.SolutionFile);
98-
}
99-
else if (options.AnalyseCsProjFiles)
103+
// These files can sometimes prevent `dotnet restore` from working correctly.
104+
using (new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories)))
105+
using (new FileRenamer(sourceDir.GetFiles("Directory.Build.props", SearchOption.AllDirectories)))
100106
{
101-
AnalyseProjectFiles();
102-
}
107+
var solutions = options.SolutionFile != null ?
108+
new[] { options.SolutionFile } :
109+
sourceDir.GetFiles("*.sln", SearchOption.AllDirectories).Select(d => d.FullName);
103110

104-
if (!options.AnalyseCsProjFiles)
105-
{
106-
usedReferences = new HashSet<string>(assemblyCache.AllAssemblies.Select(a => a.Filename));
111+
RestoreSolutions(solutions);
112+
dllDirNames.Add(PackageDirectory.DirInfo.FullName);
113+
assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress);
114+
AnalyseSolutions(solutions);
115+
116+
foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename))
117+
UseReference(filename);
107118
}
108119

109120
ResolveConflicts();
@@ -114,7 +125,7 @@ public BuildAnalysis(Options options, IProgressMonitor progress)
114125
}
115126

116127
// Output the findings
117-
foreach (var r in usedReferences)
128+
foreach (var r in usedReferences.Keys)
118129
{
119130
progressMonitor.ResolvedReference(r);
120131
}
@@ -132,7 +143,27 @@ public BuildAnalysis(Options options, IProgressMonitor progress)
132143
UnresolvedReferences.Count(),
133144
conflictedReferences,
134145
succeededProjects + failedProjects,
135-
failedProjects);
146+
failedProjects,
147+
DateTime.Now - startTime);
148+
}
149+
150+
/// <summary>
151+
/// Computes a unique temp directory for the packages associated
152+
/// with this source tree. Use a SHA1 of the directory name.
153+
/// </summary>
154+
/// <param name="srcDir"></param>
155+
/// <returns>The full path of the temp directory.</returns>
156+
private static string ComputeTempDirectory(string srcDir)
157+
{
158+
var bytes = Encoding.Unicode.GetBytes(srcDir);
159+
160+
using var sha1 = new SHA1CryptoServiceProvider();
161+
var sha = sha1.ComputeHash(bytes);
162+
var sb = new StringBuilder();
163+
foreach (var b in sha.Take(8))
164+
sb.AppendFormat("{0:x2}", b);
165+
166+
return Path.Combine(Path.GetTempPath(), "GitHub", "packages", sb.ToString());
136167
}
137168

138169
/// <summary>
@@ -143,7 +174,7 @@ public BuildAnalysis(Options options, IProgressMonitor progress)
143174
void ResolveConflicts()
144175
{
145176
var sortedReferences = usedReferences.
146-
Select(r => assemblyCache.GetAssemblyInfo(r)).
177+
Select(r => assemblyCache.GetAssemblyInfo(r.Key)).
147178
OrderBy(r => r.Version).
148179
ToArray();
149180

@@ -154,7 +185,9 @@ void ResolveConflicts()
154185
finalAssemblyList[r.Name] = r;
155186

156187
// Update the used references list
157-
usedReferences = new HashSet<string>(finalAssemblyList.Select(r => r.Value.Filename));
188+
usedReferences.Clear();
189+
foreach (var r in finalAssemblyList.Select(r => r.Value.Filename))
190+
UseReference(r);
158191

159192
// Report the results
160193
foreach (var r in sortedReferences)
@@ -183,7 +216,7 @@ void ReadNugetFiles()
183216
/// <param name="reference">The filename of the reference.</param>
184217
void UseReference(string reference)
185218
{
186-
usedReferences.Add(reference);
219+
usedReferences[reference] = true;
187220
}
188221

189222
/// <summary>
@@ -192,25 +225,18 @@ void UseReference(string reference)
192225
/// <param name="sourceFile">The source file.</param>
193226
void UseSource(FileInfo sourceFile)
194227
{
195-
if (sourceFile.Exists)
196-
{
197-
usedSources.Add(sourceFile.FullName);
198-
}
199-
else
200-
{
201-
missingSources.Add(sourceFile.FullName);
202-
}
228+
sources[sourceFile.FullName] = sourceFile.Exists;
203229
}
204230

205231
/// <summary>
206232
/// The list of resolved reference files.
207233
/// </summary>
208-
public IEnumerable<string> ReferenceFiles => this.usedReferences;
234+
public IEnumerable<string> ReferenceFiles => this.usedReferences.Keys;
209235

210236
/// <summary>
211237
/// The list of source files used in projects.
212238
/// </summary>
213-
public IEnumerable<string> ProjectSourceFiles => usedSources;
239+
public IEnumerable<string> ProjectSourceFiles => sources.Where(s => s.Value).Select(s => s.Key);
214240

215241
/// <summary>
216242
/// All of the source files in the source directory.
@@ -226,7 +252,7 @@ void UseSource(FileInfo sourceFile)
226252
/// List of source files which were mentioned in project files but
227253
/// do not exist on the file system.
228254
/// </summary>
229-
public IEnumerable<string> MissingSourceFiles => missingSources;
255+
public IEnumerable<string> MissingSourceFiles => sources.Where(s => !s.Value).Select(s => s.Key);
230256

231257
/// <summary>
232258
/// Record that a particular reference couldn't be resolved.
@@ -239,74 +265,101 @@ void UnresolvedReference(string id, string projectFile)
239265
unresolvedReferences[id] = projectFile;
240266
}
241267

242-
/// <summary>
243-
/// Performs an analysis of all .csproj files.
244-
/// </summary>
245-
void AnalyseProjectFiles()
246-
{
247-
AnalyseProjectFiles(sourceDir.GetFiles("*.csproj", SearchOption.AllDirectories));
248-
}
268+
readonly TemporaryDirectory PackageDirectory;
249269

250270
/// <summary>
251271
/// Reads all the source files and references from the given list of projects.
252272
/// </summary>
253273
/// <param name="projectFiles">The list of projects to analyse.</param>
254-
void AnalyseProjectFiles(FileInfo[] projectFiles)
274+
void AnalyseProjectFiles(IEnumerable<FileInfo> projectFiles)
255275
{
256-
progressMonitor.AnalysingProjectFiles(projectFiles.Count());
257-
258276
foreach (var proj in projectFiles)
277+
AnalyseProject(proj);
278+
}
279+
280+
void AnalyseProject(FileInfo project)
281+
{
282+
if(!project.Exists)
259283
{
260-
try
261-
{
262-
var csProj = new CsProjFile(proj);
284+
progressMonitor.MissingProject(project.FullName);
285+
return;
286+
}
287+
288+
try
289+
{
290+
var csProj = new CsProjFile(project);
263291

264-
foreach (var @ref in csProj.References)
292+
foreach (var @ref in csProj.References)
293+
{
294+
AssemblyInfo resolved = assemblyCache.ResolveReference(@ref);
295+
if (!resolved.Valid)
265296
{
266-
AssemblyInfo resolved = assemblyCache.ResolveReference(@ref);
267-
if (!resolved.Valid)
268-
{
269-
UnresolvedReference(@ref, proj.FullName);
270-
}
271-
else
272-
{
273-
UseReference(resolved.Filename);
274-
}
297+
UnresolvedReference(@ref, project.FullName);
275298
}
276-
277-
foreach (var src in csProj.Sources)
299+
else
278300
{
279-
// Make a note of which source files the projects use.
280-
// This information doesn't affect the build but is dumped
281-
// as diagnostic output.
282-
UseSource(new FileInfo(src));
301+
UseReference(resolved.Filename);
283302
}
284-
++succeededProjects;
285303
}
286-
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
304+
305+
foreach (var src in csProj.Sources)
287306
{
288-
++failedProjects;
289-
progressMonitor.FailedProjectFile(proj.FullName, ex.Message);
307+
// Make a note of which source files the projects use.
308+
// This information doesn't affect the build but is dumped
309+
// as diagnostic output.
310+
UseSource(new FileInfo(src));
290311
}
312+
313+
++succeededProjects;
314+
}
315+
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
316+
{
317+
++failedProjects;
318+
progressMonitor.FailedProjectFile(project.FullName, ex.Message);
291319
}
320+
292321
}
293322

294-
/// <summary>
295-
/// Delete packages directory.
296-
/// </summary>
297-
public void Cleanup()
323+
void Restore(string projectOrSolution)
324+
{
325+
int exit = DotNet.RestoreToDirectory(projectOrSolution, PackageDirectory.DirInfo.FullName);
326+
switch(exit)
327+
{
328+
case 0:
329+
case 1:
330+
// No errors
331+
break;
332+
default:
333+
progressMonitor.CommandFailed("dotnet", $"restore \"{projectOrSolution}\"", exit);
334+
break;
335+
}
336+
}
337+
338+
public void RestoreSolutions(IEnumerable<string> solutions)
298339
{
299-
if (nuget != null) nuget.Cleanup(progressMonitor);
340+
Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 }, Restore);
300341
}
301342

302-
/// <summary>
303-
/// Analyse all project files in a given solution only.
304-
/// </summary>
305-
/// <param name="solutionFile">The filename of the solution.</param>
306-
public void AnalyseSolution(string solutionFile)
343+
public void AnalyseSolutions(IEnumerable<string> solutions)
344+
{
345+
Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 } , solutionFile =>
346+
{
347+
try
348+
{
349+
var sln = new SolutionFile(solutionFile);
350+
progressMonitor.AnalysingSolution(solutionFile);
351+
AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).Where(p => p.Exists));
352+
}
353+
catch (Microsoft.Build.Exceptions.InvalidProjectFileException ex)
354+
{
355+
progressMonitor.FailedProjectFile(solutionFile, ex.BaseMessage);
356+
}
357+
});
358+
}
359+
360+
public void Dispose()
307361
{
308-
var sln = new SolutionFile(solutionFile);
309-
AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).ToArray());
362+
PackageDirectory?.Dispose();
310363
}
311364
}
312365
}

0 commit comments

Comments
 (0)