diff --git a/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs b/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs index c585dd97a..ada2d3493 100644 --- a/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs +++ b/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs @@ -22,7 +22,7 @@ public class MavenCommandService : IMavenCommandService /// /// Per-location semaphores to prevent concurrent Maven CLI executions for the same pom.xml. - /// This allows multiple detectors (e.g., MvnCliComponentDetector and MavenWithFallbackDetector) + /// This allows the MvnCliComponentDetector /// to safely share the same output file without race conditions. /// private readonly ConcurrentDictionary locationLocks = new(); @@ -74,7 +74,7 @@ public async Task GenerateDependenciesFileAsync(ProcessRequest p } // Use semaphore to prevent concurrent Maven CLI executions for the same pom.xml. - // This allows MvnCliComponentDetector and MavenWithFallbackDetector to safely share the output file. + // This allows the MvnCliComponentDetector to reuse any existing output file. var semaphore = this.locationLocks.GetOrAdd(pomFile.Location, _ => new SemaphoreSlim(1, 1)); await semaphore.WaitAsync(cancellationToken); diff --git a/src/Microsoft.ComponentDetection.Detectors/maven/MavenWithFallbackDetector.cs b/src/Microsoft.ComponentDetection.Detectors/maven/MavenWithFallbackDetector.cs deleted file mode 100644 index 7fe3b8d45..000000000 --- a/src/Microsoft.ComponentDetection.Detectors/maven/MavenWithFallbackDetector.cs +++ /dev/null @@ -1,817 +0,0 @@ -#nullable disable -namespace Microsoft.ComponentDetection.Detectors.Maven; - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reactive.Linq; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using System.Threading.Tasks.Dataflow; -using System.Xml; -using Microsoft.ComponentDetection.Common; -using Microsoft.ComponentDetection.Contracts; -using Microsoft.ComponentDetection.Contracts.Internal; -using Microsoft.ComponentDetection.Contracts.TypedComponent; -using Microsoft.Extensions.Logging; - -/// -/// Enum representing which detection method was used. -/// -internal enum MavenDetectionMethod -{ - /// No detection performed. - None, - - /// MvnCli was used successfully for all files. - MvnCliOnly, - - /// Static parser was used for all files (MvnCli not available or failed completely). - StaticParserOnly, - - /// MvnCli succeeded for some files, static parser used for failed files. - Mixed, -} - -/// -/// Enum representing why fallback occurred. -/// -internal enum MavenFallbackReason -{ - /// No fallback was needed. - None, - - /// Maven CLI was explicitly disabled via the CD_MAVEN_DISABLE_CLI environment variable. - MvnCliDisabledByUser, - - /// Maven CLI was not available in PATH. - MavenCliNotAvailable, - - /// MvnCli failed due to authentication error (401/403). - AuthenticationFailure, - - /// MvnCli failed due to other reasons. - OtherMvnCliFailure, -} - -/// -/// Experimental Maven detector that combines MvnCli detection with static pom.xml parsing fallback. -/// Runs MvnCli detection first (like standard MvnCliComponentDetector), then checks if detection -/// produced any results. If MvnCli fails for any pom.xml, falls back to static parsing for failed files. -/// -public class MavenWithFallbackDetector : FileComponentDetector, IExperimentalDetector -{ - /// - /// Environment variable to disable MvnCli and use only static pom.xml parsing. - /// Set to "true" to disable MvnCli detection. - /// Usage: Set CD_MAVEN_DISABLE_CLI=true as a pipeline/environment variable. - /// - internal const string DisableMvnCliEnvVar = "CD_MAVEN_DISABLE_CLI"; - - private const string MavenManifest = "pom.xml"; - private const string MavenXmlNamespace = "http://maven.apache.org/POM/4.0.0"; - private const string ProjNamespace = "proj"; - private const string DependencyNode = "//proj:dependency"; - - private const string GroupIdSelector = "groupId"; - private const string ArtifactIdSelector = "artifactId"; - private const string VersionSelector = "version"; - - private static readonly Regex VersionRegex = new( - @"^\$\{(.*)\}$", - RegexOptions.Compiled | RegexOptions.IgnoreCase); - - // Auth error patterns to detect in Maven error output - private static readonly string[] AuthErrorPatterns = - [ - "401", - "403", - "Unauthorized", - "Access denied", - ]; - - // Pattern to extract failed endpoint URL from Maven error messages - private static readonly Regex EndpointRegex = new( - @"https?://[^\s\]\)>]+", - RegexOptions.Compiled | RegexOptions.IgnoreCase); - - /// - /// Maximum time allowed for the OnPrepareDetectionAsync phase. - /// This is a safety guardrail to prevent hangs in the experimental detector. - /// Most repos should complete the full Maven CLI scan within this window. - /// - private static readonly TimeSpan PrepareDetectionTimeout = TimeSpan.FromMinutes(5); - - private readonly IMavenCommandService mavenCommandService; - private readonly IEnvironmentVariableService envVarService; - private readonly IFileUtilityService fileUtilityService; - private readonly Dictionary documentsLoaded = []; - - // Track original pom.xml files for potential fallback - private readonly ConcurrentBag originalPomFiles = []; - - // Track Maven CLI errors for analysis - private readonly ConcurrentBag mavenCliErrors = []; - private readonly ConcurrentBag failedEndpoints = []; - - // Telemetry tracking - private MavenDetectionMethod usedDetectionMethod = MavenDetectionMethod.None; - private MavenFallbackReason fallbackReason = MavenFallbackReason.None; - private int mvnCliComponentCount; - private int staticParserComponentCount; - private bool mavenCliAvailable; - - public MavenWithFallbackDetector( - IComponentStreamEnumerableFactory componentStreamEnumerableFactory, - IObservableDirectoryWalkerFactory walkerFactory, - IMavenCommandService mavenCommandService, - IEnvironmentVariableService envVarService, - IFileUtilityService fileUtilityService, - ILogger logger) - { - this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; - this.Scanner = walkerFactory; - this.mavenCommandService = mavenCommandService; - this.envVarService = envVarService; - this.fileUtilityService = fileUtilityService; - this.Logger = logger; - } - - public override string Id => "MavenWithFallback"; - - public override IList SearchPatterns => [MavenManifest]; - - public override IEnumerable SupportedComponentTypes => [ComponentType.Maven]; - - public override int Version => 1; - - public override IEnumerable Categories => [Enum.GetName(typeof(DetectorClass), DetectorClass.Maven)]; - - private void LogDebug(string message) => - this.Logger.LogDebug("{DetectorId}: {Message}", this.Id, message); - - private void LogInfo(string message) => - this.Logger.LogInformation("{DetectorId}: {Message}", this.Id, message); - - private void LogWarning(string message) => - this.Logger.LogWarning("{DetectorId}: {Message}", this.Id, message); - - protected override async Task> OnPrepareDetectionAsync( - IObservable processRequests, - IDictionary detectorArgs, - CancellationToken cancellationToken = default) - { - // Wrap the entire method in a try-catch with timeout to protect against hangs. - // OnPrepareDetectionAsync doesn't have the same guardrails as OnFileFoundAsync, - // so we need to be extra careful in this experimental detector. - try - { - using var timeoutCts = new CancellationTokenSource(PrepareDetectionTimeout); - using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCts.Token); - - return await this.OnPrepareDetectionCoreAsync(processRequests, linkedCts.Token); - } - catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) - { - // Timeout occurred (not user cancellation) - this.LogWarning($"OnPrepareDetectionAsync timed out after {PrepareDetectionTimeout.TotalMinutes} minutes. Falling back to static pom.xml parsing."); - this.Telemetry["TimedOut"] = "true"; - this.usedDetectionMethod = MavenDetectionMethod.StaticParserOnly; - this.fallbackReason = MavenFallbackReason.OtherMvnCliFailure; - return processRequests; - } - catch (Exception ex) - { - // Unexpected error - log and fall back to static parsing - this.LogWarning($"OnPrepareDetectionAsync failed with unexpected error: {ex.Message}. Falling back to static pom.xml parsing."); - this.Telemetry["PrepareDetectionError"] = ex.GetType().Name; - this.usedDetectionMethod = MavenDetectionMethod.StaticParserOnly; - this.fallbackReason = MavenFallbackReason.OtherMvnCliFailure; - return processRequests; - } - } - - /// - /// Core implementation of OnPrepareDetectionAsync, called within the timeout wrapper. - /// - private async Task> OnPrepareDetectionCoreAsync( - IObservable processRequests, - CancellationToken cancellationToken) - { - // Check if we should skip Maven CLI and use static parsing only - if (this.ShouldSkipMavenCli()) - { - return processRequests; - } - - // Check if Maven CLI is available - if (!await this.TryInitializeMavenCliAsync()) - { - return processRequests; - } - - // Run Maven CLI detection on all pom.xml files - // Returns deps files for CLI successes, pom.xml files for CLI failures - return await this.RunMavenCliDetectionAsync(processRequests, cancellationToken); - } - - /// - /// Checks if Maven CLI should be skipped due to environment variable configuration. - /// - /// True if Maven CLI should be skipped; otherwise, false. - private bool ShouldSkipMavenCli() - { - if (this.envVarService.IsEnvironmentVariableValueTrue(DisableMvnCliEnvVar)) - { - this.LogInfo($"MvnCli detection disabled via {DisableMvnCliEnvVar} environment variable. Using static pom.xml parsing only."); - this.usedDetectionMethod = MavenDetectionMethod.StaticParserOnly; - this.fallbackReason = MavenFallbackReason.MvnCliDisabledByUser; - this.mavenCliAvailable = false; - return true; - } - - return false; - } - - /// - /// Checks if Maven CLI is available. - /// - /// True if Maven CLI is available; otherwise, false. - private async Task TryInitializeMavenCliAsync() - { - this.mavenCliAvailable = await this.mavenCommandService.MavenCLIExistsAsync(); - - if (!this.mavenCliAvailable) - { - this.LogInfo("Maven CLI not found in PATH. Will use static pom.xml parsing only."); - this.usedDetectionMethod = MavenDetectionMethod.StaticParserOnly; - this.fallbackReason = MavenFallbackReason.MavenCliNotAvailable; - return false; - } - - this.LogDebug("Maven CLI is available. Running MvnCli detection."); - return true; - } - - /// - /// Runs Maven CLI detection on all root pom.xml files. - /// For each pom.xml, if CLI succeeds, the deps file is added to results. - /// If CLI fails, all pom.xml files under that directory are added for static parsing fallback. - /// - /// The incoming process requests. - /// Cancellation token for the operation. - /// An observable of process requests (deps files for CLI success, pom.xml for CLI failure). - private async Task> RunMavenCliDetectionAsync( - IObservable processRequests, - CancellationToken cancellationToken) - { - var results = new ConcurrentBag(); - var failedDirectories = new ConcurrentBag(); - var cliSuccessCount = 0; - var cliFailureCount = 0; - - // Process pom.xml files sequentially to match MvnCliComponentDetector behavior. - // Sequential execution avoids Maven local repository lock contention and - // reduces memory pressure from concurrent Maven JVM processes. - var processPomFile = new ActionBlock( - async processRequest => - { - // Check for cancellation before processing each pom.xml - cancellationToken.ThrowIfCancellationRequested(); - - // Store original pom.xml for telemetry - this.originalPomFiles.Add(processRequest); - - var pomFile = processRequest.ComponentStream; - var pomDir = Path.GetDirectoryName(pomFile.Location); - var depsFileName = this.mavenCommandService.BcdeMvnDependencyFileName; - var depsFilePath = Path.Combine(pomDir, depsFileName); - - // Generate dependency file using Maven CLI. - // Note: If both MvnCliComponentDetector and this detector are enabled, - // they may run Maven CLI on the same pom.xml independently. - var result = await this.mavenCommandService.GenerateDependenciesFileAsync( - processRequest, - cancellationToken); - - if (result.Success) - { - // CLI succeeded - read the generated deps file - // We read the file here (rather than in MavenCommandService) to avoid - // unnecessary I/O for callers like MvnCliComponentDetector that scan for files later. - string depsFileContent = null; - if (this.fileUtilityService.Exists(depsFilePath)) - { - depsFileContent = this.fileUtilityService.ReadAllText(depsFilePath); - } - - if (!string.IsNullOrEmpty(depsFileContent)) - { - Interlocked.Increment(ref cliSuccessCount); - results.Add(new ProcessRequest - { - ComponentStream = new ComponentStream - { - Stream = new MemoryStream(Encoding.UTF8.GetBytes(depsFileContent)), - Location = depsFilePath, - Pattern = depsFileName, - }, - SingleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder( - Path.Combine(pomDir, MavenManifest)), - }); - } - else - { - // CLI reported success but deps file is missing or empty - treat as failure - Interlocked.Increment(ref cliFailureCount); - failedDirectories.Add(pomDir); - this.LogWarning($"Maven CLI succeeded but deps file not found or empty: {depsFilePath}"); - } - } - else - { - // CLI failed - track directory for nested pom.xml scanning - Interlocked.Increment(ref cliFailureCount); - failedDirectories.Add(pomDir); - - // Capture error output for later analysis - if (!string.IsNullOrWhiteSpace(result.ErrorOutput)) - { - this.mavenCliErrors.Add(result.ErrorOutput); - } - } - }, - new ExecutionDataflowBlockOptions - { - CancellationToken = cancellationToken, - }); - - await this.RemoveNestedPomXmls(processRequests, cancellationToken).ForEachAsync( - processRequest => - { - processPomFile.Post(processRequest); - }, - cancellationToken); - - processPomFile.Complete(); - await processPomFile.Completion; - - // For failed directories, scan and add all pom.xml files for static parsing - if (!failedDirectories.IsEmpty) - { - cancellationToken.ThrowIfCancellationRequested(); - - var staticParsingRequests = this.GetAllPomFilesInDirectories( - failedDirectories.ToHashSet(StringComparer.OrdinalIgnoreCase), - cancellationToken); - - foreach (var request in staticParsingRequests) - { - cancellationToken.ThrowIfCancellationRequested(); - results.Add(request); - } - } - - // Determine detection method based on results - this.DetermineDetectionMethod(cliSuccessCount, cliFailureCount); - - this.LogDebug($"Maven CLI processing complete: {cliSuccessCount} succeeded, {cliFailureCount} failed out of {this.originalPomFiles.Count} root pom.xml files."); - - return results.ToObservable(); - } - - /// - /// Gets all pom.xml files in the specified directories and their subdirectories for static parsing. - /// - /// The directories to scan for pom.xml files. - /// Cancellation token for the operation. - /// ProcessRequests for all pom.xml files in the specified directories. - private List GetAllPomFilesInDirectories(HashSet directories, CancellationToken cancellationToken) - { - this.LogDebug($"Scanning for pom.xml files in {directories.Count} failed directories for static parsing fallback."); - - // Normalize directories once for efficient lookup - var normalizedDirs = directories - .Select(d => d.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar) - .ToList(); - - var results = new List(); - - foreach (var componentStream in this.ComponentStreamEnumerableFactory - .GetComponentStreams( - this.CurrentScanRequest.SourceDirectory, - [MavenManifest], - this.CurrentScanRequest.DirectoryExclusionPredicate)) - { - cancellationToken.ThrowIfCancellationRequested(); - - var fileDir = Path.GetDirectoryName(componentStream.Location); - var normalizedFileDir = fileDir.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar; - - // Include if this file is in or under any failed directory - // Use pre-normalized directories for efficient comparison - var isInFailedDirectory = normalizedDirs.Any(fd => - normalizedFileDir.Equals(fd, StringComparison.OrdinalIgnoreCase) || - normalizedFileDir.StartsWith(fd, StringComparison.OrdinalIgnoreCase)); - - if (!isInFailedDirectory) - { - continue; - } - - using var reader = new StreamReader(componentStream.Stream); - var content = reader.ReadToEnd(); - results.Add(new ProcessRequest - { - ComponentStream = new ComponentStream - { - Stream = new MemoryStream(Encoding.UTF8.GetBytes(content)), - Location = componentStream.Location, - Pattern = MavenManifest, - }, - SingleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder(componentStream.Location), - }); - } - - return results; - } - - /// - /// Determines the detection method based on CLI success/failure counts and analyzes any failures. - /// - /// Number of successful CLI executions. - /// Number of failed CLI executions. - private void DetermineDetectionMethod(int cliSuccessCount, int cliFailureCount) - { - if (cliFailureCount == 0 && cliSuccessCount > 0) - { - this.usedDetectionMethod = MavenDetectionMethod.MvnCliOnly; - this.LogDebug("All pom.xml files processed successfully with Maven CLI."); - } - else if (cliSuccessCount == 0 && cliFailureCount > 0) - { - this.usedDetectionMethod = MavenDetectionMethod.StaticParserOnly; - this.LogWarning("Maven CLI failed for all pom.xml files. Using static parsing fallback."); - this.AnalyzeMvnCliFailure(); - } - else if (cliSuccessCount > 0 && cliFailureCount > 0) - { - this.usedDetectionMethod = MavenDetectionMethod.Mixed; - this.LogWarning($"Maven CLI failed for {cliFailureCount} pom.xml files. Using mixed detection."); - this.AnalyzeMvnCliFailure(); - } - } - - protected override Task OnFileFoundAsync( - ProcessRequest processRequest, - IDictionary detectorArgs, - CancellationToken cancellationToken = default) - { - var pattern = processRequest.ComponentStream.Pattern; - - if (pattern == this.mavenCommandService.BcdeMvnDependencyFileName) - { - // Process MvnCli result - this.ProcessMvnCliResult(processRequest); - } - else - { - // Process via static XML parsing - this.ProcessPomFileStatically(processRequest); - } - - return Task.CompletedTask; - } - - protected override Task OnDetectionFinishedAsync() - { - // Record telemetry - this.Telemetry["DetectionMethod"] = this.usedDetectionMethod.ToString(); - this.Telemetry["FallbackReason"] = this.fallbackReason.ToString(); - this.Telemetry["MvnCliComponentCount"] = this.mvnCliComponentCount.ToString(); - this.Telemetry["StaticParserComponentCount"] = this.staticParserComponentCount.ToString(); - this.Telemetry["TotalComponentCount"] = (this.mvnCliComponentCount + this.staticParserComponentCount).ToString(); - this.Telemetry["MavenCliAvailable"] = this.mavenCliAvailable.ToString(); - this.Telemetry["OriginalPomFileCount"] = this.originalPomFiles.Count.ToString(); - - if (!this.failedEndpoints.IsEmpty) - { - this.Telemetry["FailedEndpoints"] = string.Join(";", this.failedEndpoints.Distinct().Take(10)); - } - - this.LogInfo($"Detection completed. Method: {this.usedDetectionMethod}, " + - $"FallbackReason: {this.fallbackReason}, " + - $"MvnCli components: {this.mvnCliComponentCount}, " + - $"Static parser components: {this.staticParserComponentCount}"); - - return Task.CompletedTask; - } - - /// - /// Analyzes Maven CLI failure by checking logged errors for authentication issues. - /// - private void AnalyzeMvnCliFailure() - { - // Check if any recorded errors indicate authentication failure - var hasAuthError = this.mavenCliErrors.Any(this.IsAuthenticationError); - - if (hasAuthError) - { - this.fallbackReason = MavenFallbackReason.AuthenticationFailure; - - // Extract failed endpoints from error messages - foreach (var endpoint in this.mavenCliErrors.SelectMany(this.ExtractFailedEndpoints)) - { - this.failedEndpoints.Add(endpoint); - } - - this.LogAuthErrorGuidance(); - } - else - { - this.fallbackReason = MavenFallbackReason.OtherMvnCliFailure; - this.LogWarning("Maven CLI failed. Check Maven logs for details."); - } - } - - private void ProcessMvnCliResult(ProcessRequest processRequest) - { - this.mavenCommandService.ParseDependenciesFile(processRequest); - - // Try to delete the deps file - try - { - File.Delete(processRequest.ComponentStream.Location); - } - catch - { - // Ignore deletion errors - } - - // Count components registered to this specific file's recorder to avoid race conditions - // when OnFileFoundAsync runs concurrently for multiple files. - var componentsInFile = processRequest.SingleFileComponentRecorder.GetDetectedComponents().Count; - Interlocked.Add(ref this.mvnCliComponentCount, componentsInFile); - } - - private void ProcessPomFileStatically(ProcessRequest processRequest) - { - var file = processRequest.ComponentStream; - var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; - var filePath = file.Location; - - try - { - var document = new XmlDocument(); - document.Load(file.Stream); - - lock (this.documentsLoaded) - { - this.documentsLoaded.TryAdd(file.Location, document); - } - - var namespaceManager = new XmlNamespaceManager(document.NameTable); - namespaceManager.AddNamespace(ProjNamespace, MavenXmlNamespace); - - var dependencyList = document.SelectNodes(DependencyNode, namespaceManager); - var componentsFoundInFile = 0; - - foreach (XmlNode dependency in dependencyList) - { - var groupId = dependency[GroupIdSelector]?.InnerText; - var artifactId = dependency[ArtifactIdSelector]?.InnerText; - - if (groupId == null || artifactId == null) - { - continue; - } - - var version = dependency[VersionSelector]; - if (version != null && !version.InnerText.Contains(',')) - { - var versionRef = version.InnerText.Trim('[', ']'); - string versionString; - - if (versionRef.StartsWith("${")) - { - versionString = this.ResolveVersion(versionRef, document, file.Location); - } - else - { - versionString = versionRef; - } - - if (!versionString.StartsWith("${")) - { - var component = new MavenComponent(groupId, artifactId, versionString); - var detectedComponent = new DetectedComponent(component); - singleFileComponentRecorder.RegisterUsage(detectedComponent); - componentsFoundInFile++; - } - else - { - this.Logger.LogDebug( - "Version string {Version} for component {Group}/{Artifact} is invalid or unsupported and a component will not be recorded.", - versionString, - groupId, - artifactId); - } - } - else - { - this.Logger.LogDebug( - "Version string for component {Group}/{Artifact} is invalid or unsupported and a component will not be recorded.", - groupId, - artifactId); - } - } - - Interlocked.Add(ref this.staticParserComponentCount, componentsFoundInFile); - } - catch (Exception e) - { - this.Logger.LogError(e, "Failed to read file {Path}", filePath); - } - } - - private bool IsAuthenticationError(string errorMessage) - { - if (string.IsNullOrWhiteSpace(errorMessage)) - { - return false; - } - - return AuthErrorPatterns.Any(pattern => - errorMessage.Contains(pattern, StringComparison.OrdinalIgnoreCase)); - } - - private IEnumerable ExtractFailedEndpoints(string errorMessage) - { - if (string.IsNullOrWhiteSpace(errorMessage)) - { - return []; - } - - return EndpointRegex.Matches(errorMessage) - .Select(m => m.Value) - .Distinct(); - } - - private void LogAuthErrorGuidance() - { - var guidance = new StringBuilder(); - guidance.AppendLine("Maven CLI failed with authentication errors."); - - if (!this.failedEndpoints.IsEmpty) - { - guidance.AppendLine("The following Maven repository endpoints had authentication failures:"); - foreach (var endpoint in this.failedEndpoints.Distinct().Take(5)) - { - guidance.AppendLine($" - {endpoint}"); - } - - guidance.AppendLine(" Ensure your pipeline has access to these Maven repositories."); - } - - guidance.AppendLine("Note: Falling back to static pom.xml parsing."); - - this.LogWarning(guidance.ToString()); - } - - private string ResolveVersion(string versionString, XmlDocument currentDocument, string currentDocumentFileLocation) - { - var returnedVersionString = versionString; - var match = VersionRegex.Match(versionString); - - if (match.Success) - { - var variable = match.Groups[1].Captures[0].ToString(); - var replacement = this.ReplaceVariable(variable, currentDocument, currentDocumentFileLocation); - returnedVersionString = versionString.Replace("${" + variable + "}", replacement); - } - - return returnedVersionString; - } - - private string ReplaceVariable(string variable, XmlDocument currentDocument, string currentDocumentFileLocation) - { - var result = this.FindVariableInDocument(currentDocument, currentDocumentFileLocation, variable); - if (result != null) - { - return result; - } - - lock (this.documentsLoaded) - { - foreach (var pathDocumentPair in this.documentsLoaded) - { - var path = pathDocumentPair.Key; - var document = pathDocumentPair.Value; - result = this.FindVariableInDocument(document, path, variable); - if (result != null) - { - return result; - } - } - } - - return $"${{{variable}}}"; - } - - private string FindVariableInDocument(XmlDocument document, string path, string variable) - { - try - { - var namespaceManager = new XmlNamespaceManager(document.NameTable); - namespaceManager.AddNamespace(ProjNamespace, MavenXmlNamespace); - - var nodeListProject = document.SelectNodes($"//proj:{variable}", namespaceManager); - var nodeListProperties = document.SelectNodes($"//proj:properties/proj:{variable}", namespaceManager); - - if (nodeListProject.Count != 0) - { - return nodeListProject.Item(0).InnerText; - } - - if (nodeListProperties.Count != 0) - { - return nodeListProperties.Item(0).InnerText; - } - } - catch (Exception e) - { - this.Logger.LogError(e, "Failed to read file {Path}", path); - } - - return null; - } - - /// - /// Filters out nested pom.xml files, keeping only root-level ones. - /// A pom.xml is considered nested if there's another pom.xml in a parent directory. - /// - /// The incoming process requests for pom.xml files. - /// Cancellation token for the operation. - /// Process requests for only root-level pom.xml files. - private IObservable RemoveNestedPomXmls(IObservable componentStreams, CancellationToken cancellationToken) - { - return componentStreams - .ToList() - .SelectMany(allRequests => - { - cancellationToken.ThrowIfCancellationRequested(); - - // Build a list of all directories that contain a pom.xml, ordered by path length (shortest first). - // This ensures parent directories are checked before their children. - var pomDirectories = allRequests - .Select(r => NormalizeDirectoryPath(Path.GetDirectoryName(r.ComponentStream.Location))) - .Distinct(StringComparer.OrdinalIgnoreCase) - .OrderBy(d => d.Length) - .ToList(); - - var filteredRequests = new List(); - - foreach (var request in allRequests) - { - cancellationToken.ThrowIfCancellationRequested(); - - var location = NormalizeDirectoryPath(Path.GetDirectoryName(request.ComponentStream.Location)); - var isNested = false; - - foreach (var pomDirectory in pomDirectories) - { - if (pomDirectory.Length >= location.Length) - { - // Since the list is ordered by length, if the pomDirectory is longer than - // or equal to the location, there are no possible parent directories left. - break; - } - - if (location.StartsWith(pomDirectory, StringComparison.OrdinalIgnoreCase)) - { - this.LogDebug($"Ignoring {MavenManifest} at {location}, as it has a parent {MavenManifest} at {pomDirectory}."); - isNested = true; - break; - } - } - - if (!isNested) - { - this.LogDebug($"Discovered {request.ComponentStream.Location}."); - filteredRequests.Add(request); - } - } - - return filteredRequests; - }); - - // Normalizes a directory path by ensuring it ends with a directory separator. - // This prevents false matches like "C:\foo" matching "C:\foobar". - static string NormalizeDirectoryPath(string path) => - path.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar; - } -} diff --git a/src/Microsoft.ComponentDetection.Detectors/maven/MvnCliComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/maven/MvnCliComponentDetector.cs index a93f5c6b2..642b03445 100644 --- a/src/Microsoft.ComponentDetection.Detectors/maven/MvnCliComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/maven/MvnCliComponentDetector.cs @@ -8,30 +8,136 @@ namespace Microsoft.ComponentDetection.Detectors.Maven; using System.Linq; using System.Reactive.Linq; using System.Text; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; +using System.Xml; using Microsoft.ComponentDetection.Common; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.Extensions.Logging; +/// +/// Enum representing which detection method was used. +/// +internal enum MavenDetectionMethod +{ + /// No detection performed. + None, + + /// MvnCli was used successfully for all files. + MvnCliOnly, + + /// Static parser was used for all files (MvnCli not available or failed completely). + StaticParserOnly, + + /// MvnCli succeeded for some files, static parser used for failed files. + Mixed, +} + +/// +/// Enum representing why fallback occurred. +/// +internal enum MavenFallbackReason +{ + /// No fallback was needed. + None, + + /// Maven CLI was explicitly disabled via the CD_MAVEN_DISABLE_CLI environment variable. + MvnCliDisabledByUser, + + /// Maven CLI was not available in PATH. + MavenCliNotAvailable, + + /// MvnCli failed due to authentication error (401/403). + AuthenticationFailure, + + /// MvnCli failed due to other reasons. + OtherMvnCliFailure, +} + +/// +/// Maven detector that combines Maven CLI detection with static pom.xml parsing fallback. +/// Runs Maven CLI detection first, then checks if detection produced any results. +/// If Maven CLI fails for any pom.xml, falls back to static parsing for failed files. +/// public class MvnCliComponentDetector : FileComponentDetector { + /// + /// Environment variable to disable MvnCli and use only static pom.xml parsing. + /// Set to "true" to disable MvnCli detection. + /// Usage: Set CD_MAVEN_DISABLE_CLI=true as a pipeline/environment variable. + /// + internal const string DisableMvnCliEnvVar = "CD_MAVEN_DISABLE_CLI"; + private const string MavenManifest = "pom.xml"; + private const string MavenXmlNamespace = "http://maven.apache.org/POM/4.0.0"; + private const string ProjNamespace = "proj"; + private const string DependencyNode = "//proj:dependency"; + + private const string GroupIdSelector = "groupId"; + private const string ArtifactIdSelector = "artifactId"; + private const string VersionSelector = "version"; + + private static readonly Regex VersionRegex = new( + @"^\$\{(.*)\}$", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + + // Auth error patterns to detect in Maven error output + private static readonly string[] AuthErrorPatterns = + [ + "401", + "403", + "Unauthorized", + "Access denied", + ]; + + // Pattern to extract failed endpoint URL from Maven error messages + private static readonly Regex EndpointRegex = new( + @"https?://[^\s\]\)>]+", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + + /// + /// Maximum time allowed for the OnPrepareDetectionAsync phase. + /// This is a safety guardrail to prevent hangs. + /// Most repos should complete the full Maven CLI scan within this window. + /// + private static readonly TimeSpan PrepareDetectionTimeout = TimeSpan.FromMinutes(5); private readonly IMavenCommandService mavenCommandService; + private readonly IEnvironmentVariableService envVarService; + private readonly IFileUtilityService fileUtilityService; + private readonly Dictionary documentsLoaded = []; + + // Track original pom.xml files for potential fallback + private readonly ConcurrentBag originalPomFiles = []; + + // Track Maven CLI errors for analysis + private readonly ConcurrentBag mavenCliErrors = []; + private readonly ConcurrentBag failedEndpoints = []; + + // Telemetry tracking + private MavenDetectionMethod usedDetectionMethod = MavenDetectionMethod.None; + private MavenFallbackReason fallbackReason = MavenFallbackReason.None; + private int mvnCliComponentCount; + private int staticParserComponentCount; + private bool mavenCliAvailable; public MvnCliComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, IMavenCommandService mavenCommandService, + IEnvironmentVariableService envVarService, + IFileUtilityService fileUtilityService, ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; this.mavenCommandService = mavenCommandService; + this.envVarService = envVarService; + this.fileUtilityService = fileUtilityService; this.Logger = logger; } @@ -41,41 +147,247 @@ public MvnCliComponentDetector( public override IEnumerable SupportedComponentTypes => [ComponentType.Maven]; - public override int Version => 4; + public override int Version => 5; public override IEnumerable Categories => [Enum.GetName(typeof(DetectorClass), DetectorClass.Maven)]; - private void LogDebugWithId(string message) + private void LogDebug(string message) => + this.Logger.LogDebug("{DetectorId}: {Message}", this.Id, message); + + private void LogInfo(string message) => + this.Logger.LogInformation("{DetectorId}: {Message}", this.Id, message); + + private void LogWarning(string message) => + this.Logger.LogWarning("{DetectorId}: {Message}", this.Id, message); + + protected override async Task> OnPrepareDetectionAsync( + IObservable processRequests, + IDictionary detectorArgs, + CancellationToken cancellationToken = default) { - this.Logger.LogDebug("{DetectorId} detector: {Message}", this.Id, message); + // Wrap the entire method in a try-catch with timeout to protect against hangs. + // OnPrepareDetectionAsync doesn't have the same guardrails as OnFileFoundAsync, + // so we need to be extra careful. + try + { + using var timeoutCts = new CancellationTokenSource(PrepareDetectionTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCts.Token); + + return await this.OnPrepareDetectionCoreAsync(processRequests, linkedCts.Token); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + // Timeout occurred (not user cancellation) + this.LogWarning($"OnPrepareDetectionAsync timed out after {PrepareDetectionTimeout.TotalMinutes} minutes. Falling back to static pom.xml parsing."); + this.Telemetry["TimedOut"] = "true"; + this.usedDetectionMethod = MavenDetectionMethod.StaticParserOnly; + this.fallbackReason = MavenFallbackReason.OtherMvnCliFailure; + return processRequests; + } + catch (Exception ex) + { + // Unexpected error - log and fall back to static parsing + this.LogWarning($"OnPrepareDetectionAsync failed with unexpected error: {ex.Message}. Falling back to static pom.xml parsing."); + this.Telemetry["PrepareDetectionError"] = ex.GetType().Name; + this.usedDetectionMethod = MavenDetectionMethod.StaticParserOnly; + this.fallbackReason = MavenFallbackReason.OtherMvnCliFailure; + return processRequests; + } } - protected override async Task> OnPrepareDetectionAsync(IObservable processRequests, IDictionary detectorArgs, CancellationToken cancellationToken = default) + /// + /// Core implementation of OnPrepareDetectionAsync, called within the timeout wrapper. + /// + private async Task> OnPrepareDetectionCoreAsync( + IObservable processRequests, + CancellationToken cancellationToken) { - if (!await this.mavenCommandService.MavenCLIExistsAsync()) + // Check if we should skip Maven CLI and use static parsing only + if (this.ShouldSkipMavenCli()) { - this.LogDebugWithId("Skipping maven detection as maven is not available in the local PATH."); - return Enumerable.Empty().ToObservable(); + return processRequests; } - var processPomFile = new ActionBlock(x => this.mavenCommandService.GenerateDependenciesFileAsync(x, cancellationToken)); + // Check if Maven CLI is available + if (!await this.TryInitializeMavenCliAsync()) + { + return processRequests; + } + + // Run Maven CLI detection on all pom.xml files + // Returns deps files for CLI successes, pom.xml files for CLI failures + return await this.RunMavenCliDetectionAsync(processRequests, cancellationToken); + } - await this.RemoveNestedPomXmls(processRequests).ForEachAsync(processRequest => + /// + /// Checks if Maven CLI should be skipped due to environment variable configuration. + /// + /// True if Maven CLI should be skipped; otherwise, false. + private bool ShouldSkipMavenCli() + { + if (this.envVarService.IsEnvironmentVariableValueTrue(DisableMvnCliEnvVar)) { - processPomFile.Post(processRequest); - }); + this.LogInfo($"MvnCli detection disabled via {DisableMvnCliEnvVar} environment variable. Using static pom.xml parsing only."); + this.usedDetectionMethod = MavenDetectionMethod.StaticParserOnly; + this.fallbackReason = MavenFallbackReason.MvnCliDisabledByUser; + this.mavenCliAvailable = false; + return true; + } - processPomFile.Complete(); + return false; + } + + /// + /// Checks if Maven CLI is available. + /// + /// True if Maven CLI is available; otherwise, false. + private async Task TryInitializeMavenCliAsync() + { + this.mavenCliAvailable = await this.mavenCommandService.MavenCLIExistsAsync(); + + if (!this.mavenCliAvailable) + { + this.LogInfo("Maven CLI not found in PATH. Will use static pom.xml parsing only."); + this.usedDetectionMethod = MavenDetectionMethod.StaticParserOnly; + this.fallbackReason = MavenFallbackReason.MavenCliNotAvailable; + return false; + } + + this.LogDebug("Maven CLI is available. Running MvnCli detection."); + return true; + } + + /// + /// Runs Maven CLI detection on all root pom.xml files. + /// For each pom.xml, if CLI succeeds, the deps file is added to results. + /// If CLI fails, all pom.xml files under that directory are added for static parsing fallback. + /// + /// The incoming process requests. + /// Cancellation token for the operation. + /// An observable of process requests (deps files for CLI success, pom.xml for CLI failure). + private async Task> RunMavenCliDetectionAsync( + IObservable processRequests, + CancellationToken cancellationToken) + { + var results = new ConcurrentBag(); + var failedDirectories = new ConcurrentBag(); + var cliSuccessCount = 0; + var cliFailureCount = 0; + + // Process pom.xml files sequentially to avoid Maven local repository lock contention. + // Sequential execution reduces memory pressure from concurrent Maven JVM processes. + var processPomFile = new ActionBlock( + async processRequest => + { + // Check for cancellation before processing each pom.xml + cancellationToken.ThrowIfCancellationRequested(); + + // Store original pom.xml for telemetry + this.originalPomFiles.Add(processRequest); + + var pomFile = processRequest.ComponentStream; + var pomDir = Path.GetDirectoryName(pomFile.Location); + + // Generate dependency file using Maven CLI. + var result = await this.mavenCommandService.GenerateDependenciesFileAsync( + processRequest, + cancellationToken); + + if (result.Success) + { + // CLI succeeded - read the generated deps file + // We read the file here (rather than in MavenCommandService) to avoid + // unnecessary I/O for callers that scan for files later. + string depsFileContent = null; + var depsFileName = this.mavenCommandService.BcdeMvnDependencyFileName; + var depsFilePath = Path.Combine(pomDir, depsFileName); + if (this.fileUtilityService.Exists(depsFilePath)) + { + depsFileContent = this.fileUtilityService.ReadAllText(depsFilePath); + } + if (!string.IsNullOrEmpty(depsFileContent)) + { + Interlocked.Increment(ref cliSuccessCount); + results.Add(new ProcessRequest + { + ComponentStream = new ComponentStream + { + Stream = new MemoryStream(Encoding.UTF8.GetBytes(depsFileContent)), + Location = depsFilePath, + Pattern = depsFileName, + }, + SingleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder( + Path.Combine(pomDir, MavenManifest)), + }); + } + else + { + // CLI reported success but deps file is missing or empty - treat as failure + Interlocked.Increment(ref cliFailureCount); + failedDirectories.Add(pomDir); + this.LogWarning($"Maven CLI succeeded but deps file not found or empty: {depsFilePath}"); + } + } + else + { + // CLI failed - track directory for nested pom.xml scanning + Interlocked.Increment(ref cliFailureCount); + failedDirectories.Add(pomDir); + + // Capture error output for later analysis + if (!string.IsNullOrWhiteSpace(result.ErrorOutput)) + { + this.mavenCliErrors.Add(result.ErrorOutput); + } + } + }, + new ExecutionDataflowBlockOptions + { + CancellationToken = cancellationToken, + }); + + await this.RemoveNestedPomXmls(processRequests, cancellationToken).ForEachAsync( + processRequest => + { + processPomFile.Post(processRequest); + }, + cancellationToken); + + processPomFile.Complete(); await processPomFile.Completion; - this.LogDebugWithId($"Nested {MavenManifest} files processed successfully, retrieving generated dependency graphs."); + // For failed directories, scan and add all pom.xml files for static parsing + if (!failedDirectories.IsEmpty) + { + cancellationToken.ThrowIfCancellationRequested(); + + var staticParsingRequests = this.GetAllPomFilesInDirectories( + failedDirectories.ToHashSet(StringComparer.OrdinalIgnoreCase), + cancellationToken); - return this.ComponentStreamEnumerableFactory.GetComponentStreams(this.CurrentScanRequest.SourceDirectory, [this.mavenCommandService.BcdeMvnDependencyFileName], this.CurrentScanRequest.DirectoryExclusionPredicate) + foreach (var request in staticParsingRequests) + { + cancellationToken.ThrowIfCancellationRequested(); + results.Add(request); + } + } + + // Determine detection method based on results + this.DetermineDetectionMethod(cliSuccessCount, cliFailureCount); + + this.LogDebug($"Maven CLI processing complete: {cliSuccessCount} succeeded, {cliFailureCount} failed out of {this.originalPomFiles.Count} root pom.xml files. Retrieving generated dependency graphs."); + + // Use original MvnCliComponentDetector approach: scan entire source directory for ALL generated dependency files + // This ensures we find dependency files from submodules even if Maven CLI was only run on parent pom.xml + var allGeneratedDependencyFiles = this.ComponentStreamEnumerableFactory + .GetComponentStreams( + this.CurrentScanRequest.SourceDirectory, + [this.mavenCommandService.BcdeMvnDependencyFileName], + this.CurrentScanRequest.DirectoryExclusionPredicate) .Select(componentStream => { - // The file stream is going to be disposed after the iteration is finished - // so is necessary to read the content and keep it in memory, for further processing. + // Read and store content to avoid stream disposal issues using var reader = new StreamReader(componentStream.Stream); var content = reader.ReadToEnd(); return new ProcessRequest @@ -89,74 +401,440 @@ await this.RemoveNestedPomXmls(processRequests).ForEachAsync(processRequest => SingleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder( Path.Combine(Path.GetDirectoryName(componentStream.Location), MavenManifest)), }; - }) - .ToObservable(); + }); + + // Combine dependency files from CLI success with pom.xml files from CLI failures + return results.Concat(allGeneratedDependencyFiles).ToObservable(); + } + + protected override Task OnFileFoundAsync( + ProcessRequest processRequest, + IDictionary detectorArgs, + CancellationToken cancellationToken = default) + { + var pattern = processRequest.ComponentStream.Pattern; + + if (pattern == this.mavenCommandService.BcdeMvnDependencyFileName) + { + // Process MvnCli result + this.ProcessMvnCliResult(processRequest); + } + else + { + // Process via static XML parsing + this.ProcessPomFileStatically(processRequest); + } + + return Task.CompletedTask; } - protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) + protected override Task OnDetectionFinishedAsync() + { + // Record telemetry + this.Telemetry["DetectionMethod"] = this.usedDetectionMethod.ToString(); + this.Telemetry["FallbackReason"] = this.fallbackReason.ToString(); + this.Telemetry["MvnCliComponentCount"] = this.mvnCliComponentCount.ToString(); + this.Telemetry["StaticParserComponentCount"] = this.staticParserComponentCount.ToString(); + this.Telemetry["TotalComponentCount"] = (this.mvnCliComponentCount + this.staticParserComponentCount).ToString(); + this.Telemetry["MavenCliAvailable"] = this.mavenCliAvailable.ToString(); + this.Telemetry["OriginalPomFileCount"] = this.originalPomFiles.Count.ToString(); + + if (!this.failedEndpoints.IsEmpty) + { + this.Telemetry["FailedEndpoints"] = string.Join(";", this.failedEndpoints.Distinct().Take(10)); + } + + this.LogInfo($"Detection completed. Method: {this.usedDetectionMethod}, " + + $"FallbackReason: {this.fallbackReason}, " + + $"MvnCli components: {this.mvnCliComponentCount}, " + + $"Static parser components: {this.staticParserComponentCount}"); + + return Task.CompletedTask; + } + + /// + /// Gets all pom.xml files in the specified directories and their subdirectories for static parsing. + /// + /// The directories to scan for pom.xml files. + /// Cancellation token for the operation. + /// ProcessRequests for all pom.xml files in the specified directories. + private List GetAllPomFilesInDirectories(HashSet directories, CancellationToken cancellationToken) + { + this.LogDebug($"Scanning for pom.xml files in {directories.Count} failed directories for static parsing fallback."); + + // Normalize directories once for efficient lookup + var normalizedDirs = directories + .Select(d => d.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar) + .ToList(); + + var results = new List(); + + foreach (var componentStream in this.ComponentStreamEnumerableFactory + .GetComponentStreams( + this.CurrentScanRequest.SourceDirectory, + [MavenManifest], + this.CurrentScanRequest.DirectoryExclusionPredicate)) + { + cancellationToken.ThrowIfCancellationRequested(); + + var fileDir = Path.GetDirectoryName(componentStream.Location); + var normalizedFileDir = fileDir.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar; + + // Include if this file is in or under any failed directory + // Use pre-normalized directories for efficient comparison + var isInFailedDirectory = normalizedDirs.Any(fd => + normalizedFileDir.Equals(fd, StringComparison.OrdinalIgnoreCase) || + normalizedFileDir.StartsWith(fd, StringComparison.OrdinalIgnoreCase)); + + if (!isInFailedDirectory) + { + continue; + } + + using var reader = new StreamReader(componentStream.Stream); + var content = reader.ReadToEnd(); + results.Add(new ProcessRequest + { + ComponentStream = new ComponentStream + { + Stream = new MemoryStream(Encoding.UTF8.GetBytes(content)), + Location = componentStream.Location, + Pattern = MavenManifest, + }, + SingleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder(componentStream.Location), + }); + } + + return results; + } + + /// + /// Determines the detection method based on CLI success/failure counts and analyzes any failures. + /// + /// Number of successful CLI executions. + /// Number of failed CLI executions. + private void DetermineDetectionMethod(int cliSuccessCount, int cliFailureCount) + { + if (cliFailureCount == 0 && cliSuccessCount > 0) + { + this.usedDetectionMethod = MavenDetectionMethod.MvnCliOnly; + this.LogDebug("All pom.xml files processed successfully with Maven CLI."); + } + else if (cliSuccessCount == 0 && cliFailureCount > 0) + { + this.usedDetectionMethod = MavenDetectionMethod.StaticParserOnly; + this.LogWarning("Maven CLI failed for all pom.xml files. Using static parsing fallback."); + this.AnalyzeMvnCliFailure(); + } + else if (cliSuccessCount > 0 && cliFailureCount > 0) + { + this.usedDetectionMethod = MavenDetectionMethod.Mixed; + this.LogWarning($"Maven CLI failed for {cliFailureCount} pom.xml files. Using mixed detection."); + this.AnalyzeMvnCliFailure(); + } + } + + /// + /// Analyzes Maven CLI failure by checking logged errors for authentication issues. + /// + private void AnalyzeMvnCliFailure() + { + // Check if any recorded errors indicate authentication failure + var hasAuthError = this.mavenCliErrors.Any(this.IsAuthenticationError); + + if (hasAuthError) + { + this.fallbackReason = MavenFallbackReason.AuthenticationFailure; + + // Extract failed endpoints from error messages + foreach (var endpoint in this.mavenCliErrors.SelectMany(this.ExtractFailedEndpoints)) + { + this.failedEndpoints.Add(endpoint); + } + + this.LogAuthErrorGuidance(); + } + else + { + this.fallbackReason = MavenFallbackReason.OtherMvnCliFailure; + this.LogWarning("Maven CLI failed. Check Maven logs for details."); + } + } + + private void ProcessMvnCliResult(ProcessRequest processRequest) { this.mavenCommandService.ParseDependenciesFile(processRequest); - File.Delete(processRequest.ComponentStream.Location); + // Try to delete the deps file + try + { + File.Delete(processRequest.ComponentStream.Location); + } + catch + { + // Ignore deletion errors + } - await Task.CompletedTask; + // Count components registered to this specific file's recorder to avoid race conditions + // when OnFileFoundAsync runs concurrently for multiple files. + var componentsInFile = processRequest.SingleFileComponentRecorder.GetDetectedComponents().Count; + Interlocked.Add(ref this.mvnCliComponentCount, componentsInFile); } - private IObservable RemoveNestedPomXmls(IObservable componentStreams) + private void ProcessPomFileStatically(ProcessRequest processRequest) { - var directoryItemFacades = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); - var topLevelDirectories = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + var file = processRequest.ComponentStream; + var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; + var filePath = file.Location; - return Observable.Create(s => + try { - return componentStreams.Subscribe( - processRequest => + var document = new XmlDocument(); + document.Load(file.Stream); + + lock (this.documentsLoaded) + { + this.documentsLoaded.TryAdd(file.Location, document); + } + + var namespaceManager = new XmlNamespaceManager(document.NameTable); + namespaceManager.AddNamespace(ProjNamespace, MavenXmlNamespace); + + var dependencyList = document.SelectNodes(DependencyNode, namespaceManager); + var componentsFoundInFile = 0; + + foreach (XmlNode dependency in dependencyList) + { + var groupId = dependency[GroupIdSelector]?.InnerText; + var artifactId = dependency[ArtifactIdSelector]?.InnerText; + + if (groupId == null || artifactId == null) { - var item = processRequest.ComponentStream; - var currentDir = item.Location; - DirectoryItemFacadeOptimized last = null; - while (!string.IsNullOrWhiteSpace(currentDir)) + continue; + } + + var version = dependency[VersionSelector]; + if (version != null && !version.InnerText.Contains(',')) + { + var versionRef = version.InnerText.Trim('[', ']'); + string versionString; + + if (versionRef.StartsWith("${")) + { + versionString = this.ResolveVersion(versionRef, document, file.Location); + } + else { - currentDir = Path.GetDirectoryName(currentDir); + versionString = versionRef; + } - // We've reached the top / root - if (string.IsNullOrWhiteSpace(currentDir)) - { - // If our last directory isn't in our list of top level nodes, it should be added. This happens for the first processed item and then subsequent times we have a new root (edge cases with multiple hard drives, for example) - if (last != null && !topLevelDirectories.ContainsKey(last.Name)) - { - topLevelDirectories.TryAdd(last.Name, last); - } + if (!versionString.StartsWith("${")) + { + var component = new MavenComponent(groupId, artifactId, versionString); + var detectedComponent = new DetectedComponent(component); + singleFileComponentRecorder.RegisterUsage(detectedComponent); + componentsFoundInFile++; + } + else + { + this.Logger.LogDebug( + "Version string {Version} for component {Group}/{Artifact} is invalid or unsupported and a component will not be recorded.", + versionString, + groupId, + artifactId); + } + } + else + { + this.Logger.LogDebug( + "Version string for component {Group}/{Artifact} is invalid or unsupported and a component will not be recorded.", + groupId, + artifactId); + } + } + + Interlocked.Add(ref this.staticParserComponentCount, componentsFoundInFile); + } + catch (Exception e) + { + this.Logger.LogError(e, "Failed to read file {Path}", filePath); + } + } - this.LogDebugWithId($"Discovered {item.Location}."); + private bool IsAuthenticationError(string errorMessage) + { + if (string.IsNullOrWhiteSpace(errorMessage)) + { + return false; + } - // If we got to the top without finding a directory that had a pom.xml on the way, we yield. - s.OnNext(processRequest); - break; - } + return AuthErrorPatterns.Any(pattern => + errorMessage.Contains(pattern, StringComparison.OrdinalIgnoreCase)); + } - var current = directoryItemFacades.GetOrAdd(currentDir, _ => new DirectoryItemFacadeOptimized - { - Name = currentDir, - FileNames = [], - }); + private IEnumerable ExtractFailedEndpoints(string errorMessage) + { + if (string.IsNullOrWhiteSpace(errorMessage)) + { + return []; + } + + return EndpointRegex.Matches(errorMessage) + .Select(m => m.Value) + .Distinct(); + } - // If we didn't come from a directory, it's because we're just getting started. Our current directory should include the file that led to it showing up in the graph. - if (last == null) + private void LogAuthErrorGuidance() + { + var guidance = new StringBuilder(); + guidance.AppendLine("Maven CLI failed with authentication errors."); + + if (!this.failedEndpoints.IsEmpty) + { + guidance.AppendLine("The following Maven repository endpoints had authentication failures:"); + foreach (var endpoint in this.failedEndpoints.Distinct().Take(5)) + { + guidance.AppendLine($" - {endpoint}"); + } + + guidance.AppendLine(" Ensure your pipeline has access to these Maven repositories."); + } + + guidance.AppendLine("Note: Falling back to static pom.xml parsing."); + + this.LogWarning(guidance.ToString()); + } + + private string ResolveVersion(string versionString, XmlDocument currentDocument, string currentDocumentFileLocation) + { + var returnedVersionString = versionString; + var match = VersionRegex.Match(versionString); + + if (match.Success) + { + var variable = match.Groups[1].Captures[0].ToString(); + var replacement = this.ReplaceVariable(variable, currentDocument, currentDocumentFileLocation); + returnedVersionString = versionString.Replace("${" + variable + "}", replacement); + } + + return returnedVersionString; + } + + private string ReplaceVariable(string variable, XmlDocument currentDocument, string currentDocumentFileLocation) + { + var result = this.FindVariableInDocument(currentDocument, currentDocumentFileLocation, variable); + if (result != null) + { + return result; + } + + lock (this.documentsLoaded) + { + foreach (var pathDocumentPair in this.documentsLoaded) + { + var path = pathDocumentPair.Key; + var document = pathDocumentPair.Value; + result = this.FindVariableInDocument(document, path, variable); + if (result != null) + { + return result; + } + } + } + + return $"${{{variable}}}"; + } + + private string FindVariableInDocument(XmlDocument document, string path, string variable) + { + try + { + var namespaceManager = new XmlNamespaceManager(document.NameTable); + namespaceManager.AddNamespace(ProjNamespace, MavenXmlNamespace); + + var nodeListProject = document.SelectNodes($"//proj:{variable}", namespaceManager); + var nodeListProperties = document.SelectNodes($"//proj:properties/proj:{variable}", namespaceManager); + + if (nodeListProject.Count != 0) + { + return nodeListProject.Item(0).InnerText; + } + + if (nodeListProperties.Count != 0) + { + return nodeListProperties.Item(0).InnerText; + } + } + catch (Exception e) + { + this.Logger.LogError(e, "Failed to read file {Path}", path); + } + + return null; + } + + /// + /// Filters out nested pom.xml files, keeping only root-level ones. + /// A pom.xml is considered nested if there's another pom.xml in a parent directory. + /// + /// The incoming process requests for pom.xml files. + /// Cancellation token for the operation. + /// Process requests for only root-level pom.xml files. + private IObservable RemoveNestedPomXmls(IObservable componentStreams, CancellationToken cancellationToken) + { + return componentStreams + .ToList() + .SelectMany(allRequests => + { + cancellationToken.ThrowIfCancellationRequested(); + + // Build a list of all directories that contain a pom.xml, ordered by path length (shortest first). + // This ensures parent directories are checked before their children. + var pomDirectories = allRequests + .Select(r => NormalizeDirectoryPath(Path.GetDirectoryName(r.ComponentStream.Location))) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(d => d.Length) + .ToList(); + + var filteredRequests = new List(); + + foreach (var request in allRequests) + { + cancellationToken.ThrowIfCancellationRequested(); + + var location = NormalizeDirectoryPath(Path.GetDirectoryName(request.ComponentStream.Location)); + var isNested = false; + + foreach (var pomDirectory in pomDirectories) + { + if (pomDirectory.Length >= location.Length) { - current.FileNames.Add(Path.GetFileName(item.Location)); + // Since the list is ordered by length, if the pomDirectory is longer than + // or equal to the location, there are no possible parent directories left. + break; } - if (last != null && current.FileNames.Contains(MavenManifest)) + if (location.StartsWith(pomDirectory, StringComparison.OrdinalIgnoreCase)) { - this.LogDebugWithId($"Ignoring {MavenManifest} at {item.Location}, as it has a parent {MavenManifest} that will be processed at {current.Name}\\{MavenManifest} ."); + this.LogDebug($"Ignoring {MavenManifest} at {location}, as it has a parent {MavenManifest} at {pomDirectory}."); + isNested = true; break; } + } - last = current; + if (!isNested) + { + this.LogDebug($"Discovered {request.ComponentStream.Location}."); + filteredRequests.Add(request); } - }, - s.OnCompleted); - }); + } + + return filteredRequests; + }); + + // Normalizes a directory path by ensuring it ends with a directory separator. + // This prevents false matches like "C:\foo" matching "C:\foobar". + static string NormalizeDirectoryPath(string path) => + path.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar; } } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs index ab86692c6..9a9a2b2c3 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs @@ -120,7 +120,6 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); // npm services.AddSingleton(); diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/MavenWithFallbackDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/MavenWithFallbackDetectorTests.cs deleted file mode 100644 index b81b7ec15..000000000 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/MavenWithFallbackDetectorTests.cs +++ /dev/null @@ -1,1098 +0,0 @@ -#nullable disable -namespace Microsoft.ComponentDetection.Detectors.Tests; - -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using AwesomeAssertions; -using Microsoft.ComponentDetection.Contracts; -using Microsoft.ComponentDetection.Contracts.Internal; -using Microsoft.ComponentDetection.Contracts.TypedComponent; -using Microsoft.ComponentDetection.Detectors.Maven; -using Microsoft.ComponentDetection.TestsUtilities; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using Moq; - -[TestClass] -[TestCategory("Governance/All")] -[TestCategory("Governance/ComponentDetection")] -public class MavenWithFallbackDetectorTests : BaseDetectorTest -{ - /// - /// The shared deps filename used by MavenCommandService. - /// Must match BcdeMvnDependencyFileName from MavenCommandService. - /// - private const string BcdeMvnFileName = "bcde.mvndeps"; - - private readonly Mock mavenCommandServiceMock; - private readonly Mock envVarServiceMock; - private readonly Mock fileUtilityServiceMock; - - public MavenWithFallbackDetectorTests() - { - this.mavenCommandServiceMock = new Mock(); - this.mavenCommandServiceMock.Setup(x => x.BcdeMvnDependencyFileName).Returns(BcdeMvnFileName); - - // Default setup for GenerateDependenciesFileAsync - this.mavenCommandServiceMock.Setup(x => x.GenerateDependenciesFileAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(new MavenCliResult(true, null)); - - this.DetectorTestUtility.AddServiceMock(this.mavenCommandServiceMock); - - this.envVarServiceMock = new Mock(); - this.DetectorTestUtility.AddServiceMock(this.envVarServiceMock); - - this.fileUtilityServiceMock = new Mock(); - this.DetectorTestUtility.AddServiceMock(this.fileUtilityServiceMock); - } - - [TestMethod] - public async Task WhenMavenCliNotAvailable_FallsBackToStaticParsing_Async() - { - // Arrange - this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) - .ReturnsAsync(false); - - var pomXmlContent = @" - - 4.0.0 - com.test - my-app - 1.0.0 - - - org.apache.commons - commons-lang3 - 3.12.0 - - -"; - - // Act - var (detectorResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("pom.xml", pomXmlContent) - .ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().ContainSingle(); - - var mavenComponent = detectedComponents.First().Component as MavenComponent; - mavenComponent.Should().NotBeNull(); - mavenComponent.GroupId.Should().Be("org.apache.commons"); - mavenComponent.ArtifactId.Should().Be("commons-lang3"); - mavenComponent.Version.Should().Be("3.12.0"); - } - - [TestMethod] - public async Task WhenMavenCliNotAvailable_DetectsMultipleDependencies_Async() - { - // Arrange - this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) - .ReturnsAsync(false); - - var pomXmlContent = @" - - 4.0.0 - com.test - my-app - 1.0.0 - - - org.apache.commons - commons-lang3 - 3.12.0 - - - com.google.guava - guava - 31.1-jre - - - junit - junit - 4.13.2 - - -"; - - // Act - var (detectorResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("pom.xml", pomXmlContent) - .ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().HaveCount(3); - - var groupIds = detectedComponents - .Select(dc => (dc.Component as MavenComponent)?.GroupId) - .ToList(); - - groupIds.Should().Contain("org.apache.commons"); - groupIds.Should().Contain("com.google.guava"); - groupIds.Should().Contain("junit"); - } - - [TestMethod] - public async Task WhenMavenCliSucceeds_UsesMvnCliResults_Async() - { - // Arrange - const string componentString = "org.apache.maven:maven-compat:jar:3.6.1-SNAPSHOT"; - - this.SetupMvnCliSuccess(componentString); - - this.mavenCommandServiceMock.Setup(x => x.ParseDependenciesFile(It.IsAny())) - .Callback((ProcessRequest pr) => - { - pr.SingleFileComponentRecorder.RegisterUsage( - new DetectedComponent(new MavenComponent("org.apache.maven", "maven-compat", "3.6.1-SNAPSHOT"))); - }); - - // Act - var (detectorResult, componentRecorder) = await this.DetectorTestUtility.ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().ContainSingle(); - - var mavenComponent = detectedComponents.First().Component as MavenComponent; - mavenComponent.Should().NotBeNull(); - mavenComponent.GroupId.Should().Be("org.apache.maven"); - mavenComponent.ArtifactId.Should().Be("maven-compat"); - mavenComponent.Version.Should().Be("3.6.1-SNAPSHOT"); - } - - [TestMethod] - public async Task WhenMavenCliSucceeds_PreservesTransitiveDependencies_Async() - { - // Arrange - const string rootComponent = "com.test:my-app:jar:1.0.0"; - const string directDependency = "org.apache.commons:commons-lang3:jar:3.12.0"; - const string transitiveDependency = "org.apache.commons:commons-text:jar:1.9"; - - var content = $@"{rootComponent} -\- {directDependency} - \- {transitiveDependency}"; - - this.SetupMvnCliSuccess(content); - - this.mavenCommandServiceMock.Setup(x => x.ParseDependenciesFile(It.IsAny())) - .Callback((ProcessRequest pr) => - { - pr.SingleFileComponentRecorder.RegisterUsage( - new DetectedComponent(new MavenComponent("com.test", "my-app", "1.0.0")), - isExplicitReferencedDependency: true); - pr.SingleFileComponentRecorder.RegisterUsage( - new DetectedComponent(new MavenComponent("org.apache.commons", "commons-lang3", "3.12.0")), - isExplicitReferencedDependency: true); - pr.SingleFileComponentRecorder.RegisterUsage( - new DetectedComponent(new MavenComponent("org.apache.commons", "commons-text", "1.9")), - isExplicitReferencedDependency: false, - parentComponentId: "org.apache.commons commons-lang3 3.12.0 - Maven"); - }); - - // Act - var (detectorResult, componentRecorder) = await this.DetectorTestUtility.ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().HaveCount(3); - - // Verify dependency graph has the transitive relationship - var dependencyGraph = componentRecorder.GetDependencyGraphsByLocation().Values.First(); - dependencyGraph.Should().NotBeNull(); - - // Verify the transitive component is reachable from the direct dependency - var directComponentId = "org.apache.commons commons-lang3 3.12.0 - Maven"; - var transitiveComponentId = "org.apache.commons commons-text 1.9 - Maven"; - - var directDependencies = dependencyGraph.GetDependenciesForComponent(directComponentId); - directDependencies.Should().Contain( - transitiveComponentId, - "the transitive dependency should be a child of the direct dependency"); - } - - [TestMethod] - public async Task WhenMavenCliProducesNoOutput_FallsBackToStaticParsing_Async() - { - // Arrange - this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) - .ReturnsAsync(true); - - // MvnCli runs but produces no bcde.mvndeps files (simulating failure) - this.mavenCommandServiceMock.Setup(x => x.GenerateDependenciesFileAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(new MavenCliResult(true, null)); - - var pomXmlContent = @" - - 4.0.0 - com.test - my-app - 1.0.0 - - - org.apache.commons - commons-lang3 - 3.12.0 - - -"; - - // Act - var (detectorResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("pom.xml", pomXmlContent) - .ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().ContainSingle(); - - var mavenComponent = detectedComponents.First().Component as MavenComponent; - mavenComponent.Should().NotBeNull(); - mavenComponent.GroupId.Should().Be("org.apache.commons"); - mavenComponent.ArtifactId.Should().Be("commons-lang3"); - mavenComponent.Version.Should().Be("3.12.0"); - } - - [TestMethod] - public async Task StaticParser_IgnoresDependenciesWithoutVersion_Async() - { - // Arrange - this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) - .ReturnsAsync(false); - - var pomXmlContent = @" - - 4.0.0 - com.test - my-app - 1.0.0 - - - org.apache.commons - commons-lang3 - - - - com.google.guava - guava - 31.1-jre - - -"; - - // Act - var (detectorResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("pom.xml", pomXmlContent) - .ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().ContainSingle(); - - var mavenComponent = detectedComponents.First().Component as MavenComponent; - mavenComponent.ArtifactId.Should().Be("guava"); - } - - [TestMethod] - public async Task StaticParser_IgnoresDependenciesWithVersionRanges_Async() - { - // Arrange - this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) - .ReturnsAsync(false); - - var pomXmlContent = @" - - 4.0.0 - com.test - my-app - 1.0.0 - - - org.apache.commons - commons-lang3 - [3.0,4.0) - - - com.google.guava - guava - 31.1-jre - - -"; - - // Act - var (detectorResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("pom.xml", pomXmlContent) - .ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - // Version ranges with commas are ignored - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().ContainSingle(); - - var mavenComponent = detectedComponents.First().Component as MavenComponent; - mavenComponent.ArtifactId.Should().Be("guava"); - } - - [TestMethod] - public async Task StaticParser_ResolvesPropertyVersions_Async() - { - // Arrange - this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) - .ReturnsAsync(false); - - var pomXmlContent = @" - - 4.0.0 - com.test - my-app - 1.0.0 - - 3.12.0 - - - - org.apache.commons - commons-lang3 - ${commons.version} - - -"; - - // Act - var (detectorResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("pom.xml", pomXmlContent) - .ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().ContainSingle(); - - var mavenComponent = detectedComponents.First().Component as MavenComponent; - mavenComponent.Should().NotBeNull(); - mavenComponent.Version.Should().Be("3.12.0"); - } - - [TestMethod] - public async Task StaticParser_IgnoresDependenciesWithUnresolvablePropertyVersions_Async() - { - // Arrange - this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) - .ReturnsAsync(false); - - var pomXmlContent = @" - - 4.0.0 - com.test - my-app - 1.0.0 - - - org.apache.commons - commons-lang3 - ${undefined.property} - - - com.google.guava - guava - 31.1-jre - - -"; - - // Act - var (detectorResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("pom.xml", pomXmlContent) - .ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - // Unresolvable property versions are ignored - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().ContainSingle(); - - var mavenComponent = detectedComponents.First().Component as MavenComponent; - mavenComponent.ArtifactId.Should().Be("guava"); - } - - [TestMethod] - public async Task WhenNoPomXmlFiles_ReturnsSuccessWithNoComponents_Async() - { - // Arrange - this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) - .ReturnsAsync(true); - - // Act - var (detectorResult, componentRecorder) = await this.DetectorTestUtility - .ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - componentRecorder.GetDetectedComponents().Should().BeEmpty(); - } - - [TestMethod] - public async Task WhenPomXmlHasNoDependencies_ReturnsSuccessWithNoComponents_Async() - { - // Arrange - this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) - .ReturnsAsync(false); - - var pomXmlContent = @" - - 4.0.0 - com.test - my-app - 1.0.0 -"; - - // Act - var (detectorResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("pom.xml", pomXmlContent) - .ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - componentRecorder.GetDetectedComponents().Should().BeEmpty(); - } - - [TestMethod] - public async Task WhenDisableMvnCliTrue_UsesStaticParsing_Async() - { - // Arrange - DisableMvnCliEnvVar is true (explicitly disable Maven CLI) - this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) - .ReturnsAsync(true); - - // Disable MvnCli explicitly - this.envVarServiceMock.Setup(x => x.IsEnvironmentVariableValueTrue(MavenWithFallbackDetector.DisableMvnCliEnvVar)) - .Returns(true); - - var pomXmlContent = @" - - 4.0.0 - com.test - my-app - 1.0.0 - - - org.apache.commons - commons-lang3 - 3.12.0 - - -"; - - // Act - var (detectorResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("pom.xml", pomXmlContent) - .ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - // Should detect component via static parsing even though Maven CLI is available - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().ContainSingle(); - - var mavenComponent = detectedComponents.First().Component as MavenComponent; - mavenComponent.Should().NotBeNull(); - mavenComponent.GroupId.Should().Be("org.apache.commons"); - mavenComponent.ArtifactId.Should().Be("commons-lang3"); - mavenComponent.Version.Should().Be("3.12.0"); - - // Verify MavenCLIExistsAsync was never called since we disabled MvnCli - this.mavenCommandServiceMock.Verify(x => x.MavenCLIExistsAsync(), Times.Never); - } - - [TestMethod] - public async Task WhenDisableMvnCliEnvVarIsFalse_UsesMvnCliNormally_Async() - { - // Arrange - Maven CLI is available and CD_MAVEN_DISABLE_CLI is false - const string componentString = "org.apache.maven:maven-compat:jar:3.6.1-SNAPSHOT"; - const string validPomXml = @" - - 4.0.0 - com.test - test-app - 1.0.0 -"; - - this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) - .ReturnsAsync(true); - - // Setup GenerateDependenciesFileAsync to return success - this.mavenCommandServiceMock.Setup(x => x.GenerateDependenciesFileAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(new MavenCliResult(true, null)); - - // Setup file utility to return the deps file content - this.fileUtilityServiceMock.Setup(x => x.Exists(It.Is(s => s.EndsWith(BcdeMvnFileName)))) - .Returns(true); - this.fileUtilityServiceMock.Setup(x => x.ReadAllText(It.Is(s => s.EndsWith(BcdeMvnFileName)))) - .Returns(componentString); - - this.mavenCommandServiceMock.Setup(x => x.ParseDependenciesFile(It.IsAny())) - .Callback((ProcessRequest pr) => - { - pr.SingleFileComponentRecorder.RegisterUsage( - new DetectedComponent(new MavenComponent("org.apache.maven", "maven-compat", "3.6.1-SNAPSHOT"))); - }); - - // Set up the environment variable to NOT disable MvnCli (false) - this.envVarServiceMock.Setup(x => x.IsEnvironmentVariableValueTrue(MavenWithFallbackDetector.DisableMvnCliEnvVar)) - .Returns(false); - - // Act - var (detectorResult, _) = await this.DetectorTestUtility - .WithFile("pom.xml", validPomXml) - .ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - // Should use MvnCli since CD_MAVEN_DISABLE_CLI is false - this.mavenCommandServiceMock.Verify(x => x.MavenCLIExistsAsync(), Times.Once); - - // Verify telemetry shows MvnCliOnly detection method - detectorResult.AdditionalTelemetryDetails.Should().ContainKey("DetectionMethod"); - detectorResult.AdditionalTelemetryDetails["DetectionMethod"].Should().Be("MvnCliOnly"); - } - - [TestMethod] - public async Task WhenDisableMvnCliEnvVarNotSet_UsesMvnCliNormally_Async() - { - // Arrange - Maven CLI is available and CD_MAVEN_DISABLE_CLI is NOT set (doesn't exist) - const string componentString = "org.apache.maven:maven-compat:jar:3.6.1-SNAPSHOT"; - - this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) - .ReturnsAsync(true); - - // Setup GenerateDependenciesFileAsync to return success - this.mavenCommandServiceMock.Setup(x => x.GenerateDependenciesFileAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(new MavenCliResult(true, null)); - - // Setup file utility to return the deps file content - this.fileUtilityServiceMock.Setup(x => x.Exists(It.Is(s => s.EndsWith(BcdeMvnFileName)))) - .Returns(true); - this.fileUtilityServiceMock.Setup(x => x.ReadAllText(It.Is(s => s.EndsWith(BcdeMvnFileName)))) - .Returns(componentString); - - this.mavenCommandServiceMock.Setup(x => x.ParseDependenciesFile(It.IsAny())) - .Callback((ProcessRequest pr) => - { - pr.SingleFileComponentRecorder.RegisterUsage( - new DetectedComponent(new MavenComponent("org.apache.maven", "maven-compat", "3.6.1-SNAPSHOT"))); - }); - - // Explicitly set up the environment variable to NOT exist (returns false) - this.envVarServiceMock.Setup(x => x.IsEnvironmentVariableValueTrue(MavenWithFallbackDetector.DisableMvnCliEnvVar)) - .Returns(false); - - // Act - var (detectorResult, _) = await this.DetectorTestUtility - .WithFile("pom.xml", componentString) - .ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - // Should use MvnCli since CD_MAVEN_DISABLE_CLI doesn't exist - this.mavenCommandServiceMock.Verify(x => x.MavenCLIExistsAsync(), Times.Once); - - // Verify telemetry shows MvnCliOnly detection method - detectorResult.AdditionalTelemetryDetails.Should().ContainKey("DetectionMethod"); - detectorResult.AdditionalTelemetryDetails["DetectionMethod"].Should().Be("MvnCliOnly"); - } - - [TestMethod] - public async Task WhenDisableMvnCliEnvVarSetToInvalidValue_UsesMvnCliNormally_Async() - { - // Arrange - Maven CLI is available and CD_MAVEN_DISABLE_CLI is set to an invalid (non-boolean) value - const string componentString = "org.apache.maven:maven-compat:jar:3.6.1-SNAPSHOT"; - const string validPomXml = @" - - 4.0.0 - com.test - test-app - 1.0.0 -"; - - this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) - .ReturnsAsync(true); - - // Setup GenerateDependenciesFileAsync to return success - this.mavenCommandServiceMock.Setup(x => x.GenerateDependenciesFileAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(new MavenCliResult(true, null)); - - // Setup file utility to return the deps file content - this.fileUtilityServiceMock.Setup(x => x.Exists(It.Is(s => s.EndsWith(BcdeMvnFileName)))) - .Returns(true); - this.fileUtilityServiceMock.Setup(x => x.ReadAllText(It.Is(s => s.EndsWith(BcdeMvnFileName)))) - .Returns(componentString); - - this.mavenCommandServiceMock.Setup(x => x.ParseDependenciesFile(It.IsAny())) - .Callback((ProcessRequest pr) => - { - pr.SingleFileComponentRecorder.RegisterUsage( - new DetectedComponent(new MavenComponent("org.apache.maven", "maven-compat", "3.6.1-SNAPSHOT"))); - }); - - // Set up the environment variable with an invalid value (IsEnvironmentVariableValueTrue returns false for non-"true" values) - this.envVarServiceMock.Setup(x => x.IsEnvironmentVariableValueTrue(MavenWithFallbackDetector.DisableMvnCliEnvVar)) - .Returns(false); - - // Act - var (detectorResult, _) = await this.DetectorTestUtility - .WithFile("pom.xml", validPomXml) - .ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - // Should use MvnCli since the env var value is invalid (bool.TryParse fails) - this.mavenCommandServiceMock.Verify(x => x.MavenCLIExistsAsync(), Times.Once); - - // Verify telemetry shows MvnCliOnly detection method - detectorResult.AdditionalTelemetryDetails.Should().ContainKey("DetectionMethod"); - detectorResult.AdditionalTelemetryDetails["DetectionMethod"].Should().Be("MvnCliOnly"); - } - - [TestMethod] - public async Task WhenMvnCliSucceeds_NestedPomXmlsAreFilteredOut_Async() - { - // Arrange - Maven CLI is available and succeeds. - // In a multi-module project, only the root pom.xml should be processed by MvnCli. - this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) - .ReturnsAsync(true); - - // Setup GenerateDependenciesFileAsync to return success - this.mavenCommandServiceMock.Setup(x => x.GenerateDependenciesFileAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(new MavenCliResult(true, null)); - - // Setup file utility to return the deps file content - this.fileUtilityServiceMock.Setup(x => x.Exists(It.Is(s => s.EndsWith(BcdeMvnFileName)))) - .Returns(true); - this.fileUtilityServiceMock.Setup(x => x.ReadAllText(It.Is(s => s.EndsWith(BcdeMvnFileName)))) - .Returns("com.test:parent-app:jar:1.0.0"); - - this.mavenCommandServiceMock.Setup(x => x.ParseDependenciesFile(It.IsAny())) - .Callback((ProcessRequest pr) => - { - // MvnCli processes root and generates deps for all modules - pr.SingleFileComponentRecorder.RegisterUsage( - new DetectedComponent(new MavenComponent("com.test", "parent-app", "1.0.0"))); - pr.SingleFileComponentRecorder.RegisterUsage( - new DetectedComponent(new MavenComponent("com.test", "module-a", "1.0.0"))); - pr.SingleFileComponentRecorder.RegisterUsage( - new DetectedComponent(new MavenComponent("com.test", "module-b", "1.0.0"))); - }); - - // Root pom.xml content (doesn't matter for this test, just needs to exist) - var rootPomContent = @" - - com.test - parent-app - 1.0.0 - pom -"; - - // Nested module pom.xml content - var moduleAPomContent = @" - - - com.test - parent-app - 1.0.0 - - module-a - - - org.apache.commons - commons-lang3 - 3.12.0 - - -"; - - // Act - Add root pom.xml first, then nested module pom.xml - // The root should get MvnCli bcde.mvndeps, nested should be filtered - var (detectorResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("pom.xml", rootPomContent) - .WithFile("module-a/pom.xml", moduleAPomContent) - .ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - // Should have components from MvnCli (parent + modules), not from static parsing - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().HaveCount(3); - - // MvnCli should only be called once for root pom.xml (nested filtered out) - this.mavenCommandServiceMock.Verify( - x => x.GenerateDependenciesFileAsync(It.IsAny(), It.IsAny()), - Times.Once); - } - - [TestMethod] - public async Task WhenMvnCliFailsCompletely_AllNestedPomXmlsAreRestoredForStaticParsing_Async() - { - // Arrange - Maven CLI is available but fails for all pom.xml files (e.g., auth error) - this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) - .ReturnsAsync(true); - - // MvnCli runs but produces no bcde.mvndeps files (simulating complete failure) - this.mavenCommandServiceMock.Setup(x => x.GenerateDependenciesFileAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(new MavenCliResult(true, null)); - - // Root pom.xml content - var rootPomContent = @" - - com.test - parent-app - 1.0.0 - pom - - - org.springframework - spring-core - 5.3.0 - - -"; - - // Nested module pom.xml content - should be restored for static parsing - var moduleAPomContent = @" - - - com.test - parent-app - 1.0.0 - - module-a - - - org.apache.commons - commons-lang3 - 3.12.0 - - -"; - - // Another nested module - var moduleBPomContent = @" - - - com.test - parent-app - 1.0.0 - - module-b - - - com.google.guava - guava - 31.1-jre - - -"; - - // Act - var (detectorResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("pom.xml", rootPomContent) - .WithFile("module-a/pom.xml", moduleAPomContent) - .WithFile("module-b/pom.xml", moduleBPomContent) - .ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - // All pom.xml files should be processed via static parsing (nested poms restored) - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().HaveCount(3); // spring-core, commons-lang3, guava - - var artifactIds = detectedComponents - .Select(dc => (dc.Component as MavenComponent)?.ArtifactId) - .ToList(); - - artifactIds.Should().Contain("spring-core"); // From root pom.xml - artifactIds.Should().Contain("commons-lang3"); // From module-a/pom.xml (nested - restored) - artifactIds.Should().Contain("guava"); // From module-b/pom.xml (nested - restored) - } - - [TestMethod] - public async Task WhenMvnCliPartiallyFails_NestedPomXmlsRestoredOnlyForFailedDirectories_Async() - { - // Arrange - Maven CLI succeeds for projectA but fails for projectB. - this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) - .ReturnsAsync(true); - - // MvnCli runs: projectA succeeds, projectB fails - this.mavenCommandServiceMock.Setup(x => x.GenerateDependenciesFileAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(new MavenCliResult(true, null)); - - // Setup file utility: projectA has deps file, projectB does not - this.fileUtilityServiceMock.Setup(x => x.Exists(It.Is(s => s.EndsWith(BcdeMvnFileName)))) - .Returns((string path) => path.Contains("projectA")); - this.fileUtilityServiceMock.Setup(x => x.ReadAllText(It.Is(s => s.EndsWith(BcdeMvnFileName) && s.Contains("projectA")))) - .Returns("com.projecta:app-a:jar:1.0.0"); - - this.mavenCommandServiceMock.Setup(x => x.ParseDependenciesFile(It.IsAny())) - .Callback((ProcessRequest pr) => - { - // Only register components for projectA's bcde.mvndeps - if (pr.ComponentStream.Location.Contains("projectA")) - { - pr.SingleFileComponentRecorder.RegisterUsage( - new DetectedComponent(new MavenComponent("com.projecta", "app-a", "1.0.0"))); - pr.SingleFileComponentRecorder.RegisterUsage( - new DetectedComponent(new MavenComponent("com.projecta", "module-a1", "1.0.0"))); - } - }); - - // ProjectA - MvnCli will succeed - var projectAPomContent = @" - - com.projecta - app-a - 1.0.0 -"; - - var projectAModulePomContent = @" - - - com.projecta - app-a - 1.0.0 - - module-a1 - - - org.projecta - dep-from-nested-a - 1.0.0 - - -"; - - // ProjectB - MvnCli will fail (no bcde.mvndeps generated) - var projectBPomContent = @" - - com.projectb - app-b - 2.0.0 - - - org.projectb - dep-from-root-b - 2.0.0 - - -"; - - var projectBModulePomContent = @" - - - com.projectb - app-b - 2.0.0 - - module-b1 - - - org.projectb - dep-from-nested-b - 2.0.0 - - -"; - - // Act - projectA gets bcde.mvndeps (via mock), projectB does not - var (detectorResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("projectA/pom.xml", projectAPomContent) - .WithFile("projectA/module-a1/pom.xml", projectAModulePomContent) - .WithFile("projectB/pom.xml", projectBPomContent) - .WithFile("projectB/module-b1/pom.xml", projectBModulePomContent) - .ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - var detectedComponents = componentRecorder.GetDetectedComponents(); - - // ProjectA: 2 components from MvnCli (app-a, module-a1) - // ProjectB: 2 components from static parsing (dep-from-root-b, dep-from-nested-b) - // Note: nested pom in projectA should NOT be statically parsed (MvnCli handled it) - // Note: nested pom in projectB SHOULD be statically parsed (MvnCli failed) - detectedComponents.Should().HaveCount(4); - - var artifactIds = detectedComponents - .Select(dc => (dc.Component as MavenComponent)?.ArtifactId) - .ToList(); - - // From MvnCli for projectA - artifactIds.Should().Contain("app-a"); - artifactIds.Should().Contain("module-a1"); - - // From static parsing for projectB (both root and nested restored) - artifactIds.Should().Contain("dep-from-root-b"); - artifactIds.Should().Contain("dep-from-nested-b"); - - // Should NOT contain dep-from-nested-a (that nested pom was handled by MvnCli, not static) - artifactIds.Should().NotContain("dep-from-nested-a"); - } - - [TestMethod] - public async Task WhenMvnCliFailsWithAuthError_LogsFailedEndpointAndSetsTelemetry_Async() - { - // Arrange - this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) - .ReturnsAsync(true); - - // Simulate Maven CLI failure with authentication error message containing endpoint URL - var authErrorMessage = "[ERROR] Failed to execute goal on project my-app: Could not resolve dependencies for project com.test:my-app:jar:1.0.0: " + - "Failed to collect dependencies at com.private:private-lib:jar:2.0.0: " + - "Failed to read artifact descriptor for com.private:private-lib:jar:2.0.0: " + - "Could not transfer artifact com.private:private-lib:pom:2.0.0 from/to private-repo (https://private-maven-repo.example.com/repository/maven-releases/): " + - "status code: 401, reason phrase: Unauthorized"; - - this.mavenCommandServiceMock.Setup(x => x.GenerateDependenciesFileAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(new MavenCliResult(false, authErrorMessage)); - - var pomXmlContent = @" - - 4.0.0 - com.test - my-app - 1.0.0 - - - org.apache.commons - commons-lang3 - 3.12.0 - - -"; - - // Act - var (detectorResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("pom.xml", pomXmlContent) - .ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - // Should fall back to static parsing and detect the component - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().ContainSingle(); - - var mavenComponent = detectedComponents.First().Component as MavenComponent; - mavenComponent.Should().NotBeNull(); - mavenComponent.ArtifactId.Should().Be("commons-lang3"); - - // Verify telemetry contains auth failure info - detectorResult.AdditionalTelemetryDetails.Should().ContainKey("FallbackReason"); - detectorResult.AdditionalTelemetryDetails["FallbackReason"].Should().Be("AuthenticationFailure"); - - // Verify telemetry contains the failed endpoint - detectorResult.AdditionalTelemetryDetails.Should().ContainKey("FailedEndpoints"); - detectorResult.AdditionalTelemetryDetails["FailedEndpoints"].Should().Contain("https://private-maven-repo.example.com"); - } - - [TestMethod] - public async Task WhenMvnCliFailsWithNonAuthError_SetsFallbackReasonToOther_Async() - { - // Arrange - this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) - .ReturnsAsync(true); - - // Simulate Maven CLI failure with a non-auth error (e.g., build error) - var nonAuthErrorMessage = "[ERROR] Failed to execute goal on project my-app: Compilation failure: " + - "src/main/java/com/test/App.java:[10,5] cannot find symbol"; - - this.mavenCommandServiceMock.Setup(x => x.GenerateDependenciesFileAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(new MavenCliResult(false, nonAuthErrorMessage)); - - var pomXmlContent = @" - - 4.0.0 - com.test - my-app - 1.0.0 - - - org.apache.commons - commons-lang3 - 3.12.0 - - -"; - - // Act - var (detectorResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("pom.xml", pomXmlContent) - .ExecuteDetectorAsync(); - - // Assert - detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - // Should fall back to static parsing - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().ContainSingle(); - - // Verify telemetry shows non-auth failure - detectorResult.AdditionalTelemetryDetails.Should().ContainKey("FallbackReason"); - detectorResult.AdditionalTelemetryDetails["FallbackReason"].Should().Be("OtherMvnCliFailure"); - - // Should NOT have FailedEndpoints since this wasn't an auth error - detectorResult.AdditionalTelemetryDetails.Should().NotContainKey("FailedEndpoints"); - } - - private void SetupMvnCliSuccess(string depsFileContent) - { - this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) - .ReturnsAsync(true); - - // Setup for 3-parameter version (used by MavenWithFallbackDetector) - this.mavenCommandServiceMock.Setup(x => x.GenerateDependenciesFileAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(new MavenCliResult(true, null)); - - // Setup file utility service to return the deps file content - // The detector reads the file from disk after CLI succeeds - this.fileUtilityServiceMock.Setup(x => x.Exists(It.Is(s => s.EndsWith(BcdeMvnFileName)))) - .Returns(true); - this.fileUtilityServiceMock.Setup(x => x.ReadAllText(It.Is(s => s.EndsWith(BcdeMvnFileName)))) - .Returns(depsFileContent); - - // Use a valid minimal pom.xml - the actual content doesn't matter for MvnCli success path - // since components come from the mocked ParseDependenciesFile, but using valid XML - // makes the test more robust against future changes that might read/validate the pom. - const string validPomXml = @" - - 4.0.0 - com.test - test-app - 1.0.0 -"; - - this.DetectorTestUtility.WithFile("pom.xml", validPomXml); - } -} diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/MvnCliDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/MvnCliDetectorTests.cs index bb25c5cf3..bdac5a939 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/MvnCliDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/MvnCliDetectorTests.cs @@ -4,6 +4,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; using AwesomeAssertions; using Microsoft.ComponentDetection.Contracts; @@ -21,11 +22,18 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; public class MvnCliDetectorTests : BaseDetectorTest { private readonly Mock mavenCommandServiceMock; + private readonly Mock environmentVariableServiceMock; + private readonly Mock fileUtilityServiceMock; public MvnCliDetectorTests() { this.mavenCommandServiceMock = new Mock(); + this.environmentVariableServiceMock = new Mock(); + this.fileUtilityServiceMock = new Mock(); + this.DetectorTestUtility.AddServiceMock(this.mavenCommandServiceMock); + this.DetectorTestUtility.AddServiceMock(this.environmentVariableServiceMock); + this.DetectorTestUtility.AddServiceMock(this.fileUtilityServiceMock); } [TestMethod] @@ -180,6 +188,556 @@ public async Task MavenDependencyGraphAsync() dependencyGraph.IsComponentExplicitlyReferenced(leafComponentId).Should().BeFalse(); } + [TestMethod] + public async Task WhenMavenCliNotAvailable_FallsBackToStaticParsing_Async() + { + // Arrange + this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) + .ReturnsAsync(false); + + var pomXmlContent = @" + + 4.0.0 + com.test + my-app + 1.0.0 + + + org.apache.commons + commons-lang3 + 3.12.0 + + +"; + + // Act + var (detectorResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pom.xml", pomXmlContent) + .ExecuteDetectorAsync(); + + // Assert + detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().ContainSingle(); + + var mavenComponent = detectedComponents.First().Component as MavenComponent; + mavenComponent.Should().NotBeNull(); + mavenComponent.GroupId.Should().Be("org.apache.commons"); + mavenComponent.ArtifactId.Should().Be("commons-lang3"); + mavenComponent.Version.Should().Be("3.12.0"); + } + + [TestMethod] + public async Task WhenMavenCliNotAvailable_DetectsMultipleDependencies_Async() + { + // Arrange + this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) + .ReturnsAsync(false); + + var pomXmlContent = @" + + 4.0.0 + com.test + my-app + 1.0.0 + + + org.apache.commons + commons-lang3 + 3.12.0 + + + com.google.guava + guava + 31.1-jre + + + junit + junit + 4.13.2 + + +"; + + // Act + var (detectorResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pom.xml", pomXmlContent) + .ExecuteDetectorAsync(); + + // Assert + detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().HaveCount(3); + + var groupIds = detectedComponents + .Select(dc => (dc.Component as MavenComponent)?.GroupId) + .ToList(); + + groupIds.Should().Contain("org.apache.commons"); + groupIds.Should().Contain("com.google.guava"); + groupIds.Should().Contain("junit"); + } + + [TestMethod] + public async Task WhenMavenCliProducesNoOutput_FallsBackToStaticParsing_Async() + { + // Arrange + this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) + .ReturnsAsync(true); + this.mavenCommandServiceMock.Setup(x => x.BcdeMvnDependencyFileName) + .Returns("bcde.mvndeps"); + this.mavenCommandServiceMock.Setup(x => x.GenerateDependenciesFileAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new MavenCliResult(true, null)); + + // File exists but is empty + this.fileUtilityServiceMock.Setup(x => x.Exists(It.IsAny())).Returns(true); + this.fileUtilityServiceMock.Setup(x => x.ReadAllText(It.IsAny())).Returns(string.Empty); + + var pomXmlContent = @" + + 4.0.0 + com.test + my-app + 1.0.0 + + + org.apache.commons + commons-lang3 + 3.12.0 + + +"; + + // Act + var (detectorResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pom.xml", pomXmlContent) + .ExecuteDetectorAsync(); + + // Assert + detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().ContainSingle(); + + var mavenComponent = detectedComponents.First().Component as MavenComponent; + mavenComponent.Should().NotBeNull(); + mavenComponent.GroupId.Should().Be("org.apache.commons"); + mavenComponent.ArtifactId.Should().Be("commons-lang3"); + mavenComponent.Version.Should().Be("3.12.0"); + } + + [TestMethod] + public async Task StaticParser_IgnoresDependenciesWithoutVersion_Async() + { + // Arrange + this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) + .ReturnsAsync(false); + + var pomXmlContent = @" + + 4.0.0 + com.test + my-app + 1.0.0 + + + org.apache.commons + commons-lang3 + + +"; + + // Act + var (detectorResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pom.xml", pomXmlContent) + .ExecuteDetectorAsync(); + + // Assert + detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().BeEmpty(); + } + + [TestMethod] + public async Task StaticParser_IgnoresDependenciesWithVersionRanges_Async() + { + // Arrange + this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) + .ReturnsAsync(false); + + var pomXmlContent = @" + + 4.0.0 + com.test + my-app + 1.0.0 + + + org.apache.commons + commons-lang3 + [3.0,4.0) + + +"; + + // Act + var (detectorResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pom.xml", pomXmlContent) + .ExecuteDetectorAsync(); + + // Assert + detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().BeEmpty(); + } + + [TestMethod] + public async Task StaticParser_ResolvesPropertyVersions_Async() + { + // Arrange + this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) + .ReturnsAsync(false); + + var pomXmlContent = @" + + 4.0.0 + com.test + my-app + 1.0.0 + + 3.12.0 + + + + org.apache.commons + commons-lang3 + ${commons.version} + + +"; + + // Act + var (detectorResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pom.xml", pomXmlContent) + .ExecuteDetectorAsync(); + + // Assert + detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().ContainSingle(); + + var mavenComponent = detectedComponents.First().Component as MavenComponent; + mavenComponent.Should().NotBeNull(); + mavenComponent.GroupId.Should().Be("org.apache.commons"); + mavenComponent.ArtifactId.Should().Be("commons-lang3"); + mavenComponent.Version.Should().Be("3.12.0"); + } + + [TestMethod] + public async Task StaticParser_IgnoresDependenciesWithUnresolvablePropertyVersions_Async() + { + // Arrange + this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) + .ReturnsAsync(false); + + var pomXmlContent = @" + + 4.0.0 + com.test + my-app + 1.0.0 + + + org.apache.commons + commons-lang3 + ${undefined.property} + + +"; + + // Act + var (detectorResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pom.xml", pomXmlContent) + .ExecuteDetectorAsync(); + + // Assert + detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().BeEmpty(); + } + + [TestMethod] + public async Task WhenNoPomXmlFiles_ReturnsSuccessWithNoComponents_Async() + { + // Act + var (detectorResult, componentRecorder) = await this.DetectorTestUtility + .ExecuteDetectorAsync(); + + // Assert + detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().BeEmpty(); + } + + [TestMethod] + public async Task WhenPomXmlHasNoDependencies_ReturnsSuccessWithNoComponents_Async() + { + // Arrange + this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) + .ReturnsAsync(false); + + var pomXmlContent = @" + + 4.0.0 + com.test + my-app + 1.0.0 +"; + + // Act + var (detectorResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pom.xml", pomXmlContent) + .ExecuteDetectorAsync(); + + // Assert + detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().BeEmpty(); + } + + [TestMethod] + public async Task WhenDisableMvnCliTrue_UsesStaticParsing_Async() + { + // Arrange + this.environmentVariableServiceMock.Setup(x => x.IsEnvironmentVariableValueTrue("CD_MAVEN_DISABLE_CLI")) + .Returns(true); + + var pomXmlContent = @" + + 4.0.0 + com.test + my-app + 1.0.0 + + + org.apache.commons + commons-lang3 + 3.12.0 + + +"; + + // Act + var (detectorResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pom.xml", pomXmlContent) + .ExecuteDetectorAsync(); + + // Assert + detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().ContainSingle(); + + var mavenComponent = detectedComponents.First().Component as MavenComponent; + mavenComponent.Should().NotBeNull(); + mavenComponent.GroupId.Should().Be("org.apache.commons"); + mavenComponent.ArtifactId.Should().Be("commons-lang3"); + mavenComponent.Version.Should().Be("3.12.0"); + + // Maven CLI should not have been called + this.mavenCommandServiceMock.Verify(x => x.MavenCLIExistsAsync(), Times.Never); + } + + [TestMethod] + public async Task WhenMvnCliFailsWithAuthError_FallsBackToStaticParsingAndSetsTelemetry_Async() + { + // Arrange + this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) + .ReturnsAsync(true); + this.mavenCommandServiceMock.Setup(x => x.BcdeMvnDependencyFileName) + .Returns("bcde.mvndeps"); + + // Simulate Maven CLI failure with authentication error message containing endpoint URL + var authErrorMessage = "[ERROR] Failed to execute goal on project my-app: Could not resolve dependencies for project com.test:my-app:jar:1.0.0: " + + "Failed to collect dependencies at com.private:private-lib:jar:2.0.0: " + + "Failed to read artifact descriptor for com.private:private-lib:jar:2.0.0: " + + "Could not transfer artifact com.private:private-lib:pom:2.0.0 from/to private-repo (https://private-maven-repo.example.com/repository/maven-releases/): " + + "status code: 401, reason phrase: Unauthorized"; + + this.mavenCommandServiceMock.Setup(x => x.GenerateDependenciesFileAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new MavenCliResult(false, authErrorMessage)); + + var pomXmlContent = @" + + 4.0.0 + com.test + my-app + 1.0.0 + + + org.apache.commons + commons-lang3 + 3.12.0 + + +"; + + // Act + var (detectorResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pom.xml", pomXmlContent) + .ExecuteDetectorAsync(); + + // Assert + detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + // Should fall back to static parsing and detect the component + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().ContainSingle(); + + var mavenComponent = detectedComponents.First().Component as MavenComponent; + mavenComponent.Should().NotBeNull(); + mavenComponent.ArtifactId.Should().Be("commons-lang3"); + + // Verify telemetry contains auth failure info + detectorResult.AdditionalTelemetryDetails.Should().ContainKey("FallbackReason"); + detectorResult.AdditionalTelemetryDetails["FallbackReason"].Should().Be("AuthenticationFailure"); + + // Verify telemetry contains the failed endpoint + detectorResult.AdditionalTelemetryDetails.Should().ContainKey("FailedEndpoints"); + detectorResult.AdditionalTelemetryDetails["FailedEndpoints"].Should().Contain("https://private-maven-repo.example.com"); + } + + [TestMethod] + public async Task WhenMvnCliFailsWithNonAuthError_SetsFallbackReasonToOther_Async() + { + // Arrange + this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) + .ReturnsAsync(true); + this.mavenCommandServiceMock.Setup(x => x.BcdeMvnDependencyFileName) + .Returns("bcde.mvndeps"); + + // Simulate Maven CLI failure with non-auth error + var nonAuthErrorMessage = "[ERROR] Failed to execute goal on project my-app: Compilation failure"; + + this.mavenCommandServiceMock.Setup(x => x.GenerateDependenciesFileAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new MavenCliResult(false, nonAuthErrorMessage)); + + var pomXmlContent = @" + + 4.0.0 + com.test + my-app + 1.0.0 + + + org.apache.commons + commons-lang3 + 3.12.0 + + +"; + + // Act + var (detectorResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pom.xml", pomXmlContent) + .ExecuteDetectorAsync(); + + // Assert + detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + // Should fall back to static parsing and detect the component + var detectedComponents = componentRecorder.GetDetectedComponents(); + detectedComponents.Should().ContainSingle(); + + // Verify telemetry shows fallback reason as other + detectorResult.AdditionalTelemetryDetails.Should().ContainKey("FallbackReason"); + detectorResult.AdditionalTelemetryDetails["FallbackReason"].Should().Be("OtherMvnCliFailure"); + } + + [TestMethod] + public async Task WhenMvnCliSucceeds_NestedPomXmlsAreFilteredOut_Async() + { + // Arrange - Maven CLI is available and succeeds for root pom.xml + this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) + .ReturnsAsync(true); + this.mavenCommandServiceMock.Setup(x => x.BcdeMvnDependencyFileName) + .Returns("bcde.mvndeps"); + + // Setup GenerateDependenciesFileAsync to return success only for root + this.mavenCommandServiceMock.Setup(x => x.GenerateDependenciesFileAsync( + It.Is(pr => pr.ComponentStream.Location.EndsWith("pom.xml")), + It.IsAny())) + .ReturnsAsync(new MavenCliResult(true, null)); + + // Setup file utility to return the deps file content for root + this.fileUtilityServiceMock.Setup(x => x.Exists(It.Is(s => s.EndsWith("bcde.mvndeps") && !s.Contains("module")))) + .Returns(true); + this.fileUtilityServiceMock.Setup(x => x.ReadAllText(It.Is(s => s.EndsWith("bcde.mvndeps") && !s.Contains("module")))) + .Returns("com.test:parent-app:jar:1.0.0"); + + // Mock ParseDependenciesFile for the root deps file + this.mavenCommandServiceMock.Setup(x => x.ParseDependenciesFile( + It.Is(pr => pr.ComponentStream.Location.EndsWith("bcde.mvndeps")))) + .Callback((ProcessRequest pr) => + { + pr.SingleFileComponentRecorder.RegisterUsage( + new DetectedComponent(new MavenComponent("com.test", "parent-app", "1.0.0"))); + }); + + // Root pom.xml content + var rootPomContent = @" + + com.test + parent-app + 1.0.0 + pom +"; + + // Nested module pom.xml content that should be ignored since MvnCli succeeded for root + var moduleAPomContent = @" + + + com.test + parent-app + 1.0.0 + + module-a + + + org.apache.commons + commons-lang3 + 3.12.0 + + +"; + + // Act + var (detectorResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pom.xml", rootPomContent) + .WithFile("module-a/pom.xml", moduleAPomContent) + .ExecuteDetectorAsync(); + + // Assert + detectorResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + + // Should only contain the component from MvnCli (root), not from static parsing of nested pom + detectedComponents.Should().ContainSingle(); + + var mavenComponent = detectedComponents.First().Component as MavenComponent; + mavenComponent.Should().NotBeNull(); + mavenComponent.ArtifactId.Should().Be("parent-app"); + + // Verify MvnCli was called only once (for root pom.xml) + this.mavenCommandServiceMock.Verify( + x => x.GenerateDependenciesFileAsync(It.IsAny(), It.IsAny()), + Times.Once); + } + protected bool ShouldBeEquivalentTo(IEnumerable result, IEnumerable expected) { result.Should().BeEquivalentTo(expected); @@ -194,6 +752,10 @@ private void MvnCliHappyPath(string content) .Returns(bcdeMvnFileName); this.mavenCommandServiceMock.Setup(x => x.MavenCLIExistsAsync()) .ReturnsAsync(true); + this.mavenCommandServiceMock.Setup(x => x.GenerateDependenciesFileAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new MavenCliResult(true, null)); + this.fileUtilityServiceMock.Setup(x => x.Exists(It.IsAny())).Returns(true); + this.fileUtilityServiceMock.Setup(x => x.ReadAllText(It.IsAny())).Returns(content); this.DetectorTestUtility.WithFile("pom.xml", content) .WithFile("pom.xml", content, searchPatterns: [bcdeMvnFileName]); } diff --git a/test/Microsoft.ComponentDetection.VerificationTests/ComponentDetectionIntegrationTests.cs b/test/Microsoft.ComponentDetection.VerificationTests/ComponentDetectionIntegrationTests.cs index a3e3b1fe3..2115051fd 100644 --- a/test/Microsoft.ComponentDetection.VerificationTests/ComponentDetectionIntegrationTests.cs +++ b/test/Microsoft.ComponentDetection.VerificationTests/ComponentDetectionIntegrationTests.cs @@ -151,7 +151,7 @@ private void CompareGraphs(DependencyGraphCollection leftGraphs, DependencyGraph rightDependencies.Should().Contain(leftDependency, $"Component dependency {leftDependency} for component {leftComponent} was not in the {rightGraphName} dependency graph."); } - leftDependencies.Should().BeEquivalentTo(rightDependencies, $"{rightGraphName} has the following components that were not found in {leftGraphName}, please verify this is expected behavior. {System.Text.Json.JsonSerializer.Serialize(rightDependencies.Except(leftDependencies))}"); + leftDependencies.Should().BeEquivalentTo(rightDependencies, $"{rightGraphName} has the following components that were not found in {leftGraphName}, please verify this is expected behavior. {JsonSerializer.Serialize(rightDependencies.Except(leftDependencies))}"); } } } @@ -172,6 +172,9 @@ private string GetKey(ScannedComponent component) [TestMethod] public void CheckDetectorsRunTimesAndCounts() { + // List of detectors that were intentionally removed + var expectedRemovedDetectors = new HashSet { "MavenWithFallback" }; + // makes sure that all detectors have the same number of components found. // if some are lost, error. // if some are new, check if version of detector is updated. if it isn't error @@ -183,7 +186,9 @@ public void CheckDetectorsRunTimesAndCounts() var oldMatches = Regex.Matches(this.oldLogFileContents, regexPattern); var newMatches = Regex.Matches(this.newLogFileContents, regexPattern); - newMatches.Should().HaveCountGreaterThanOrEqualTo(oldMatches.Count, "A detector was lost, make sure this was intentional."); + // Account for expected removed detectors + var expectedMatchCount = oldMatches.Count - expectedRemovedDetectors.Count; + newMatches.Should().HaveCountGreaterThanOrEqualTo(expectedMatchCount, "A detector was lost, make sure this was intentional."); var detectorTimes = new Dictionary(); var detectorCounts = new Dictionary(); @@ -196,6 +201,13 @@ public void CheckDetectorsRunTimesAndCounts() else { var detectorId = match.Groups[2].Value; + + // Skip expected removed detectors + if (expectedRemovedDetectors.Contains(detectorId)) + { + continue; + } + detectorTimes.Add(detectorId, float.Parse(match.Groups[3].Value)); detectorCounts.Add(detectorId, int.Parse(match.Groups[4].Value)); } @@ -231,6 +243,9 @@ public void CheckDetectorsRunTimesAndCounts() private void ProcessDetectorVersions() { + // List of detectors that were intentionally removed + var expectedRemovedDetectors = new HashSet { "MavenWithFallback" }; + var oldDetectors = this.oldScanResult.DetectorsInScan; var newDetectors = this.newScanResult.DetectorsInScan; this.bumpedDetectorVersions = []; @@ -240,7 +255,11 @@ private void ProcessDetectorVersions() if (newDetector == null) { - newDetector.Should().NotBeNull($"the detector {cd.DetectorId} was lost, verify this is expected behavior"); + if (!expectedRemovedDetectors.Contains(cd.DetectorId)) + { + newDetector.Should().NotBeNull($"the detector {cd.DetectorId} was lost, verify this is expected behavior"); + } + continue; }