diff --git a/modules/build/src/test/scala/scala/build/tests/ActionableDiagnosticTests.scala b/modules/build/src/test/scala/scala/build/tests/ActionableDiagnosticTests.scala index 9e4913399a..fafe5345be 100644 --- a/modules/build/src/test/scala/scala/build/tests/ActionableDiagnosticTests.scala +++ b/modules/build/src/test/scala/scala/build/tests/ActionableDiagnosticTests.scala @@ -1,14 +1,23 @@ package scala.build.tests import com.eed3si9n.expecty.Expecty.expect +import com.sun.net.httpserver.HttpServer +import coursier.cache.FileCache +import coursier.util.Task import coursier.version.Version +import java.net.InetSocketAddress +import java.util.UUID +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.{ConcurrentLinkedQueue, Executors} + import scala.build.Ops.* import scala.build.Position.File import scala.build.actionable.ActionableDiagnostic.* import scala.build.actionable.ActionablePreprocessor import scala.build.options.{BuildOptions, InternalOptions, SuppressWarningOptions} import scala.build.{BuildThreads, Directories, LocalRepo} +import scala.jdk.CollectionConverters.* class ActionableDiagnosticTests extends TestUtil.ScalaCliBuildSuite { @@ -23,6 +32,57 @@ class ActionableDiagnosticTests extends TestUtil.ScalaCliBuildSuite { def path2url(p: os.Path): String = p.toIO.toURI.toURL.toString + /** Minimal HTTP Maven repo: records every request path, then optional delay on + * `maven-metadata.xml` when `delayWhen()` is true, then serves a body from `responses` or 404. + */ + def withRecordingMavenRepo( + responses: Map[String, Array[Byte]], + delayOnMetadataMs: Long = 0, + delayWhen: () => Boolean = () => false + )(body: (String, ConcurrentLinkedQueue[String]) => Unit): Unit = + val recorded = new ConcurrentLinkedQueue[String]() + val server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0) + server.setExecutor(Executors.newCachedThreadPool()) + server.createContext( + "/", + ex => { + val path = ex.getRequestURI.getPath + recorded.offer(path) + if delayOnMetadataMs > 0 && delayWhen() && path.endsWith("maven-metadata.xml") then + Thread.sleep(delayOnMetadataMs) + responses.get(path) match + case Some(bytes) => + ex.getResponseHeaders.set("Content-Type", "application/xml") + ex.sendResponseHeaders(200, bytes.length) + ex.getResponseBody.write(bytes) + ex.getResponseBody.close() + case None => + ex.sendResponseHeaders(404, -1) + ex.close() + } + ) + server.start() + try + val base = s"http://127.0.0.1:${server.getAddress.getPort}/" + body(base, recorded) + finally server.stop(0) + + def clearRecordedPaths(q: ConcurrentLinkedQueue[String]): Unit = + while q.poll() != null do () + + /** `Build.build` populates Coursier's disk cache with `maven-metadata.xml`. A later + * `cache.versions` during actionable checks often reuses it and never hits HTTP again, so + * recording-server assertions cannot tell pre-fix from post-fix. Use an empty cache directory + * for passes where we assert on network traffic. + */ + def buildOptionsWithEmptyCoursierCache(opts: BuildOptions): BuildOptions = + val dir = os.temp.dir(prefix = "scala-cli-actionable-diagnostic-coursier-") + opts.copy(internal = + opts.internal.copy( + cache = Some(FileCache[Task]().withLocation(dir.toString)) + ) + ) + test("using outdated os-lib") { val dependencyOsLib = "com.lihaoyi::os-lib:0.7.8" val testInputs = TestInputs( @@ -263,4 +323,226 @@ class ActionableDiagnosticTests extends TestUtil.ScalaCliBuildSuite { expect(testLibDiagnosticOpt.isEmpty) } } + + test("actionable outdated check for toolkit skips user repository metadata") { + val meta = + """ + | + | org.scala-lang + | toolkit_3 + | + | 99.0.0 + | 99.0.0 + | + | 0.3.0 + | 99.0.0 + | + | + | + |""".stripMargin.getBytes("UTF-8") + val responses = Map("/org/scala-lang/toolkit_3/maven-metadata.xml" -> meta) + withRecordingMavenRepo(responses)((repoUrl, recorded) => + val testInputs = TestInputs( + os.rel / "Foo.scala" -> + """//> using toolkit 0.3.0 + | + |object Hello extends App { + | println("Hello") + |} + |""".stripMargin + ) + val withRepo = baseOptions.copy( + classPathOptions = + baseOptions.classPathOptions.copy(extraRepositories = Seq(repoUrl)) + ) + testInputs.withBuild(withRepo, buildThreads, None, actionableDiagnostics = true) { + (_, _, maybeBuild) => + val build = maybeBuild.orThrow + clearRecordedPaths(recorded) + ActionablePreprocessor + .generateActionableDiagnostics(buildOptionsWithEmptyCoursierCache(build.options)) + .orThrow + val paths = recorded.asScala.toSeq + expect(!paths.exists(_.contains("toolkit_3/maven-metadata.xml"))) + } + ) + } + + test("actionable outdated check for org.scala-lang skips user repository metadata") { + val u = UUID.randomUUID().toString.replace("-", "") + val art = s"scala_cli_fake_$u" + val meta = + s""" + | + | org.scala-lang + | ${art}_3 + | + | 99.0.0 + | 99.0.0 + | + | 0.1.0 + | 99.0.0 + | + | + | + |""".stripMargin.getBytes("UTF-8") + val pom = + s""" + | + | org.scala-lang + | ${art}_3 + | 0.1.0 + |""".stripMargin.getBytes("UTF-8") + val responses = Map( + s"/org/scala-lang/${art}_3/maven-metadata.xml" -> meta, + s"/org/scala-lang/${art}_3/0.1.0/${art}_3-0.1.0.pom" -> pom + ) + withRecordingMavenRepo(responses)((repoUrl, recorded) => + val testInputs = TestInputs( + os.rel / "Foo.scala" -> + s"""//> using dep org.scala-lang::$art:0.1.0 + | + |object Hello extends App { + | println("Hello") + |} + |""".stripMargin + ) + val withRepo = baseOptions.copy( + classPathOptions = + baseOptions.classPathOptions.copy(extraRepositories = Seq(repoUrl)) + ) + testInputs.withBuild(withRepo, buildThreads, None, actionableDiagnostics = true) { + (_, _, maybeBuild) => + val build = maybeBuild.orThrow + clearRecordedPaths(recorded) + ActionablePreprocessor + .generateActionableDiagnostics(buildOptionsWithEmptyCoursierCache(build.options)) + .orThrow + val paths = recorded.asScala.toSeq + expect(!paths.exists(p => p.contains(s"${art}_3/") && p.contains("maven-metadata.xml"))) + } + ) + } + + test("actionable outdated check still consults user repository for other organizations") { + val u = UUID.randomUUID().toString.replace("-", "") + val art = s"scala_cli_fake_$u" + val meta = + s""" + | + | test-org + | ${art}_3 + | + | 99.0.0 + | 99.0.0 + | + | 0.1.0 + | 99.0.0 + | + | + | + |""".stripMargin.getBytes("UTF-8") + val pom = + s""" + | + | test-org + | ${art}_3 + | 0.1.0 + |""".stripMargin.getBytes("UTF-8") + val responses = Map( + s"/test-org/${art}_3/maven-metadata.xml" -> meta, + s"/test-org/${art}_3/0.1.0/${art}_3-0.1.0.pom" -> pom + ) + withRecordingMavenRepo(responses)((repoUrl, recorded) => + val testInputs = TestInputs( + os.rel / "Foo.scala" -> + s"""//> using dep test-org::$art:0.1.0 + | + |object Hello extends App { + | println("Hello") + |} + |""".stripMargin + ) + val withRepo = baseOptions.copy( + classPathOptions = + baseOptions.classPathOptions.copy(extraRepositories = Seq(repoUrl)) + ) + testInputs.withBuild(withRepo, buildThreads, None, actionableDiagnostics = true) { + (_, _, maybeBuild) => + val build = maybeBuild.orThrow + val paths = recorded.asScala.toSeq + expect(paths.exists(p => p.contains(s"${art}_3/") && p.contains("maven-metadata.xml"))) + val updateDiagnostics = + ActionablePreprocessor.generateActionableDiagnostics(build.options).orThrow + val dOpt = updateDiagnostics.collectFirst { + case diagnostic: ActionableDependencyUpdateDiagnostic => diagnostic + } + expect(dOpt.nonEmpty) + expect(dOpt.get.newVersion == "99.0.0") + } + ) + } + + test("actionable outdated check times out slow user repository") { + val u = UUID.randomUUID().toString.replace("-", "") + val art = s"scala_cli_fake_$u" + val meta = + s""" + | + | test-org + | ${art}_3 + | + | 0.2.0 + | 0.2.0 + | + | 0.1.0 + | 0.2.0 + | + | + | + |""".stripMargin.getBytes("UTF-8") + val pom = + s""" + | + | test-org + | ${art}_3 + | 0.1.0 + |""".stripMargin.getBytes("UTF-8") + val responses = Map( + s"/test-org/${art}_3/maven-metadata.xml" -> meta, + s"/test-org/${art}_3/0.1.0/${art}_3-0.1.0.pom" -> pom + ) + val slowAfterClear = new AtomicBoolean(false) + withRecordingMavenRepo( + responses, + delayOnMetadataMs = 30_000L, + delayWhen = () => slowAfterClear.get() + )((repoUrl, recorded) => + val testInputs = TestInputs( + os.rel / "Foo.scala" -> + s"""//> using dep test-org::$art:0.1.0 + | + |object Hello extends App { + | println("Hello") + |} + |""".stripMargin + ) + val withRepo = baseOptions.copy( + classPathOptions = + baseOptions.classPathOptions.copy(extraRepositories = Seq(repoUrl)) + ) + testInputs.withBuild(withRepo, buildThreads, None, actionableDiagnostics = true) { + (_, _, maybeBuild) => + val build = maybeBuild.orThrow + clearRecordedPaths(recorded) + slowAfterClear.set(true) + val t0 = System.nanoTime() + ActionablePreprocessor + .generateActionableDiagnostics(buildOptionsWithEmptyCoursierCache(build.options)) + .orThrow + val elapsedMs = (System.nanoTime() - t0) / 1_000_000 + expect(elapsedMs < 15_000) + } + ) + } } diff --git a/modules/options/src/main/scala/scala/build/actionable/ActionableDependencyHandler.scala b/modules/options/src/main/scala/scala/build/actionable/ActionableDependencyHandler.scala index fd538d882a..6db1c79a9c 100644 --- a/modules/options/src/main/scala/scala/build/actionable/ActionableDependencyHandler.scala +++ b/modules/options/src/main/scala/scala/build/actionable/ActionableDependencyHandler.scala @@ -1,6 +1,6 @@ package scala.build.actionable import coursier.cache.FileCache -import coursier.core.Repository +import coursier.core.{Repository, Versions as CoreVersions} import coursier.util.Task import coursier.version.{Latest, Version} import dependency.* @@ -13,6 +13,9 @@ import scala.build.internal.Util.* import scala.build.options.BuildOptions import scala.build.options.ScalaVersionUtil.versions import scala.build.{Logger, Positioned} +import scala.concurrent.duration.{DurationInt, FiniteDuration} +import scala.concurrent.{Await, ExecutionContext, Future, TimeoutException} +import scala.util.control.NonFatal case object ActionableDependencyHandler extends ActionableHandler[ActionableDependencyUpdateDiagnostic] { @@ -68,6 +71,20 @@ case object ActionableDependencyHandler /** Versions like 'latest.*': 'latest.release', 'latest.integration', 'latest.stable' */ private def isLatestSyntaxVersion(version: String): Boolean = Latest(version).nonEmpty + + /** Artifacts under this organization are only published to public Maven Central (not to user + * `//> using repository` hosts). Listing versions against private repos can hang or add large + * latency. + */ + private def isScalaLangOrganization(dep: AnyDependency): Boolean = + dep.module.organization == Constants.toolkitOrganization + + private val perRepoVersionsTimeout: FiniteDuration = 5.seconds + + private def mergeCoreVersions(parts: Seq[CoreVersions]): CoreVersions = + val mergedAvailable = parts.flatMap(_.available0).distinctBy(_.asString).toList + CoreVersions.empty.withAvailable0(mergedAvailable) + private def findLatestVersion( buildOptions: BuildOptions, setting: Positioned[AnyDependency], @@ -77,13 +94,54 @@ case object ActionableDependencyHandler val scalaParams: Option[ScalaParameters] = value(buildOptions.scalaParams) val cache: FileCache[Task] = buildOptions.finalCache val csModule: coursier.core.Module = value(dependency.toCs(scalaParams)).module - val repositories: Seq[Repository] = value(buildOptions.finalRepositories) + val includeUserExtraRepositories = !isScalaLangOrganization(dependency) + val repositories: Seq[Repository] = + value(buildOptions.finalRepositories(includeUserExtraRepositories)) + + given ExecutionContext = ExecutionContext.global + + val perRepoParts = try + Await.result( + Future.sequence(repositories.map { repo => + val label = repo.toString.take(200) + Future { + val listing = + Future(cache.versions(csModule, Seq(repo)).versions)(using ExecutionContext.global) + try Await.result(listing, perRepoVersionsTimeout) + catch { + case _: TimeoutException => + loggerOpt.foreach(_.debug( + s"Timeout listing versions for ${dependency.render} from repository $label (after $perRepoVersionsTimeout)" + )) + CoreVersions.empty + case NonFatal(e) => + loggerOpt.foreach(_.debug( + s"Failed listing versions for ${dependency.render} from repository $label: ${e.getMessage}" + )) + CoreVersions.empty + } + }(using ExecutionContext.global) + }), + perRepoVersionsTimeout + 3.seconds + ) + catch { + case _: TimeoutException => + loggerOpt.foreach(_.debug( + s"Timeout traversing repositories for ${dependency.render} (after ${perRepoVersionsTimeout + + 3.seconds})" + )) + Seq.empty[CoreVersions] + case NonFatal(e) => + loggerOpt.foreach(_.debug( + s"Failed traversing repositories for ${dependency.render}: ${e.getMessage}" + )) + Seq.empty[CoreVersions] + } - val latestVersionOpt = cache.versions(csModule, repositories) - .versions - .latest(Latest.Stable) + val mergedVersions = mergeCoreVersions(perRepoParts) + val latestVersionOpt = mergedVersions.latest(Latest.Stable) - if (latestVersionOpt.isEmpty) + if latestVersionOpt.isEmpty then loggerOpt.foreach(_.diagnostic( s"No latest version found for ${dependency.render}", Severity.Warning, diff --git a/modules/options/src/main/scala/scala/build/options/BuildOptions.scala b/modules/options/src/main/scala/scala/build/options/BuildOptions.scala index aa96da1d41..404de4a25b 100644 --- a/modules/options/src/main/scala/scala/build/options/BuildOptions.scala +++ b/modules/options/src/main/scala/scala/build/options/BuildOptions.scala @@ -261,7 +261,16 @@ final case class BuildOptions( private val scala2NightlyRepo = Seq(coursier.Repositories.scalaIntegration.root) private val scala3NightlyRepo = Seq(RepositoryUtils.scala3NightlyRepository.root) - def finalRepositories: Either[BuildException, Seq[Repository]] = either { + def finalRepositories: Either[BuildException, Seq[Repository]] = + finalRepositories(includeUserExtraRepositories = true) + + /** @param includeUserExtraRepositories + * when false, repositories from `//> using repository` are omitted (nightly, internal local, + * and snapshot repositories are kept). + */ + def finalRepositories( + includeUserExtraRepositories: Boolean + ): Either[BuildException, Seq[Repository]] = either { val maybeSv = scalaOptions.scalaVersion .map(_.asString) .orElse(scalaOptions.defaultScalaVersion) @@ -277,7 +286,10 @@ final case class BuildOptions( RepositoryUtils.scala3NightlyRepository ) else Nil - val extraRepositories = classPathOptions.extraRepositories.filterNot(_ == "snapshots") + val userExtraRepositories = classPathOptions.extraRepositories.filterNot(_ == "snapshots") + val extraRepositories = + if includeUserExtraRepositories then userExtraRepositories + else Nil val repositories = nightlyRepos ++ extraRepositories ++