diff --git a/docs/help.md b/docs/help.md index cdd5cf8326..580c9a5c55 100644 --- a/docs/help.md +++ b/docs/help.md @@ -4,7 +4,7 @@ All command line arguments for the `scala-steward` application. ``` Usage: - scala-steward --workspace --repos-file [--repos-file ]... [--git-author-name ] --git-author-email [--git-author-signing-key ] --git-ask-pass [--sign-commits] [--signoff] [--forge-type ] [--forge-api-host ] --forge-login [--do-not-fork] [--add-labels] [--ignore-opts-files] [--env-var ]... [--process-timeout ] [--whitelist ]... [--read-only ]... [--enable-sandbox | --disable-sandbox] [--max-buffer-size ] [--repo-config ]... [--disable-default-repo-config] [--scalafix-migrations ]... [--disable-default-scalafix-migrations] [--artifact-migrations ]... [--disable-default-artifact-migrations] [--cache-ttl ] [--bitbucket-use-default-reviewers] [--bitbucket-server-use-default-reviewers] [--gitlab-merge-when-pipeline-succeeds] [--gitlab-required-reviewers ] [--gitlab-remove-source-branch] [--azure-repos-organization ] [--github-app-id --github-app-key-file ] [--url-checker-test-url ]... [--default-maven-repo ]... [--refresh-backoff-period ] [--exit-code-success-if-any-repo-succeeds] + scala-steward --workspace --repos-file [--repos-file ]... [--git-author-name ] --git-author-email [--git-author-signing-key ] --git-ask-pass [--sign-commits] [--signoff] [--forge-type ] [--forge-api-host ] --forge-login [--do-not-fork] [--add-labels] [--ignore-opts-files] [--env-var ]... [--process-timeout ] [--whitelist ]... [--read-only ]... [--enable-sandbox | --disable-sandbox] [--max-buffer-size ] [--repo-config ]... [--disable-default-repo-config] [--scalafix-migrations ]... [--disable-default-scalafix-migrations] [--artifact-migrations ]... [--disable-default-artifact-migrations] [--cache-ttl ] [--bitbucket-use-default-reviewers] [--bitbucket-server-use-default-reviewers] [--gitlab-merge-when-pipeline-succeeds] [--gitlab-required-reviewers ] [--gitlab-remove-source-branch] [--azure-repos-organization ] [--github-app-id --github-app-key-file ] [--url-checker-test-url ]... [--default-maven-repo ]... [--refresh-backoff-period ] [--exit-code-success-if-any-repo-succeeds] [--whitelist-organization ]... scala-steward validate-repo-config @@ -92,6 +92,8 @@ Options and flags: Period of time a failed build won't be triggered again; default: 0days --exit-code-success-if-any-repo-succeeds Whether the Scala Steward process should exit with success (exit code 0) if any repo succeeds; default: false + --whitelist-organization + List of organizations to bypass UrlChecker Subcommands: validate-repo-config diff --git a/modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala b/modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala index 3da3c65c8e..dd75ef226a 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala @@ -153,6 +153,11 @@ object Cli { options[EnvVar]("env-var", help).orEmpty } + private val whiteListOrganizations: Opts[List[String]] = { + val help = s"List of organizations to bypass UrlChecker" + options[String]("whitelist-organization", help).orEmpty + } + private val processTimeout: Opts[FiniteDuration] = { val default = 10.minutes val help = @@ -351,7 +356,8 @@ object Cli { urlCheckerTestUrls, defaultMavenRepos, refreshBackoffPeriod, - exitCodePolicy + exitCodePolicy, + whiteListOrganizations ).mapN(Config.apply).map(Usage.Regular.apply) private val validateRepoConfig: Opts[Usage] = diff --git a/modules/core/src/main/scala/org/scalasteward/core/application/Config.scala b/modules/core/src/main/scala/org/scalasteward/core/application/Config.scala index 2afed608bf..7c599d0731 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/application/Config.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/application/Config.scala @@ -64,7 +64,8 @@ final case class Config( urlCheckerTestUrls: Nel[Uri], defaultResolvers: List[Resolver], refreshBackoffPeriod: FiniteDuration, - exitCodePolicy: ExitCodePolicy + exitCodePolicy: ExitCodePolicy, + whiteListOrganizations: List[String] ) { def forgeSpecificCfg: ForgeSpecificCfg = forgeCfg.tpe match { diff --git a/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala b/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala index 75d2e32ee5..6e05aee645 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/application/Context.scala @@ -23,7 +23,6 @@ import eu.timepit.refined.types.numeric.PosInt import org.http4s.Uri import org.http4s.client.Client import org.http4s.headers.`User-Agent` -import org.scalasteward.core.application.Config.ForgeCfg import org.scalasteward.core.buildtool.BuildToolDispatcher import org.scalasteward.core.buildtool.gradle.GradleAlg import org.scalasteward.core.buildtool.maven.MavenAlg @@ -167,8 +166,7 @@ object Context { implicit val forgeApiAlg: ForgeApiAlg[F] = ForgeSelection .forgeApiAlg[F](config.forgeCfg, config.forgeSpecificCfg, forgeAuthAlg.authenticateApi) implicit val forgeRepoAlg: ForgeRepoAlg[F] = new ForgeRepoAlg[F](config) - implicit val forgeCfg: ForgeCfg = config.forgeCfg - implicit val updateInfoUrlFinder: UpdateInfoUrlFinder[F] = new UpdateInfoUrlFinder[F] + implicit val updateInfoUrlFinder: UpdateInfoUrlFinder[F] = new UpdateInfoUrlFinder[F](config) implicit val pullRequestRepository: PullRequestRepository[F] = new PullRequestRepository[F](pullRequestsStore) implicit val scalafixCli: ScalafixCli[F] = new ScalafixCli[F] diff --git a/modules/core/src/main/scala/org/scalasteward/core/coursier/DependencyMetadata.scala b/modules/core/src/main/scala/org/scalasteward/core/coursier/DependencyMetadata.scala index 2fa9f101f0..423543617e 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/coursier/DependencyMetadata.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/coursier/DependencyMetadata.scala @@ -49,8 +49,8 @@ final case class DependencyMetadata( urls.find(_.scheme.exists(uri.httpSchemes)).orElse(urls.headOption) } - def forgeRepo(implicit config: ForgeCfg): Option[ForgeRepo] = - repoUrl.flatMap(ForgeRepo.fromRepoUrl) + def forgeRepo(config: ForgeCfg): Option[ForgeRepo] = + repoUrl.flatMap(ForgeRepo.fromRepoUrl(_)(config)) } object DependencyMetadata { diff --git a/modules/core/src/main/scala/org/scalasteward/core/nurture/UpdateInfoUrlFinder.scala b/modules/core/src/main/scala/org/scalasteward/core/nurture/UpdateInfoUrlFinder.scala index 5341a6aeec..b8579f7a45 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/nurture/UpdateInfoUrlFinder.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/nurture/UpdateInfoUrlFinder.scala @@ -19,7 +19,6 @@ package org.scalasteward.core.nurture import cats.Monad import cats.syntax.all.* import org.http4s.Uri -import org.scalasteward.core.application.Config.ForgeCfg import org.scalasteward.core.coursier.DependencyMetadata import org.scalasteward.core.data.Version import org.scalasteward.core.forge.ForgeRepo @@ -27,9 +26,10 @@ import org.scalasteward.core.forge.ForgeType.* import org.scalasteward.core.nurture.UpdateInfoUrl.* import org.scalasteward.core.nurture.UpdateInfoUrlFinder.possibleUpdateInfoUrls import org.scalasteward.core.util.UrlChecker +import org.scalasteward.core.application.Config +import org.scalasteward.core.util.isWhitelisted -final class UpdateInfoUrlFinder[F[_]](implicit - config: ForgeCfg, +final class UpdateInfoUrlFinder[F[_]](config: Config)(implicit urlChecker: UrlChecker[F], F: Monad[F] ) { @@ -38,10 +38,14 @@ final class UpdateInfoUrlFinder[F[_]](implicit versionUpdate: Version.Update ): F[List[UpdateInfoUrl]] = { val updateInfoUrls: List[UpdateInfoUrl] = - dependency.releaseNotesUrl.toList.map(CustomReleaseNotes.apply) ++ - dependency.forgeRepo.toSeq.flatMap(forgeRepo => - possibleUpdateInfoUrls(forgeRepo, versionUpdate) - ) + if (dependency.releaseNotesUrl.exists(isWhitelisted(config.whiteListOrganizations, _))) + dependency.releaseNotesUrl.toList.map(CustomReleaseNotes.apply) + else + dependency.releaseNotesUrl.toList.map(CustomReleaseNotes.apply) ++ + dependency + .forgeRepo(config.forgeCfg) + .toSeq + .flatMap(forgeRepo => possibleUpdateInfoUrls(forgeRepo, versionUpdate)) updateInfoUrls .sorted(UpdateInfoUrl.updateInfoUrlOrder.toOrdering) diff --git a/modules/core/src/main/scala/org/scalasteward/core/util/UrlChecker.scala b/modules/core/src/main/scala/org/scalasteward/core/util/UrlChecker.scala index 92f2a7d536..8462b096d3 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/util/UrlChecker.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/util/UrlChecker.scala @@ -58,10 +58,19 @@ object UrlChecker { } private def status(url: Uri): F[Status] = - statusCache.cachingF(url.renderString)(None) { - val req = Request[F](method = Method.HEAD, uri = url) - modify(req).flatMap(urlCheckerClient.client.status) - } + isWhitelisted(config.whiteListOrganizations, url) + .pure[F] + .ifM( + Status.Ok.pure[F], + statusCache.cachingF(url.renderString)(None) { + val req = Request[F](method = Method.HEAD, uri = url) + modify(req) + .flatTap(req => + logger.info(s"Checking if $url exists: ${req.asCurl(_ => false)}") + ) + .flatMap(urlCheckerClient.client.status) + } + ) } } } diff --git a/modules/core/src/main/scala/org/scalasteward/core/util/package.scala b/modules/core/src/main/scala/org/scalasteward/core/util/package.scala index 9dff54833c..fdc82c5574 100644 --- a/modules/core/src/main/scala/org/scalasteward/core/util/package.scala +++ b/modules/core/src/main/scala/org/scalasteward/core/util/package.scala @@ -20,6 +20,7 @@ import cats.* import cats.syntax.all.* import fs2.Pipe import scala.collection.mutable.ListBuffer +import org.http4s.Uri package object util { final type Nel[+A] = cats.data.NonEmptyList[A] @@ -88,4 +89,10 @@ package object util { Left(s"Unexpected string '$s'. Expected one of: ${expected.mkString(", ")}.") def intellijThisImportIsUsed[A](a: A): Unit = () + + /** check if the url is in the organizations white list + */ + def isWhitelisted(whitelist: List[String], url: Uri): Boolean = + whitelist.map(Uri.Path.unsafeFromString(_)).exists(url.path.startsWith(_)) + } diff --git a/modules/core/src/test/scala/org/scalasteward/core/mock/MockConfig.scala b/modules/core/src/test/scala/org/scalasteward/core/mock/MockConfig.scala index bb757f4c7c..4a92666c65 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/mock/MockConfig.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/mock/MockConfig.scala @@ -15,6 +15,8 @@ object MockConfig { val key = mockRoot / "rsa-4096-private.pem" key.overwrite(Source.fromResource("rsa-4096-private.pem").mkString) + val scalaStewardOrg = "scala-steward-org" + private val args: List[String] = List( s"--workspace=$mockRoot/workspace", s"--repos-file=$reposFile", @@ -30,7 +32,8 @@ object MockConfig { "--add-labels", "--github-app-id=1234", s"--github-app-key-file=$key", - "--refresh-backoff-period=1hour" + "--refresh-backoff-period=1hour", + s"--whitelist-organization=$scalaStewardOrg" ) val Success(Cli.Usage.Regular(config)) = Cli.parseArgs(args): @unchecked } diff --git a/modules/core/src/test/scala/org/scalasteward/core/nurture/UpdateInfoUrlFinderTest.scala b/modules/core/src/test/scala/org/scalasteward/core/nurture/UpdateInfoUrlFinderTest.scala index 8765032eb3..bcb4c244be 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/nurture/UpdateInfoUrlFinderTest.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/nurture/UpdateInfoUrlFinderTest.scala @@ -12,7 +12,7 @@ import org.scalasteward.core.forge.ForgeType.* import org.scalasteward.core.forge.github.Repository import org.scalasteward.core.forge.{ForgeRepo, ForgeType} import org.scalasteward.core.mock.MockContext.context.* -import org.scalasteward.core.mock.{GitHubAuth, MockEff, MockEffOps, MockState} +import org.scalasteward.core.mock.{GitHubAuth, MockConfig, MockEff, MockEffOps, MockState} import org.scalasteward.core.nurture.UpdateInfoUrl.* import org.scalasteward.core.nurture.UpdateInfoUrlFinder.* @@ -84,14 +84,16 @@ class UpdateInfoUrlFinderTest extends CatsEffectSuite with Http4sDsl[MockEff] { assertIO(obtained, List.empty) } - implicit private val config: ForgeCfg = ForgeCfg( + private val forgeConfig: ForgeCfg = ForgeCfg( ForgeType.GitHub, uri"https://github.on-prem.com/", "", doNotFork = false, addLabels = false ) - private val onPremUpdateUrlFinder = new UpdateInfoUrlFinder[MockEff] + val config = MockConfig.config.copy(forgeCfg = forgeConfig) + + private val onPremUpdateUrlFinder = new UpdateInfoUrlFinder[MockEff](config) private val gitHubFooBarRepo = ForgeRepo(GitHub, uri"https://github.com/foo/bar/") private val bitbucketFooBarRepo = ForgeRepo(Bitbucket, uri"https://bitbucket.org/foo/bar/") private val gitLabFooBarRepo = ForgeRepo(GitLab, uri"https://gitlab.com/foo/bar") @@ -282,4 +284,12 @@ class UpdateInfoUrlFinderTest extends CatsEffectSuite with Http4sDsl[MockEff] { val expected = repoUrl / "browse" / "ReleaseNotes.md" assert(clue(obtained).contains(expected)) } + + test("possibleUpdateInfoUrls: CustomReleaseNotes Ok if whitelisted") { + val releaseNotesUrl = uri"https://github.com/scala-steward-org/foo/bar/blob/master/RELEASES.md" + val metadata = DependencyMetadata.empty.copy(releaseNotesUrl = releaseNotesUrl.some) + val obtained = updateInfoUrlFinder.findUpdateInfoUrls(metadata, versionUpdate).runA(state) + assertIO(obtained, List(CustomReleaseNotes(releaseNotesUrl))) + } + } diff --git a/modules/core/src/test/scala/org/scalasteward/core/util/utilTest.scala b/modules/core/src/test/scala/org/scalasteward/core/util/utilTest.scala index 94dd2017b1..10ca1b452d 100644 --- a/modules/core/src/test/scala/org/scalasteward/core/util/utilTest.scala +++ b/modules/core/src/test/scala/org/scalasteward/core/util/utilTest.scala @@ -6,6 +6,7 @@ import munit.ScalaCheckSuite import org.scalacheck.Gen import org.scalacheck.Prop.* import scala.collection.mutable.ListBuffer +import org.http4s.Uri class utilTest extends ScalaCheckSuite { test("appendBounded") { @@ -52,4 +53,16 @@ class utilTest extends ScalaCheckSuite { s.compile.drain.unsafeRunSync() assertEquals(count, n.toInt) } + + test("isWhitelisted") { + assert( + isWhitelisted(List("org1", "org2"), Uri.unsafeFromString("https://github.com/org1/repo")) + ) + assert( + !isWhitelisted(List("org1", "org2"), Uri.unsafeFromString("https://github.com/org3/repo")) + ) + assert( + !isWhitelisted(List("org1", "org2"), Uri.unsafeFromString("https://github.com/org3/org1")) + ) + } }