diff --git a/.github/workflows/stdlib.yaml b/.github/workflows/stdlib.yaml index e8a12152c7c0..b6fed7c3d8cd 100644 --- a/.github/workflows/stdlib.yaml +++ b/.github/workflows/stdlib.yaml @@ -47,6 +47,7 @@ jobs: scala-library-bootstrapped: runs-on: ubuntu-latest + needs : [scala3-compiler-nonbootstrapped, scala3-sbt-bridge-nonbootstrapped, scala-library-nonbootstrapped, scala3-library-nonbootstrapped] steps: - name: Git Checkout uses: actions/checkout@v4 @@ -130,3 +131,54 @@ jobs: - uses: sbt/setup-sbt@v1 - name: Compile `scala3-sbt-bridge-nonbootstrapped` run: ./project/scripts/sbt scala3-sbt-bridge-nonbootstrapped/compile + + tasty-core-bootstrapped: + runs-on: ubuntu-latest + ##needs: [scala3-library-bootstrapped] Add when we add support for caching here + steps: + - name: Git Checkout + uses: actions/checkout@v4 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + cache: 'sbt' + - uses: sbt/setup-sbt@v1 + - name: Compile `tasty-core-bootstrapped` + run: ./project/scripts/sbt tasty-core-bootstrapped-new/compile + + scala3-compiler-bootstrapped: + runs-on: ubuntu-latest + ##needs: [tasty-core-bootstrapped, scala3-library-bootstrapped] Add when we add support for caching here + steps: + - name: Git Checkout + uses: actions/checkout@v4 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + cache: 'sbt' + - uses: sbt/setup-sbt@v1 + - name: Compile `scala3-compiler-bootstrapped` + run: ./project/scripts/sbt scala3-compiler-bootstrapped-new/compile + + scala3-sbt-bridge-bootstrapped: + runs-on: ubuntu-latest + ##needs: [scala3-compiler-bootstrapped] Add when we add support for caching here + steps: + - name: Git Checkout + uses: actions/checkout@v4 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + cache: 'sbt' + - uses: sbt/setup-sbt@v1 + - name: Compile `scala3-sbt-bridge-bootstrapped` + run: ./project/scripts/sbt scala3-sbt-bridge-bootstrapped/compile diff --git a/build.sbt b/build.sbt index 5b9c10dd42c0..07b481a66089 100644 --- a/build.sbt +++ b/build.sbt @@ -1,9 +1,11 @@ val scala3 = Build.scala3 val `scala3-nonbootstrapped` = Build.`scala3-nonbootstrapped` val `scala3-bootstrapped` = Build.`scala3-bootstrapped` +val `scala3-bootstrapped-new` = Build.`scala3-bootstrapped-new` val `scala3-interfaces` = Build.`scala3-interfaces` val `scala3-compiler` = Build.`scala3-compiler` val `scala3-compiler-nonbootstrapped` = Build.`scala3-compiler-nonbootstrapped` +val `scala3-compiler-bootstrapped-new` = Build.`scala3-compiler-bootstrapped-new` val `scala3-compiler-bootstrapped` = Build.`scala3-compiler-bootstrapped` val `scala-library-nonbootstrapped` = Build.`scala-library-nonbootstrapped` val `scala3-library-nonbootstrapped` = Build.`scala3-library-nonbootstrapped` @@ -13,6 +15,7 @@ val `scala3-library` = Build.`scala3-library` val `scala3-library-bootstrapped` = Build.`scala3-library-bootstrapped` val `scala3-library-bootstrappedJS` = Build.`scala3-library-bootstrappedJS` val `scala3-sbt-bridge` = Build.`scala3-sbt-bridge` +val `scala3-sbt-bridge-bootstrapped` = Build.`scala3-sbt-bridge-bootstrapped` val `scala3-sbt-bridge-nonbootstrapped` = Build.`scala3-sbt-bridge-nonbootstrapped` val `scala3-sbt-bridge-tests` = Build.`scala3-sbt-bridge-tests` val `scala3-staging` = Build.`scala3-staging` @@ -27,6 +30,7 @@ val `scala2-library-cc` = Build.`scala2-library-cc` val `scala2-library-cc-tasty` = Build.`scala2-library-cc-tasty` val `tasty-core` = Build.`tasty-core` val `tasty-core-nonbootstrapped` = Build.`tasty-core-nonbootstrapped` +val `tasty-core-bootstrapped-new` = Build.`tasty-core-bootstrapped-new` val `tasty-core-bootstrapped` = Build.`tasty-core-bootstrapped` val `tasty-core-scala2` = Build.`tasty-core-scala2` val scaladoc = Build.scaladoc diff --git a/project/Build.scala b/project/Build.scala index 5f75d156f5a4..cea839a0a54d 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -1396,6 +1396,7 @@ object Build { autoScalaLibrary := false, // do not add a dependency to stdlib, we depend transitively on the stdlib from `scala3-compiler-nonbootstrapped` // Add the source directories for the stdlib (non-boostrapped) Compile / unmanagedSourceDirectories := Seq(baseDirectory.value / "src"), + Compile / resourceDirectory := baseDirectory.value / "resources", // NOTE: The only difference here is that we drop `-Werror` and semanticDB for now Compile / scalacOptions := Seq("-deprecation", "-feature", "-unchecked", "-encoding", "UTF8", "-language:implicitConversions"), // Make sure that the produced artifacts have the minimum JVM version in the bytecode @@ -1430,6 +1431,83 @@ object Build { )}, ) + // ============================================================================================== + // =================================== BOOTSTRAPPED PROJECTS ==================================== + // ============================================================================================== + + lazy val `scala3-bootstrapped-new` = project + .aggregate(`scala3-interfaces`, `scala3-library-bootstrapped-new` , `scala-library-bootstrapped`, + `tasty-core-bootstrapped-new`, `scala3-compiler-bootstrapped-new`, `scala3-sbt-bridge-bootstrapped`) + .settings( + name := "scala3-bootstrapped", + moduleName := "scala3-bootstrapped", + version := dottyVersion, + // Nothing to be published by this project, it is only an aggregate + Compile / publishArtifact := false, + Test / publishArtifact := false, + // Nothing to be published by this project + publish / skip := true, + // Project specific target folder. sbt doesn't like having two projects using the same target folder + target := target.value / "scala3-bootstrapped", + ) + + /* Configuration of the org.scala-lang:scala3-sbt-bridge:*.**.**-bootstrapped project */ + lazy val `scala3-sbt-bridge-bootstrapped` = project.in(file("sbt-bridge")) + .dependsOn(`scala3-compiler-bootstrapped`) // TODO: Would this actually evict the reference compiler in scala-tool? + .settings( + name := "scala3-sbt-bridge-bootstrapped", + moduleName := "scala3-sbt-bridge", + version := dottyVersion, + versionScheme := Some("semver-spec"), + scalaVersion := referenceVersion, // nonbootstrapped artifacts are compiled with the reference compiler (already officially published) + crossPaths := false, // org.scala-lang:scala3-sbt-bridge doesn't have a crosspath + autoScalaLibrary := false, // do not add a dependency to stdlib, we depend transitively on the stdlib from `scala3-compiler-nonbootstrapped` + // Add the source directories for the stdlib (non-boostrapped) + Compile / unmanagedSourceDirectories := Seq(baseDirectory.value / "src"), + Compile / resourceDirectory := baseDirectory.value / "resources", + // NOTE: The only difference here is that we drop `-Werror` and semanticDB for now + Compile / scalacOptions := Seq("-deprecation", "-feature", "-unchecked", "-encoding", "UTF8", "-language:implicitConversions"), + // Make sure that the produced artifacts have the minimum JVM version in the bytecode + Compile / javacOptions ++= Seq("--target", Versions.minimumJVMVersion), + Compile / scalacOptions ++= Seq("--java-output-version", Versions.minimumJVMVersion), + // Packaging configuration of the stdlib + Compile / packageBin / publishArtifact := true, + Compile / packageDoc / publishArtifact := false, + Compile / packageSrc / publishArtifact := true, + // Only publish compilation artifacts, no test artifacts + Test / publishArtifact := false, + // non-bootstrapped stdlib is publishable (only locally) + publish / skip := false, + // Project specific target folder. sbt doesn't like having two projects using the same target folder + target := target.value / "scala3-sbt-bridge-bootstrapped", + // Configure to use the non-bootstrapped compiler + scalaInstance := { + val externalCompilerDeps = (`scala3-compiler-nonbootstrapped` / Compile / externalDependencyClasspath).value.map(_.data).toSet + + // IMPORTANT: We need to use actual jars to form the ScalaInstance and not + // just directories containing classfiles because sbt maintains a cache of + // compiler instances. This cache is invalidated based on timestamps + // however this is only implemented on jars, directories are never + // invalidated. + val tastyCore = (`tasty-core-nonbootstrapped` / Compile / packageBin).value + val scalaLibrary = (`scala-library-nonbootstrapped` / Compile / packageBin).value + val scala3Interfaces = (`scala3-interfaces` / Compile / packageBin).value + val scala3Compiler = (`scala3-compiler-nonbootstrapped` / Compile / packageBin).value + + Defaults.makeScalaInstance( + dottyNonBootstrappedVersion, + libraryJars = Array(scalaLibrary), + allCompilerJars = Seq(tastyCore, scala3Interfaces, scala3Compiler) ++ externalCompilerDeps, + allDocJars = Seq.empty, + state.value, + scalaInstanceTopLoader.value + ) + }, + scalaCompilerBridgeBinaryJar := { + Some((`scala3-sbt-bridge-nonbootstrapped` / Compile / packageBin).value) + }, + ) + // ============================================================================================== // =================================== SCALA STANDARD LIBRARY =================================== // ============================================================================================== @@ -1534,11 +1612,14 @@ object Build { // Make sure that the produced artifacts have the minimum JVM version in the bytecode Compile / javacOptions ++= Seq("--target", Versions.minimumJVMVersion), Compile / scalacOptions ++= Seq("--java-output-version", Versions.minimumJVMVersion), + // Packaging configuration of the stdlib + Compile / packageBin / publishArtifact := true, + Compile / packageDoc / publishArtifact := false, + Compile / packageSrc / publishArtifact := true, // Only publish compilation artifacts, no test artifacts - Compile / publishArtifact := true, Test / publishArtifact := false, // Do not allow to publish this project for now - publish / skip := true, + publish / skip := false, // Project specific target folder. sbt doesn't like having two projects using the same target folder target := target.value / "scala-library-bootstrapped", // we need to have the `scala-library` artifact in the classpath for `ScalaLibraryPlugin` to work @@ -1547,33 +1628,29 @@ object Build { managedScalaInstance := true, // Configure the nonbootstrapped compiler scalaInstance := { - val externalLibraryDeps = (`scala3-library` / Compile / externalDependencyClasspath).value.map(_.data).toSet - val externalCompilerDeps = (`scala3-compiler` / Compile / externalDependencyClasspath).value.map(_.data).toSet + val externalCompilerDeps = (`scala3-compiler-nonbootstrapped` / Compile / externalDependencyClasspath).value.map(_.data).toSet // IMPORTANT: We need to use actual jars to form the ScalaInstance and not // just directories containing classfiles because sbt maintains a cache of // compiler instances. This cache is invalidated based on timestamps // however this is only implemented on jars, directories are never // invalidated. - val tastyCore = (`tasty-core` / Compile / packageBin).value - val scala3Library = (`scala3-library` / Compile / packageBin).value + val tastyCore = (`tasty-core-nonbootstrapped` / Compile / packageBin).value + val scalaLibrary = (`scala-library-nonbootstrapped` / Compile / packageBin).value val scala3Interfaces = (`scala3-interfaces` / Compile / packageBin).value - val scala3Compiler = (`scala3-compiler` / Compile / packageBin).value - - val libraryJars = Array(scala3Library) ++ externalLibraryDeps - val compilerJars = Seq(tastyCore, scala3Interfaces, scala3Compiler) ++ (externalCompilerDeps -- externalLibraryDeps) + val scala3Compiler = (`scala3-compiler-nonbootstrapped` / Compile / packageBin).value Defaults.makeScalaInstance( dottyNonBootstrappedVersion, - libraryJars = libraryJars, - allCompilerJars = compilerJars, - allDocJars = Seq.empty, + libraryJars = Array(scalaLibrary), + allCompilerJars = Seq(tastyCore, scala3Interfaces, scala3Compiler) ++ externalCompilerDeps, + allDocJars = Seq.empty, state.value, scalaInstanceTopLoader.value ) }, scalaCompilerBridgeBinaryJar := { - Some((`scala3-sbt-bridge` / Compile / packageBin).value) + Some((`scala3-sbt-bridge-nonbootstrapped` / Compile / packageBin).value) }, ) @@ -1605,11 +1682,14 @@ object Build { Test / compile := (`scala-library-bootstrapped` / Test / compile).value, Test / doc := (`scala-library-bootstrapped` / Test / doc).value, Test / run := (`scala-library-bootstrapped` / Test / run).evaluated, + // Packaging configuration of the stdlib + Compile / packageBin / publishArtifact := true, + Compile / packageDoc / publishArtifact := false, + Compile / packageSrc / publishArtifact := true, // Only publish compilation artifacts, no test artifacts - Compile / publishArtifact := true, Test / publishArtifact := false, // Do not allow to publish this project for now - publish / skip := true, + publish / skip := false, // Project specific target folder. sbt doesn't like having two projects using the same target folder target := target.value / "scala3-library-bootstrapped", ) @@ -1665,11 +1745,67 @@ object Build { )}, ) + /* Configuration of the org.scala-lang:tasty-core_3:*.**.**-bootstrapped project */ + lazy val `tasty-core-bootstrapped-new` = project.in(file("tasty")) + .dependsOn(`scala3-library-bootstrapped`) + .settings( + name := "tasty-core-bootstrapped", + moduleName := "tasty-core", + version := dottyVersion, + versionScheme := Some("semver-spec"), + scalaVersion := referenceVersion, // nonbootstrapped artifacts are compiled with the reference compiler (already officially published) + crossPaths := true, // org.scala-lang:tasty-core has a crosspath + // sbt shouldn't add stdlib automatically, we depend on `scala3-library-nonbootstrapped` + autoScalaLibrary := false, + // Add the source directories for the stdlib (non-boostrapped) + Compile / unmanagedSourceDirectories := Seq(baseDirectory.value / "src"), + Compile / unmanagedSourceDirectories += baseDirectory.value / "src-bootstrapped", + // Make sure that the produced artifacts have the minimum JVM version in the bytecode + Compile / javacOptions ++= Seq("--target", Versions.minimumJVMVersion), + Compile / scalacOptions ++= Seq("--java-output-version", Versions.minimumJVMVersion), + // Packaging configuration of the stdlib + Compile / packageBin / publishArtifact := true, + Compile / packageDoc / publishArtifact := false, + Compile / packageSrc / publishArtifact := true, + // Only publish compilation artifacts, no test artifacts + Test / publishArtifact := false, + // Do not allow to publish this project for now + publish / skip := false, + // Project specific target folder. sbt doesn't like having two projects using the same target folder + target := target.value / "tasty-core-bootstrapped", + // Configure to use the non-bootstrapped compiler + scalaInstance := { + val externalCompilerDeps = (`scala3-compiler-nonbootstrapped` / Compile / externalDependencyClasspath).value.map(_.data).toSet + + // IMPORTANT: We need to use actual jars to form the ScalaInstance and not + // just directories containing classfiles because sbt maintains a cache of + // compiler instances. This cache is invalidated based on timestamps + // however this is only implemented on jars, directories are never + // invalidated. + val tastyCore = (`tasty-core-nonbootstrapped` / Compile / packageBin).value + val scalaLibrary = (`scala-library-nonbootstrapped` / Compile / packageBin).value + val scala3Interfaces = (`scala3-interfaces` / Compile / packageBin).value + val scala3Compiler = (`scala3-compiler-nonbootstrapped` / Compile / packageBin).value + + Defaults.makeScalaInstance( + dottyNonBootstrappedVersion, + libraryJars = Array(scalaLibrary), + allCompilerJars = Seq(tastyCore, scala3Interfaces, scala3Compiler) ++ externalCompilerDeps, + allDocJars = Seq.empty, + state.value, + scalaInstanceTopLoader.value + ) + }, + scalaCompilerBridgeBinaryJar := { + Some((`scala3-sbt-bridge-nonbootstrapped` / Compile / packageBin).value) + }, + ) + // ============================================================================================== // ======================================= SCALA COMPILER ======================================= // ============================================================================================== - /* Configuration of the org.scala-lang:scala3-compiler_3:*.**.**-nonbootstrapped project */ + /* Configuration of the org.scala-lang:scala3-compiler_3:*.**.**-nonbootstrapped project */ lazy val `scala3-compiler-nonbootstrapped` = project.in(file("compiler")) .dependsOn(`scala3-interfaces`, `tasty-core-nonbootstrapped`, `scala3-library-nonbootstrapped`) .settings( @@ -1790,6 +1926,136 @@ object Build { }.taskValue, ) + /* Configuration of the org.scala-lang:scala3-compiler_3:*.**.**-bootstrapped project */ + lazy val `scala3-compiler-bootstrapped-new` = project.in(file("compiler")) + .dependsOn(`scala3-interfaces`, `tasty-core-bootstrapped`, `scala3-library-bootstrapped`) + .settings( + name := "scala3-compiler-bootstrapped", + moduleName := "scala3-compiler", + version := dottyVersion, + versionScheme := Some("semver-spec"), + scalaVersion := referenceVersion, // nonbootstrapped artifacts are compiled with the reference compiler (already officially published) + crossPaths := true, // org.scala-lang:scala3-compiler has a crosspath + // sbt shouldn't add stdlib automatically, we depend on `scala3-library-nonbootstrapped` + autoScalaLibrary := false, + // Add the source directories for the stdlib (non-boostrapped) + Compile / unmanagedSourceDirectories := Seq(baseDirectory.value / "src"), + Compile / unmanagedSourceDirectories += baseDirectory.value / "src-bootstrapped", + // All the dependencies needed by the compiler + libraryDependencies ++= Seq( + "org.scala-lang.modules" % "scala-asm" % "9.8.0-scala-1", + Dependencies.compilerInterface, + "org.jline" % "jline-reader" % "3.29.0", + "org.jline" % "jline-terminal" % "3.29.0", + "org.jline" % "jline-terminal-jni" % "3.29.0", + //("io.get-coursier" %% "coursier" % "2.0.16" % Test).cross(CrossVersion.for3Use2_13), + ), + // NOTE: The only difference here is that we drop `-Werror` and semanticDB for now + Compile / scalacOptions := Seq("-deprecation", "-feature", "-unchecked", "-encoding", "UTF8", "-language:implicitConversions"), + // TODO: Enable these flags when the new stdlib is explicitelly null checked + //Compile / scalacOptions ++= Seq("-Yexplicit-nulls", "-Wsafe-init"), + // Make sure that the produced artifacts have the minimum JVM version in the bytecode + Compile / javacOptions ++= Seq("--target", Versions.minimumJVMVersion), + Compile / scalacOptions ++= Seq("--java-output-version", Versions.minimumJVMVersion), + // Packaging configuration of the stdlib + Compile / packageBin / publishArtifact := true, + Compile / packageDoc / publishArtifact := false, + Compile / packageSrc / publishArtifact := true, + // Only publish compilation artifacts, no test artifacts + Test / publishArtifact := false, + // Do not allow to publish this project for now + publish / skip := false, + // Project specific target folder. sbt doesn't like having two projects using the same target folder + target := target.value / "scala3-compiler-bootstrapped", + // Generate compiler.properties, used by sbt + Compile / resourceGenerators += Def.task { + import java.util._ + import java.text._ + val file = (Compile / resourceManaged).value / "compiler.properties" + val dateFormat = new SimpleDateFormat("yyyyMMdd-HHmmss") + dateFormat.setTimeZone(TimeZone.getTimeZone("GMT")) + val contents = //2.11.11.v20170413-090219-8a413ba7cc + s"""version.number=${version.value} + |maven.version.number=${version.value} + |git.hash=${VersionUtil.gitHash} + |copyright.string=Copyright 2002-$currentYear, LAMP/EPFL + """.stripMargin + + if (!(file.exists && IO.read(file) == contents)) { + IO.write(file, contents) + } + + Seq(file) + }.taskValue, + // Configure to use the non-bootstrapped compiler + scalaInstance := { + val externalCompilerDeps = (`scala3-compiler-nonbootstrapped` / Compile / externalDependencyClasspath).value.map(_.data).toSet + + // IMPORTANT: We need to use actual jars to form the ScalaInstance and not + // just directories containing classfiles because sbt maintains a cache of + // compiler instances. This cache is invalidated based on timestamps + // however this is only implemented on jars, directories are never + // invalidated. + val tastyCore = (`tasty-core-nonbootstrapped` / Compile / packageBin).value + val scalaLibrary = (`scala-library-nonbootstrapped` / Compile / packageBin).value + val scala3Interfaces = (`scala3-interfaces` / Compile / packageBin).value + val scala3Compiler = (`scala3-compiler-nonbootstrapped` / Compile / packageBin).value + + Defaults.makeScalaInstance( + dottyNonBootstrappedVersion, + libraryJars = Array(scalaLibrary), + allCompilerJars = Seq(tastyCore, scala3Interfaces, scala3Compiler) ++ externalCompilerDeps, + allDocJars = Seq.empty, + state.value, + scalaInstanceTopLoader.value + ) + }, + scalaCompilerBridgeBinaryJar := { + Some((`scala3-sbt-bridge-nonbootstrapped` / Compile / packageBin).value) + }, + /* Add the sources of scalajs-ir. + * To guarantee that dotty can bootstrap without depending on a version + * of scalajs-ir built with a different Scala compiler, we add its + * sources instead of depending on the binaries. + */ + ivyConfigurations += SourceDeps.hide, + transitiveClassifiers := Seq("sources"), + libraryDependencies += + ("org.scala-js" %% "scalajs-ir" % scalaJSVersion % "sourcedeps").cross(CrossVersion.for3Use2_13), + Compile / sourceGenerators += Def.task { + val s = streams.value + val cacheDir = s.cacheDirectory + val trgDir = (Compile / sourceManaged).value / "scalajs-ir-src" + + val report = updateClassifiers.value + val scalaJSIRSourcesJar = report.select( + configuration = configurationFilter("sourcedeps"), + module = (_: ModuleID).name.startsWith("scalajs-ir_"), + artifact = artifactFilter(`type` = "src")).headOption.getOrElse { + sys.error(s"Could not fetch scalajs-ir sources") + } + + FileFunction.cached(cacheDir / s"fetchScalaJSIRSource", + FilesInfo.lastModified, FilesInfo.exists) { dependencies => + s.log.info(s"Unpacking scalajs-ir sources to $trgDir...") + if (trgDir.exists) + IO.delete(trgDir) + IO.createDirectory(trgDir) + IO.unzip(scalaJSIRSourcesJar, trgDir) + + val sjsSources = (trgDir ** "*.scala").get.toSet + sjsSources.foreach(f => { + val lines = IO.readLines(f) + val linesWithPackage = replacePackage(lines) { + case "org.scalajs.ir" => "dotty.tools.sjs.ir" + } + IO.writeLines(f, insertUnsafeNullsImport(linesWithPackage)) + }) + sjsSources + } (Set(scalaJSIRSourcesJar)).toSeq + }.taskValue, + ) + def dottyLibrary(implicit mode: Mode): Project = mode match { case NonBootstrapped => `scala3-library` case Bootstrapped => `scala3-library-bootstrapped`