chore: port Scala.js to the new build (#23741)

Port the implementation of Scala.js scalalib to the new build
NOTE: We still need to overwrite the `.sjsir` of specialized classes (as
we do when we overwrite `.class`files)

Author: hamzaremmal

.github/workflows/stdlib.yaml

@@ -219,6 +219,49 @@ jobs:
       - name: Compile `scala3-tasty-inspector`
         run: ./project/scripts/sbt scala3-tasty-inspector-new/compile
 
+  scala-library-sjs:
+    runs-on: ubuntu-latest
+    ## Add when we add support for caching here
+    ##needs: [scala3-library-nonbootstrapped,
+    ##        tasty-core-nonbootstrapped,
+    ##        scala3-compiler-nonbootstrapped,
+    ##        scala3-sbt-bridge-nonbootstrapped]
+    steps:
+      - name: Git Checkout
+        uses: actions/checkout@v5
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 17
+          cache: 'sbt'
+      - uses: sbt/setup-sbt@v1
+      - name: Compile `scala3-staging`
+        run: ./project/scripts/sbt scala3-staging-new/compile
+      - name: Compile `scala-library` for Scala.js
+        run: ./project/scripts/sbt scala-library-sjs/compile
+
+  scala3-library-sjs:
+    runs-on: ubuntu-latest
+    ## Add when we add support for caching here
+    ##needs: [scala-library-sjs]
+    steps:
+      - name: Git Checkout
+        uses: actions/checkout@v5
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 17
+          cache: 'sbt'
+      - uses: sbt/setup-sbt@v1
+      - name: Compile `scala3-staging`
+        run: ./project/scripts/sbt scala3-staging-new/compile
+      - name: Compile `scala3-library` for Scala.js
+        run: ./project/scripts/sbt scala3-library-sjs/compile
+
   #################################################################################################
   ########################################### TEST JOBS ###########################################
   #################################################################################################

build.sbt

@@ -7,6 +7,8 @@ val `scala3-compiler` = Build.`scala3-compiler`
 val `scala3-compiler-nonbootstrapped` = Build.`scala3-compiler-nonbootstrapped`
 val `scala3-compiler-bootstrapped-new` = Build.`scala3-compiler-bootstrapped-new`
 val `scala3-compiler-bootstrapped` = Build.`scala3-compiler-bootstrapped`
+val `scala-library-sjs` = Build.`scala-library-sjs`
+val `scala3-library-sjs` = Build.`scala3-library-sjs`
 val `scala-library-nonbootstrapped` = Build.`scala-library-nonbootstrapped`
 val `scala3-library-nonbootstrapped` = Build.`scala3-library-nonbootstrapped`
 val `scala-library-bootstrapped` = Build.`scala-library-bootstrapped`

library-js/src/scala/Array.scala

@@ -68,7 +68,7 @@ object Array {
   /**
    * Returns a new [[scala.collection.mutable.ArrayBuilder]].
    */
-  def newBuilder[T](implicit t: ClassTag[T]): ArrayBuilder[T] = ArrayBuilder.make[T](t)
+  def newBuilder[T](implicit t: ClassTag[T]): ArrayBuilder[T] = ArrayBuilder.make[T](using t)
 
   def from[A : ClassTag](it: IterableOnce[A]): Array[A] = {
     val n = it.knownSize

library-js/src/scala/Console.scala

@@ -280,5 +280,5 @@ object Console extends AnsiColor {
    *  @throws java.lang.IllegalArgumentException if there was a problem with the format string or arguments
    *  @group console-output
    */
-  def printf(text: String, args: Any*): Unit = { out.print(text format (args : _*)) }
+  def printf(text: String, args: Any*): Unit = { out.print(text.format(args*)) }
 }

library/src/scala/collection/StringOps.scala

@@ -966,7 +966,7 @@ final class StringOps(private val s: String) extends AnyVal {
     else new WrappedString(s).toArray[B]
 
   private[this] def unwrapArg(arg: Any): AnyRef = arg match {
-    case x: ScalaNumber => x.underlying
+    case x: ScalaNumber => x.underlying()
     case x              => x.asInstanceOf[AnyRef]
   }
 

project/Build.scala

@@ -40,6 +40,9 @@ import sbttastymima.TastyMiMaPlugin.autoImport._
 
 import scala.util.Properties.isJavaAtLeast
 
+import scala.xml.{Node => XmlNode, NodeSeq => XmlNodeSeq, _}
+import scala.xml.transform.{RewriteRule, RuleTransformer}
+
 import org.portablescala.sbtplatformdeps.PlatformDepsPlugin.autoImport._
 
 object Build {
@@ -1451,7 +1454,7 @@ object Build {
   lazy val `scala3-bootstrapped-new` = project
     .aggregate(`scala3-interfaces`, `scala3-library-bootstrapped-new` , `scala-library-bootstrapped`,
       `tasty-core-bootstrapped-new`, `scala3-compiler-bootstrapped-new`, `scala3-sbt-bridge-bootstrapped`,
-      `scala3-staging-new`, `scala3-tasty-inspector-new`)
+      `scala3-staging-new`, `scala3-tasty-inspector-new`, `scala-library-sjs`, `scala3-library-sjs`)
     .settings(
       name          := "scala3-bootstrapped",
       moduleName    := "scala3-bootstrapped",
@@ -1833,6 +1836,153 @@ object Build {
       target := target.value / "scala3-library-bootstrapped",
     )
 
+  /* Configuration of the org.scala-js:scalajs-scalalib_2.13:*.**.**-bootstrapped project */
+  lazy val `scala-library-sjs` = project.in(file("library-js"))
+    // We add a dependency to the JVM library to have the classfile available
+    // (as they are not part of this artifact)
+    .dependsOn(`scala3-library-bootstrapped-new`)
+    .settings(
+      name          := "scala-library-sjs",
+      organization  := "org.scala-js",
+      // This is very tricky here since this is a Scala 3 project, but to be able to smoothly
+      // migrate the ecosystem, we need to be able to evict the Scala 2 library from the classpath.
+      // The problem is that the Scala 2 library for Scala.js has a _2.13 in the module's name, so we need
+      // to release Scala 3 for Scala.js with the same _2.13 instead of the _3.
+      // Yes, I know, this is weird and feels wrong.
+      moduleName    := "scalajs-scalalib_2.13",
+      version       := dottyVersion,
+      versionScheme := Some("semver-spec"),
+      crossPaths    := false,
+      // sbt defaults to scala 2.12.x and metals will report issues as it doesn't consider the project a scala 3 project
+      // (not the actual version we use to compile the project)
+      scalaVersion  := referenceVersion,
+      // Add the source directories for the stdlib (non-boostrapped)
+      Compile / unmanagedSourceDirectories := Seq(baseDirectory.value / "src"),
+      Compile / unmanagedSourceDirectories ++=
+        (`scala-library-bootstrapped` / Compile / unmanagedSourceDirectories).value,
+      // NOTE: The only difference here is that we drop `-Werror` and semanticDB for now
+      Compile / scalacOptions :=  Seq("-deprecation", "-feature", "-unchecked", "-encoding", "UTF8", "-language:implicitConversions", "-nowarn"),
+      Compile / scalacOptions += "-Yno-stdlib-patches",
+      Compile / scalacOptions += "-scalajs",
+      // Packaging configuration of the stdlib
+      Compile / packageBin / publishArtifact := true,
+      Compile / packageDoc / publishArtifact := false,
+      Compile / packageSrc / publishArtifact := true,
+      // Only publish compilation artifacts, no test artifacts
+      Test    / publishArtifact := false,
+      // Do not allow to publish this project for now
+      publish / skip := false,
+      // Take into account the source files from the `library` folder
+      // but give the priority to the files in `library-js` that override files in `library`
+      Compile / sources := {
+        val files = (Compile / sources).value
+        val overwrittenSources =
+          (files ++ Seq(
+              baseDirectory.value / "src" / "scala" / "runtime" / "BoxesRunTime.java",
+              baseDirectory.value / "src" / "scala" / "math" / "ScalaNumber.java",
+          ))
+          .flatMap(_.relativeTo(baseDirectory.value / "src")).toSet
+
+        files.filterNot(file =>
+          file.relativeTo((`scala-library-bootstrapped` / baseDirectory).value / "src")
+            .exists(overwrittenSources.contains))
+
+      },
+      // Drop all the tasty files and the classfiles when packaging bu the scalajs exclusive classes
+      // More info here: https://github.com/scala-js/scala-js/issues/5217
+      Compile / packageBin / mappings := {
+        (Compile / packageBin / mappings).value.filter(file =>
+          file._2.endsWith(".sjsir")
+          || file._2.endsWith("UnitOps.tasty")         || file._2.endsWith("UnitOps.class") || file._2.endsWith("UnitOps$.class")
+          || file._2.endsWith("AnonFunctionXXL.tasty") || file._2.endsWith("AnonFunctionXXL.class"))
+      },
+      libraryDependencies += ("org.scala-js" %% "scalajs-library" % scalaJSVersion % Provided).cross(CrossVersion.for3Use2_13),
+      libraryDependencies += ("org.scala-js" % "scalajs-javalib" % scalaJSVersion),
+      // Project specific target folder. sbt doesn't like having two projects using the same target folder
+      target := target.value / "scala-library",
+      // we need to have the `scala-library` artifact in the classpath for `ScalaLibraryPlugin` to work
+      // this was the only way to not get the artifact evicted by sbt. Even a custom configuration didn't work
+      // NOTE: true is the default value, just making things clearer here
+      managedScalaInstance := true,
+      autoScalaLibrary := false,
+      // Configure the nonbootstrapped compiler
+      scalaInstance := {
+        val externalCompilerDeps = (`scala3-compiler-nonbootstrapped` / Compile / externalDependencyClasspath).value.map(_.data).toSet
+
+        // IMPORTANT: We need to use actual jars to form the ScalaInstance and not
+        // just directories containing classfiles because sbt maintains a cache of
+        // compiler instances. This cache is invalidated based on timestamps
+        // however this is only implemented on jars, directories are never
+        // invalidated.
+        val tastyCore = (`tasty-core-nonbootstrapped` / Compile / packageBin).value
+        val scalaLibrary = (`scala-library-nonbootstrapped` / Compile / packageBin).value
+        val scala3Interfaces = (`scala3-interfaces` / Compile / packageBin).value
+        val scala3Compiler = (`scala3-compiler-nonbootstrapped` / Compile / packageBin).value
+
+        Defaults.makeScalaInstance(
+          dottyNonBootstrappedVersion,
+          libraryJars     = Array(scalaLibrary),
+          allCompilerJars = Seq(tastyCore, scala3Interfaces, scala3Compiler) ++ externalCompilerDeps,
+          allDocJars      = Seq.empty,
+          state.value,
+          scalaInstanceTopLoader.value
+        )
+      },
+      scalaCompilerBridgeBinaryJar := {
+        Some((`scala3-sbt-bridge-nonbootstrapped` / Compile / packageBin).value)
+      },
+      // See https://stackoverflow.com/a/51416386
+      pomPostProcess := { (node: XmlNode) =>
+        new RuleTransformer(new RewriteRule {
+          override def transform(node: XmlNode): XmlNodeSeq = node match {
+            case e: Elem if e.label == "dependency" && e.child.exists(child => child.label == "artifactId" && child.text == "scalajs-library_2.13") =>
+              XmlNodeSeq.Empty
+            case _ => node
+        }
+      }).transform(node).head
+      },
+    )
+
+  /* Configuration of the org.scala-lang:scala3-library_sjs1_3:*.**.**-bootstrapped project */
+  lazy val `scala3-library-sjs` = project.in(file("library-js"))
+    .dependsOn(`scala-library-sjs`)
+    .settings(
+      name          := "scala3-library-sjs",
+      moduleName    := "scala3-library_sjs1",
+      version       := dottyVersion,
+      versionScheme := Some("semver-spec"),
+      // sbt defaults to scala 2.12.x and metals will report issues as it doesn't consider the project a scala 3 project
+      // (not the actual version we use to compile the project)
+      scalaVersion  := referenceVersion,
+      crossPaths    := true, // org.scala-lang:scala3-library_sjs1 has a crosspath
+      // Do not depend on the `org.scala-lang:scala3-library` automatically, we manually depend on `scala-library-bootstrapped`
+      autoScalaLibrary := false,
+      // Drop all the scala tools in this project, so we can never generate any bytecode, or documentation
+      managedScalaInstance := false,
+      // This Project only has a dependency to `org.scala-js:scalajs-scalalib:*.**.**-bootstrapped`
+      Compile / sources := Seq(),
+      Compile / resources := Seq(),
+      Test / sources := Seq(),
+      Test / resources := Seq(),
+      // Bridge the common task to call the ones of the actual library project
+      Compile / compile := (`scala-library-sjs` / Compile / compile).value,
+      Compile / doc     := (`scala-library-sjs` / Compile / doc).value,
+      Compile / run     := (`scala-library-sjs` / Compile / run).evaluated,
+      Test / compile := (`scala-library-sjs` / Test / compile).value,
+      Test / doc     := (`scala-library-sjs` / Test / doc).value,
+      Test / run     := (`scala-library-sjs` / Test / run).evaluated,
+      // Packaging configuration of the stdlib
+      Compile / packageBin / publishArtifact := true,
+      Compile / packageDoc / publishArtifact := false,
+      Compile / packageSrc / publishArtifact := true,
+      // Only publish compilation artifacts, no test artifacts
+      Test    / publishArtifact := false,
+      // Do not allow to publish this project for now
+      publish / skip := false,
+      // Project specific target folder. sbt doesn't like having two projects using the same target folder
+      target := target.value / "scala3-library",
+    )
+
   // ==============================================================================================
   // ===================================== TASTY CORE LIBRARY =====================================
   // ==============================================================================================

project/ScalaLibraryPlugin.scala

@@ -11,7 +11,11 @@ object ScalaLibraryPlugin extends AutoPlugin {
 
   override def trigger = noTrigger
 
+  private val scala2Version  = "2.13.16"
+  private val scalaJSVersion = "1.19.0"
+
   val fetchScala2ClassFiles = taskKey[(Set[File], File)]("Fetch the files to use that were compiled with Scala 2")
+  val fetchScala2SJSIR = taskKey[(Set[File], File)]("Fetch the .sjsir to use from Scala 2")
 
   override def projectSettings = Seq (
     fetchScala2ClassFiles := {
@@ -37,6 +41,31 @@ object ScalaLibraryPlugin extends AutoPlugin {
         (target ** "*.class").get.toSet
       } (Set(scalaLibraryBinaryJar)), target)
 
+    },
+    fetchScala2SJSIR := {
+      val stream = streams.value
+      val lm = dependencyResolution.value
+      val log = stream.log
+      val cache  = stream.cacheDirectory
+      val retrieveDir = cache / "scalajs-scalalib" / scalaVersion.value
+      val comp = lm.retrieve("org.scala-js" % "scalajs-scalalib_2.13" % s"$scala2Version+$scalaJSVersion", scalaModuleInfo = None, retrieveDir, log)
+          .fold(w => throw w.resolveException, identity)
+
+      println(comp(0))
+
+      val target = cache / "scala-library-sjsir"
+
+
+      if (!target.exists()) {
+        IO.createDirectory(target)
+      }
+
+      (FileFunction.cached(cache / "fetch-scala-library-sjsir", FilesInfo.lastModified, FilesInfo.exists) { _ =>
+        stream.log.info(s"Unpacking scalajs-scalalib binaries to persistent directory: ${target.getAbsolutePath}")
+        IO.unzip(comp(0), target)
+        (target ** "*.sjsir").get.toSet
+      } (Set(comp(0))), target)
+
     },
     (Compile / manipulateBytecode) := {
       val stream = streams.value