Support http refs in Json Schema

.github/workflows/ci.yml

@@ -34,7 +34,7 @@ jobs:
         uses: actions/setup-java@v2
         with:
             distribution: adopt
-            java-version: 11
+            java-version: 21
 
       - name: Check formatting
         run: ./mill -k --disable-ticker __.checkFormat

build.sc

@@ -40,7 +40,7 @@ trait CompilerCoreModule
     buildDeps.collectionsCompat
   )
 
-  object test extends ScalaTests with BaseMunitTests {
+  object test extends BaseScalaTests with BaseMunitTests {
     def ivyDeps = super.ivyDeps() ++ Agg(buildDeps.smithy.diff)
   }
 }
@@ -60,7 +60,7 @@ trait JsonSchemaModule
     buildDeps.collectionsCompat
   )
 
-  object test extends ScalaTests with BaseMunitTests {
+  object test extends BaseScalaTests with BaseMunitTests {
     def moduleDeps = super.moduleDeps ++ Seq(`compiler-core`().test)
 
     def ivyDeps = super.ivyDeps() ++ Agg(
@@ -83,7 +83,7 @@ trait OpenApiModule
   def ivyDeps =
     buildDeps.swagger.parser
 
-  object test extends ScalaTests with BaseMunitTests {
+  object test extends BaseScalaTests with BaseMunitTests {
     def moduleDeps = super.moduleDeps ++ Seq(`compiler-core`().test)
 
     def ivyDeps = super.ivyDeps() ++ Agg(
@@ -125,7 +125,7 @@ object cli
     buildDeps.smithy.build
   )
 
-  object test extends ScalaTests with BaseMunitTests {
+  object test extends BaseScalaTests with BaseMunitTests {
     def ivyDeps =
       super.ivyDeps() ++ Agg(buildDeps.lihaoyi.oslib, buildDeps.lihaoyi.ujson)
   }
@@ -171,7 +171,7 @@ object formatter extends BaseModule { outer =>
     override def ivyDeps = T { super.ivyDeps() ++ deps }
     override def millSourcePath = outer.millSourcePath
 
-    object test extends ScalaTests with BaseMunitTests {
+    object test extends BaseScalaTests with BaseMunitTests {
       def ivyDeps = super.ivyDeps() ++ Agg(
         buildDeps.smithy.build,
         buildDeps.lihaoyi.oslib
@@ -328,7 +328,7 @@ trait ProtoModule
   )
 
   def moduleDeps = Seq(traits, transitive())
-  object test extends ScalaTests with BaseMunitTests with ScalaPBModule {
+  object test extends BaseScalaTests with BaseMunitTests with ScalaPBModule {
     def ivyDeps = super.ivyDeps() ++ Agg(
       buildDeps.smithy.build,
       buildDeps.scalapb.compilerPlugin,
@@ -402,7 +402,7 @@ trait TransitiveModule
     buildDeps.smithy.build,
     buildDeps.collectionsCompat
   )
-  object test extends ScalaTests with BaseMunitTests {
+  object test extends BaseScalaTests with BaseMunitTests {
     def ivyDeps = super.ivyDeps() ++ Agg(
       buildDeps.scalaJavaCompat
     )

buildSetup.sc

@@ -128,6 +128,17 @@ trait BaseScalaModule extends ScalaModule with BaseModule with ScalafmtModule {
     super.scalacPluginIvyDeps() ++ plugins
   }
 
+  trait BaseScalaTests extends ScalaTests {
+    override def scalacOptions = T {
+      // Don't force target bytecode version in tests.
+      // Our published artifacts target java 11, but the tests need some java 18 apis
+      super.scalacOptions()
+        .filterNot(_.startsWith("-release"))
+        .filterNot(_.startsWith("-java-output-version"))
+    }
+
+  }
+
   def scalacOptions = T {
     super.scalacOptions() ++ scalacOptionsFor(scalaVersion())
   }
@@ -183,6 +194,8 @@ case class ScalacOption(
 
 // format: off
 private val allScalacOptions = Seq(
+  ScalacOption("-release:11", isSupported = _ < v300),                                                                     // Emit Java 11 compat bytecode
+  ScalacOption("-java-output-version:11", isSupported = _ >= v300),                                                        // Emit Java 11 compat bytecode
   ScalacOption("-Xsource:3", isSupported = version => v211 <= version && version < v300),                                  // Treat compiler input as Scala source for the specified version, see scala/bug#8126.
   ScalacOption("-deprecation", isSupported = version => version < v213 || v300 <= version),                                // Emit warning and location for usages of deprecated APIs. Not really removed but deprecated in 2.13.
   ScalacOption("-explaintypes", isSupported = _ < v300),                                                                   // Explain type errors in more detail.
@@ -256,6 +269,6 @@ def scalacOptionsFor(scalaVersion: String): Seq[String] = {
   val commonOpts = Seq("-encoding", "utf8")
   val scalaVer = ScalaVersion(scalaVersion)
   val versionedOpts = allScalacOptions.filter(_.isSupported(scalaVer)).map(_.name)
-
+  
   commonOpts ++ versionedOpts
 }

modules/cli/src/Main.scala

@@ -68,7 +68,9 @@ object Main
                 opts.validateOutput,
                 opts.useEnumTraitSyntax,
                 opts.outputJson,
-                opts.debug
+                opts.debug,
+                opts.allowedRemoteBaseURLs,
+                opts.namespaceRemaps
               )
             SmithyBuildJsonWriter.writeDefault(opts.outputPath, opts.force)
 

modules/cli/src/opts/CommonArguments.scala

@@ -0,0 +1,61 @@
+/* Copyright 2022 Disney Streaming
+ *
+ * Licensed under the Tomorrow Open Source Technology License, Version 1.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://disneystreaming.github.io/TOST-1.0.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package smithytranslate.cli.opts
+
+import com.monovore.decline._
+import cats.data.NonEmptyList
+import cats.data.ValidatedNel
+import cats.data.Validated
+import cats.implicits._
+import cats.data.Chain
+import cats.data.NonEmptyChain
+
+object CommonArguments {
+  case class NamespaceMapping(
+      original: NonEmptyChain[String],
+      remapped: Chain[String]
+  )
+  implicit val namespaceMappingArgument: Argument[NamespaceMapping] =
+    new Argument[NamespaceMapping] {
+      val defaultMetavar: String = "source.name.space:target.name.space"
+      def read(string: String): ValidatedNel[String, NamespaceMapping] = {
+        val result: Either[String, NamespaceMapping] =
+          string.split(':') match {
+            case Array(from, to) =>
+              val sourceNs =
+                NonEmptyChain.fromSeq(from.split('.').toList.filter(_.nonEmpty))
+              val targetNs =
+                Chain.fromSeq(to.split('.').toList.filter(_.nonEmpty))
+
+              (sourceNs, targetNs) match {
+                case (None, _) =>
+                  Left("Source namespace must not be empty.")
+
+                case (Some(f), t) =>
+                  Right(NamespaceMapping(f, t))
+              }
+            case _ =>
+              Left(
+                s"""Invalid namespace remapping. 
+                   |Expected input to be formatted as 'my.source.namespace:my.target.namespace'
+                   |got: '$string'""".stripMargin
+              )
+          }
+
+        result.toValidatedNel
+      }
+    }
+}

modules/cli/src/opts/OpenAPIJsonSchemaOpts.scala

@@ -18,7 +18,11 @@ package smithytranslate.cli.opts
 import com.monovore.decline.*
 import cats.syntax.all.*
 import cats.data.NonEmptyList
+import cats.data.NonEmptyChain
+import cats.data.ValidatedNel
 import smithytranslate.cli.opts.SmithyTranslateCommand.OpenApiTranslate
+import cats.data.Chain
+import smithytranslate.cli.opts.CommonArguments.NamespaceMapping
 
 final case class OpenAPIJsonSchemaOpts(
     isOpenapi: Boolean,
@@ -30,7 +34,9 @@ final case class OpenAPIJsonSchemaOpts(
     useEnumTraitSyntax: Boolean,
     outputJson: Boolean,
     debug: Boolean,
-    force: Boolean
+    force: Boolean,
+    allowedRemoteBaseURLs: Set[String],
+    namespaceRemaps: Map[NonEmptyChain[String], Chain[String]]
 )
 
 object OpenAPIJsonSchemaOpts {
@@ -84,6 +90,26 @@ object OpenAPIJsonSchemaOpts {
     )
     .orFalse
 
+  private val allowedRemoteBaseURLs: Opts[Set[String]] = Opts
+    .options[String](
+      "allow-remote-base-url",
+      help =
+        "A base path for allowed remote references, e.g. 'https://example.com/schemas/'"
+    )
+    .map(_.toList.toSet)
+    .withDefault(Set.empty)
+
+  private val namespaceRemaps: Opts[Map[NonEmptyChain[String], Chain[String]]] =
+    Opts
+      .options[NamespaceMapping](
+        "remap-namespace",
+        help =
+          """A namespace remapping rule in the form of 'from1.from2:to1.to2', which remaps the 'from' prefix to the 'to' prefix.
+            |A prefix can be stripped by specifying no replacement. Eg: 'prefix.to.remove:'""".stripMargin
+      )
+      .map(mappings => mappings.map(m => m.original -> m.remapped).toList.toMap)
+      .withDefault(Map.empty)
+
   private def getOpts(isOpenapi: Boolean) =
     (
       Opts(isOpenapi),
@@ -95,7 +121,9 @@ object OpenAPIJsonSchemaOpts {
       useEnumTraitSyntax,
       outputJson,
       debug,
-      force
+      force,
+      allowedRemoteBaseURLs,
+      namespaceRemaps
     ).mapN(OpenAPIJsonSchemaOpts.apply)
 
   private val openApiToSmithyCmd = Command(

modules/cli/test/src/NamespaceMappingArgumentSpec.scala

@@ -0,0 +1,81 @@
+/* Copyright 2022 Disney Streaming
+ *
+ * Licensed under the Tomorrow Open Source Technology License, Version 1.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://disneystreaming.github.io/TOST-1.0.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package smithytranslate.cli
+
+import smithytranslate.cli.opts.CommonArguments
+import cats.data.Validated
+import cats.data.Chain
+import cats.data.NonEmptyChain
+
+class NamespaceMappingArgumentSpec extends munit.FunSuite {
+  test("NamespaceMapping Argument should - Parse a valid source:target") {
+    CommonArguments.namespaceMappingArgument.read("a.b.c:d.e.f") match {
+      case Validated.Valid(mapping) =>
+        assertEquals(
+          mapping,
+          CommonArguments.NamespaceMapping(
+            original = NonEmptyChain("a", "b", "c"),
+            remapped = Chain("d", "e", "f")
+          )
+        )
+      case Validated.Invalid(errors) =>
+        fail(s"Failed to parse valid input: ${errors.toList.mkString(", ")}")
+    }
+  }
+
+  test("NamespaceMapping Argument should - Parse a valid source:target with single part namespaces") {
+    CommonArguments.namespaceMappingArgument.read("a:b") match {
+      case Validated.Valid(mapping) =>
+        assertEquals(
+          mapping,
+          CommonArguments.NamespaceMapping(
+            original = NonEmptyChain("a"),
+            remapped = Chain("b")
+          )
+        )
+      case Validated.Invalid(errors) =>
+        fail(s"Failed to parse valid input: ${errors.toList.mkString(", ")}")
+    }
+  }
+
+  test("NamespaceMapping Argument should - successfully parse with an empty target") {
+    CommonArguments.namespaceMappingArgument.read("a.b.c:") match {
+      case Validated.Valid(mapping) =>
+        assertEquals(
+          mapping,
+          CommonArguments.NamespaceMapping(
+            original = NonEmptyChain("a", "b", "c"),
+            remapped = Chain()
+          )
+        )
+      case Validated.Invalid(_) =>
+    }
+  }
+
+  test("NamespaceMapping Argument should - fail to parse an invalid input missing colon") {
+    CommonArguments.namespaceMappingArgument.read("a.b.c") match {
+      case Validated.Valid(mapping) => fail(s"Parsed invalid input successfully: $mapping")
+      case Validated.Invalid(_) =>
+    }
+  }
+
+  test("NamespaceMapping Argument should - fail to parse an invalid input with empty source") {
+    CommonArguments.namespaceMappingArgument.read(":a.b.c") match {
+      case Validated.Valid(mapping) => fail(s"Parsed invalid input successfully: $mapping")
+      case Validated.Invalid(_) => 
+    }
+  }
+}

modules/compiler-core/src/ToSmithyCompilerOptions.scala

@@ -16,12 +16,37 @@
 package smithytranslate.compiler
 
 import software.amazon.smithy.build.ProjectionTransformer
+import cats.data.NonEmptyChain
+import cats.data.Chain
 
 final case class ToSmithyCompilerOptions(
     useVerboseNames: Boolean,
     validateInput: Boolean,
     validateOutput: Boolean,
     transformers: List[ProjectionTransformer],
     useEnumTraitSyntax: Boolean,
-    debug: Boolean
+    debug: Boolean,
+    allowedRemoteBaseURLs: Set[String],
+    namespaceRemaps: Map[NonEmptyChain[String], Chain[String]]
 )
+
+object ToSmithyCompilerOptions {
+  def apply(
+      useVerboseNames: Boolean,
+      validateInput: Boolean,
+      validateOutput: Boolean,
+      transformers: List[ProjectionTransformer],
+      useEnumTraitSyntax: Boolean,
+      debug: Boolean
+  ): ToSmithyCompilerOptions =
+    ToSmithyCompilerOptions(
+      useVerboseNames,
+      validateInput,
+      validateOutput,
+      transformers,
+      useEnumTraitSyntax,
+      debug,
+      Set.empty,
+      Map.empty
+    )
+}

modules/compiler-core/src/ToSmithyError.scala

@@ -19,6 +19,7 @@ import scala.util.control.NoStackTrace
 import cats.data.NonEmptyChain
 import cats.syntax.all._
 import software.amazon.smithy.model.validation.ValidationEvent
+import java.net.URI
 
 sealed trait ToSmithyError extends Throwable {
   // Force all subtypes to provide a message
@@ -31,8 +32,16 @@ object ToSmithyError {
   implicit val order: cats.Order[ToSmithyError] = cats.Order.by(_.getMessage())
 
   final case class Restriction(message: String) extends ToSmithyError
+
+  final case class ProcessingError(message: String, errorCause: Option[Throwable] = None) extends ToSmithyError {
+    override def getCause(): Throwable = errorCause.orNull
+  }
+
+  final case class HttpError(uri: URI, refStack: List[String], error: Throwable) extends ToSmithyError {
+    override def message: String = "Failed to fetch remote schema from " + uri.toString + ". Error: " + error.getMessage
+    override def getCause(): Throwable = error
+  }
 
-  final case class ProcessingError(message: String) extends ToSmithyError
 
   final case class SmithyValidationFailed(
       smithyValidationEvents: List[ValidationEvent]
@@ -55,5 +64,4 @@ object ToSmithyError {
       s"Unable to parse openapi file located at ${namespace.mkString_("/")} with errors: ${errorMessages
           .mkString(", ")}"
   }
-
 }