Skip to content

Commit ce37713

Browse files
lewisjkllewisjkl
authored
Merge pull request #273 from Jacoby6000/support-http-refs
Support http refs in Json Schema
2 parents bf37aaa + 79404a0 commit ce37713

33 files changed

+1697
-259
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ jobs:
3434
uses: actions/setup-java@v2
3535
with:
3636
distribution: adopt
37-
java-version: 11
37+
java-version: 21
3838

3939
- name: Check formatting
4040
run: ./mill -k --disable-ticker __.checkFormat

‎build.sc‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ trait CompilerCoreModule
4040
buildDeps.collectionsCompat
4141
)
4242

43-
object test extends ScalaTests with BaseMunitTests {
43+
object test extends BaseScalaTests with BaseMunitTests {
4444
def ivyDeps = super.ivyDeps() ++ Agg(buildDeps.smithy.diff)
4545
}
4646
}
@@ -60,7 +60,7 @@ trait JsonSchemaModule
6060
buildDeps.collectionsCompat
6161
)
6262

63-
object test extends ScalaTests with BaseMunitTests {
63+
object test extends BaseScalaTests with BaseMunitTests {
6464
def moduleDeps = super.moduleDeps ++ Seq(`compiler-core`().test)
6565

6666
def ivyDeps = super.ivyDeps() ++ Agg(
@@ -83,7 +83,7 @@ trait OpenApiModule
8383
def ivyDeps =
8484
buildDeps.swagger.parser
8585

86-
object test extends ScalaTests with BaseMunitTests {
86+
object test extends BaseScalaTests with BaseMunitTests {
8787
def moduleDeps = super.moduleDeps ++ Seq(`compiler-core`().test)
8888

8989
def ivyDeps = super.ivyDeps() ++ Agg(
@@ -125,7 +125,7 @@ object cli
125125
buildDeps.smithy.build
126126
)
127127

128-
object test extends ScalaTests with BaseMunitTests {
128+
object test extends BaseScalaTests with BaseMunitTests {
129129
def ivyDeps =
130130
super.ivyDeps() ++ Agg(buildDeps.lihaoyi.oslib, buildDeps.lihaoyi.ujson)
131131
}
@@ -171,7 +171,7 @@ object formatter extends BaseModule { outer =>
171171
override def ivyDeps = T { super.ivyDeps() ++ deps }
172172
override def millSourcePath = outer.millSourcePath
173173

174-
object test extends ScalaTests with BaseMunitTests {
174+
object test extends BaseScalaTests with BaseMunitTests {
175175
def ivyDeps = super.ivyDeps() ++ Agg(
176176
buildDeps.smithy.build,
177177
buildDeps.lihaoyi.oslib
@@ -328,7 +328,7 @@ trait ProtoModule
328328
)
329329

330330
def moduleDeps = Seq(traits, transitive())
331-
object test extends ScalaTests with BaseMunitTests with ScalaPBModule {
331+
object test extends BaseScalaTests with BaseMunitTests with ScalaPBModule {
332332
def ivyDeps = super.ivyDeps() ++ Agg(
333333
buildDeps.smithy.build,
334334
buildDeps.scalapb.compilerPlugin,
@@ -402,7 +402,7 @@ trait TransitiveModule
402402
buildDeps.smithy.build,
403403
buildDeps.collectionsCompat
404404
)
405-
object test extends ScalaTests with BaseMunitTests {
405+
object test extends BaseScalaTests with BaseMunitTests {
406406
def ivyDeps = super.ivyDeps() ++ Agg(
407407
buildDeps.scalaJavaCompat
408408
)

‎buildSetup.sc‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,17 @@ trait BaseScalaModule extends ScalaModule with BaseModule with ScalafmtModule {
128128
super.scalacPluginIvyDeps() ++ plugins
129129
}
130130

131+
trait BaseScalaTests extends ScalaTests {
132+
override def scalacOptions = T {
133+
// Don't force target bytecode version in tests.
134+
// Our published artifacts target java 11, but the tests need some java 18 apis
135+
super.scalacOptions()
136+
.filterNot(_.startsWith("-release"))
137+
.filterNot(_.startsWith("-java-output-version"))
138+
}
139+
140+
}
141+
131142
def scalacOptions = T {
132143
super.scalacOptions() ++ scalacOptionsFor(scalaVersion())
133144
}
@@ -183,6 +194,8 @@ case class ScalacOption(
183194

184195
// format: off
185196
private val allScalacOptions = Seq(
197+
ScalacOption("-release:11", isSupported = _ < v300), // Emit Java 11 compat bytecode
198+
ScalacOption("-java-output-version:11", isSupported = _ >= v300), // Emit Java 11 compat bytecode
186199
ScalacOption("-Xsource:3", isSupported = version => v211 <= version && version < v300), // Treat compiler input as Scala source for the specified version, see scala/bug#8126.
187200
ScalacOption("-deprecation", isSupported = version => version < v213 || v300 <= version), // Emit warning and location for usages of deprecated APIs. Not really removed but deprecated in 2.13.
188201
ScalacOption("-explaintypes", isSupported = _ < v300), // Explain type errors in more detail.
@@ -256,6 +269,6 @@ def scalacOptionsFor(scalaVersion: String): Seq[String] = {
256269
val commonOpts = Seq("-encoding", "utf8")
257270
val scalaVer = ScalaVersion(scalaVersion)
258271
val versionedOpts = allScalacOptions.filter(_.isSupported(scalaVer)).map(_.name)
259-
272+
260273
commonOpts ++ versionedOpts
261274
}

‎modules/cli/src/Main.scala‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,9 @@ object Main
6868
opts.validateOutput,
6969
opts.useEnumTraitSyntax,
7070
opts.outputJson,
71-
opts.debug
71+
opts.debug,
72+
opts.allowedRemoteBaseURLs,
73+
opts.namespaceRemaps
7274
)
7375
SmithyBuildJsonWriter.writeDefault(opts.outputPath, opts.force)
7476

‎modules/cli/src/opts/CommonArguments.scala‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
/* Copyright 2022 Disney Streaming
2+
*
3+
* Licensed under the Tomorrow Open Source Technology License, Version 1.0 (the "License");
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
*
7+
* https://disneystreaming.github.io/TOST-1.0.txt
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
package smithytranslate.cli.opts
17+
18+
import com.monovore.decline._
19+
import cats.data.NonEmptyList
20+
import cats.data.ValidatedNel
21+
import cats.data.Validated
22+
import cats.implicits._
23+
import cats.data.Chain
24+
import cats.data.NonEmptyChain
25+
26+
object CommonArguments {
27+
case class NamespaceMapping(
28+
original: NonEmptyChain[String],
29+
remapped: Chain[String]
30+
)
31+
implicit val namespaceMappingArgument: Argument[NamespaceMapping] =
32+
new Argument[NamespaceMapping] {
33+
val defaultMetavar: String = "source.name.space:target.name.space"
34+
def read(string: String): ValidatedNel[String, NamespaceMapping] = {
35+
val result: Either[String, NamespaceMapping] =
36+
string.split(':') match {
37+
case Array(from, to) =>
38+
val sourceNs =
39+
NonEmptyChain.fromSeq(from.split('.').toList.filter(_.nonEmpty))
40+
val targetNs =
41+
Chain.fromSeq(to.split('.').toList.filter(_.nonEmpty))
42+
43+
(sourceNs, targetNs) match {
44+
case (None, _) =>
45+
Left("Source namespace must not be empty.")
46+
47+
case (Some(f), t) =>
48+
Right(NamespaceMapping(f, t))
49+
}
50+
case _ =>
51+
Left(
52+
s"""Invalid namespace remapping.
53+
|Expected input to be formatted as 'my.source.namespace:my.target.namespace'
54+
|got: '$string'""".stripMargin
55+
)
56+
}
57+
58+
result.toValidatedNel
59+
}
60+
}
61+
}

‎modules/cli/src/opts/OpenAPIJsonSchemaOpts.scala‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,11 @@ package smithytranslate.cli.opts
1818
import com.monovore.decline.*
1919
import cats.syntax.all.*
2020
import cats.data.NonEmptyList
21+
import cats.data.NonEmptyChain
22+
import cats.data.ValidatedNel
2123
import smithytranslate.cli.opts.SmithyTranslateCommand.OpenApiTranslate
24+
import cats.data.Chain
25+
import smithytranslate.cli.opts.CommonArguments.NamespaceMapping
2226

2327
final case class OpenAPIJsonSchemaOpts(
2428
isOpenapi: Boolean,
@@ -30,7 +34,9 @@ final case class OpenAPIJsonSchemaOpts(
3034
useEnumTraitSyntax: Boolean,
3135
outputJson: Boolean,
3236
debug: Boolean,
33-
force: Boolean
37+
force: Boolean,
38+
allowedRemoteBaseURLs: Set[String],
39+
namespaceRemaps: Map[NonEmptyChain[String], Chain[String]]
3440
)
3541

3642
object OpenAPIJsonSchemaOpts {
@@ -84,6 +90,26 @@ object OpenAPIJsonSchemaOpts {
8490
)
8591
.orFalse
8692

93+
private val allowedRemoteBaseURLs: Opts[Set[String]] = Opts
94+
.options[String](
95+
"allow-remote-base-url",
96+
help =
97+
"A base path for allowed remote references, e.g. 'https://example.com/schemas/'"
98+
)
99+
.map(_.toList.toSet)
100+
.withDefault(Set.empty)
101+
102+
private val namespaceRemaps: Opts[Map[NonEmptyChain[String], Chain[String]]] =
103+
Opts
104+
.options[NamespaceMapping](
105+
"remap-namespace",
106+
help =
107+
"""A namespace remapping rule in the form of 'from1.from2:to1.to2', which remaps the 'from' prefix to the 'to' prefix.
108+
|A prefix can be stripped by specifying no replacement. Eg: 'prefix.to.remove:'""".stripMargin
109+
)
110+
.map(mappings => mappings.map(m => m.original -> m.remapped).toList.toMap)
111+
.withDefault(Map.empty)
112+
87113
private def getOpts(isOpenapi: Boolean) =
88114
(
89115
Opts(isOpenapi),
@@ -95,7 +121,9 @@ object OpenAPIJsonSchemaOpts {
95121
useEnumTraitSyntax,
96122
outputJson,
97123
debug,
98-
force
124+
force,
125+
allowedRemoteBaseURLs,
126+
namespaceRemaps
99127
).mapN(OpenAPIJsonSchemaOpts.apply)
100128

101129
private val openApiToSmithyCmd = Command(

‎modules/cli/test/src/NamespaceMappingArgumentSpec.scala‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/* Copyright 2022 Disney Streaming
2+
*
3+
* Licensed under the Tomorrow Open Source Technology License, Version 1.0 (the "License");
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
*
7+
* https://disneystreaming.github.io/TOST-1.0.txt
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
package smithytranslate.cli
17+
18+
import smithytranslate.cli.opts.CommonArguments
19+
import cats.data.Validated
20+
import cats.data.Chain
21+
import cats.data.NonEmptyChain
22+
23+
class NamespaceMappingArgumentSpec extends munit.FunSuite {
24+
test("NamespaceMapping Argument should - Parse a valid source:target") {
25+
CommonArguments.namespaceMappingArgument.read("a.b.c:d.e.f") match {
26+
case Validated.Valid(mapping) =>
27+
assertEquals(
28+
mapping,
29+
CommonArguments.NamespaceMapping(
30+
original = NonEmptyChain("a", "b", "c"),
31+
remapped = Chain("d", "e", "f")
32+
)
33+
)
34+
case Validated.Invalid(errors) =>
35+
fail(s"Failed to parse valid input: ${errors.toList.mkString(", ")}")
36+
}
37+
}
38+
39+
test("NamespaceMapping Argument should - Parse a valid source:target with single part namespaces") {
40+
CommonArguments.namespaceMappingArgument.read("a:b") match {
41+
case Validated.Valid(mapping) =>
42+
assertEquals(
43+
mapping,
44+
CommonArguments.NamespaceMapping(
45+
original = NonEmptyChain("a"),
46+
remapped = Chain("b")
47+
)
48+
)
49+
case Validated.Invalid(errors) =>
50+
fail(s"Failed to parse valid input: ${errors.toList.mkString(", ")}")
51+
}
52+
}
53+
54+
test("NamespaceMapping Argument should - successfully parse with an empty target") {
55+
CommonArguments.namespaceMappingArgument.read("a.b.c:") match {
56+
case Validated.Valid(mapping) =>
57+
assertEquals(
58+
mapping,
59+
CommonArguments.NamespaceMapping(
60+
original = NonEmptyChain("a", "b", "c"),
61+
remapped = Chain()
62+
)
63+
)
64+
case Validated.Invalid(_) =>
65+
}
66+
}
67+
68+
test("NamespaceMapping Argument should - fail to parse an invalid input missing colon") {
69+
CommonArguments.namespaceMappingArgument.read("a.b.c") match {
70+
case Validated.Valid(mapping) => fail(s"Parsed invalid input successfully: $mapping")
71+
case Validated.Invalid(_) =>
72+
}
73+
}
74+
75+
test("NamespaceMapping Argument should - fail to parse an invalid input with empty source") {
76+
CommonArguments.namespaceMappingArgument.read(":a.b.c") match {
77+
case Validated.Valid(mapping) => fail(s"Parsed invalid input successfully: $mapping")
78+
case Validated.Invalid(_) =>
79+
}
80+
}
81+
}

‎modules/compiler-core/src/ToSmithyCompilerOptions.scala‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,37 @@
1616
package smithytranslate.compiler
1717

1818
import software.amazon.smithy.build.ProjectionTransformer
19+
import cats.data.NonEmptyChain
20+
import cats.data.Chain
1921

2022
final case class ToSmithyCompilerOptions(
2123
useVerboseNames: Boolean,
2224
validateInput: Boolean,
2325
validateOutput: Boolean,
2426
transformers: List[ProjectionTransformer],
2527
useEnumTraitSyntax: Boolean,
26-
debug: Boolean
28+
debug: Boolean,
29+
allowedRemoteBaseURLs: Set[String],
30+
namespaceRemaps: Map[NonEmptyChain[String], Chain[String]]
2731
)
32+
33+
object ToSmithyCompilerOptions {
34+
def apply(
35+
useVerboseNames: Boolean,
36+
validateInput: Boolean,
37+
validateOutput: Boolean,
38+
transformers: List[ProjectionTransformer],
39+
useEnumTraitSyntax: Boolean,
40+
debug: Boolean
41+
): ToSmithyCompilerOptions =
42+
ToSmithyCompilerOptions(
43+
useVerboseNames,
44+
validateInput,
45+
validateOutput,
46+
transformers,
47+
useEnumTraitSyntax,
48+
debug,
49+
Set.empty,
50+
Map.empty
51+
)
52+
}

‎modules/compiler-core/src/ToSmithyError.scala‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import scala.util.control.NoStackTrace
1919
import cats.data.NonEmptyChain
2020
import cats.syntax.all._
2121
import software.amazon.smithy.model.validation.ValidationEvent
22+
import java.net.URI
2223

2324
sealed trait ToSmithyError extends Throwable {
2425
// Force all subtypes to provide a message
@@ -31,8 +32,16 @@ object ToSmithyError {
3132
implicit val order: cats.Order[ToSmithyError] = cats.Order.by(_.getMessage())
3233

3334
final case class Restriction(message: String) extends ToSmithyError
35+
36+
final case class ProcessingError(message: String, errorCause: Option[Throwable] = None) extends ToSmithyError {
37+
override def getCause(): Throwable = errorCause.orNull
38+
}
39+
40+
final case class HttpError(uri: URI, refStack: List[String], error: Throwable) extends ToSmithyError {
41+
override def message: String = "Failed to fetch remote schema from " + uri.toString + ". Error: " + error.getMessage
42+
override def getCause(): Throwable = error
43+
}
3444

35-
final case class ProcessingError(message: String) extends ToSmithyError
3645

3746
final case class SmithyValidationFailed(
3847
smithyValidationEvents: List[ValidationEvent]
@@ -55,5 +64,4 @@ object ToSmithyError {
5564
s"Unable to parse openapi file located at ${namespace.mkString_("/")} with errors: ${errorMessages
5665
.mkString(", ")}"
5766
}
58-
5967
}

0 commit comments

Comments
 (0)