Skip to content

Commit 7121ca3

Browse files
manonthegithubmanonthegithub
committed
#39, resolve relative uris, but save them as relative
1 parent 5ac7c47 commit 7121ca3

File tree

6 files changed

+91
-40
lines changed

6 files changed

+91
-40
lines changed

‎src/main/scala/org/dbpedia/databus/ApiImpl.scala‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,13 @@ class ApiImpl(config: Config) extends DatabusApi {
3535

3636
def stop() = JenaSystem.shutdown()
3737

38-
38+
// todo NOTICE! this may fail with relative URIS, NOT TESTED!
3939
override def dataidSubgraph(body: String)(request: HttpServletRequest): Try[String] =
4040
readModel(
4141
body.getBytes,
4242
defaultLang,
4343
contextUrl(body.getBytes, defaultLang)
44-
.map(jenaJsonLdContextWithFallbackForLocalhost(_, request.getRemoteHost).get)
44+
.map(jenaJsonLdContextWithFallbackForLocalhost(_, request.getRemoteHost, None).get)
4545
)
4646
.flatMap(m => Tractate.extract(m._1.getGraph, TractateV1.Version))
4747
.map(_.stringForSigning)
@@ -67,8 +67,8 @@ class ApiImpl(config: Config) extends DatabusApi {
6767
)
6868
}
6969

70-
override def getFile(repo: String, path: String)(request: HttpServletRequest): Try[String] =
71-
readFile(repo, path)(request)
70+
override def getFile(repo: String, path: String, prefix: Option[String])(request: HttpServletRequest): Try[String] =
71+
readFile(repo, path, prefix)(request)
7272

7373

7474
override def saveFile(repo: String,
@@ -86,12 +86,12 @@ class ApiImpl(config: Config) extends DatabusApi {
8686
.getOrElse("")
8787
val lang = mapContentType(ct, defaultLang)
8888
val ctxU = contextUrl(body.getBytes, lang)
89-
val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, request.getRemoteHost).get)
89+
val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, request.getRemoteHost, Some(graphId)).get)
9090
validateEmail(author_email).flatMap(email =>
9191
readModel(body.getBytes, lang, ctx)
9292
.flatMap(model => {
9393
saveToVirtuoso(model._1, graphId)({
94-
graphToBytes(model._1.getGraph, defaultLang, ctxU)
94+
graphToBytes(model._1.getGraph, defaultLang, ctx, ctxU)
9595
.flatMap(a => saveFiles(
9696
repo,
9797
Map(
@@ -118,18 +118,20 @@ class ApiImpl(config: Config) extends DatabusApi {
118118
val lang = getLangFromAcceptHeader(request)
119119
setResponseHeaders(Map("Content-Type" -> lang.getContentType.toHeaderString))(request)
120120
val ctxU = contextUrl(dataid.getBytes, lang)
121-
val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, request.getRemoteHost).get)
121+
val ctx = ctxU.map(cu =>
122+
jenaJsonLdContextWithFallbackForLocalhost(cu, request.getRemoteHost, None).get)
122123

123124
val shaclU = contextUrl(shacl.getBytes, RdfConversions.DefaultShaclLang)
124-
val shaclCtx = shaclU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, request.getRemoteHost).get)
125+
val shaclCtx = shaclU.map(cu =>
126+
jenaJsonLdContextWithFallbackForLocalhost(cu, request.getRemoteHost, None).get)
125127

126128
RdfConversions.validateWithShacl(
127129
dataid.getBytes,
128130
shacl.getBytes,
129131
ctx,
130132
shaclCtx,
131133
defaultLang
132-
).flatMap(r => RdfConversions.graphToBytes(r.getGraph, lang, None))
134+
).flatMap(r => RdfConversions.graphToBytes(r.getGraph, lang, shaclCtx, None))
133135
.map(new String(_))
134136
}
135137

@@ -177,21 +179,22 @@ class ApiImpl(config: Config) extends DatabusApi {
177179
s"${url.getProtocol}://${url.getHost}:${url.getPort}${config.defaultGraphIdPrefix}/"
178180
}
179181

180-
private def readFile(username: String, path: String)(request: HttpServletRequest): Try[String] = {
182+
private def readFile(username: String, path: String, prefix: Option[String])(request: HttpServletRequest): Try[String] = {
181183
val p = gitPath(path)
182184
val lang = getLangFromAcceptHeader(request)
185+
val graphId = generateGraphId(prefix.getOrElse(getPrefix(request)), username, path)
183186
setResponseHeaders(Map("Content-Type" -> lang.getContentType.toHeaderString))(request)
184187
client.readFile(username, p)
185188
.flatMap(body => {
186189
val ctxUri = contextUrl(body, defaultLang)
190+
val ctx = ctxUri.map(jenaJsonLdContextWithFallbackForLocalhost(_, request.getRemoteHost, Some(graphId)).get)
187191
readModel(
188192
body,
189193
defaultLang,
190-
contextUrl(body, defaultLang)
191-
.map(jenaJsonLdContextWithFallbackForLocalhost(_, request.getRemoteHost).get)
194+
ctx
192195
)
193196
.flatMap(m =>
194-
graphToBytes(m._1.getGraph, lang, ctxUri)
197+
graphToBytes(m._1.getGraph, lang, ctx, ctxUri)
195198
)
196199
})
197200
.map(new String(_))

‎src/main/scala/org/dbpedia/databus/SparqlClient.scala‎

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,10 @@ import org.apache.jena.rdf.model.{Model, ModelFactory}
1414
import org.apache.jena.riot.lang.LangJSONLD10
1515
import org.apache.jena.riot.system.{ErrorHandler, ErrorHandlerFactory, StreamRDFLib}
1616
import org.apache.jena.riot.writer.JsonLD10Writer
17-
import org.apache.jena.riot.{Lang, RDFDataMgr, RDFFormat, RDFLanguages, RDFParserBuilder, RDFWriter, RIOT}
17+
import org.apache.jena.riot.{Lang, RDFDataMgr, RDFFormat, RDFLanguages, RDFParser, RDFWriter, RIOT}
1818
import org.apache.jena.shacl.{ShaclValidator, Shapes, ValidationReport}
1919
import org.apache.jena.sparql.util
20+
import org.apache.jena.sparql.util.Context
2021
import org.dbpedia.databus.ApiImpl.Config
2122
import org.slf4j.LoggerFactory
2223
import sttp.client3.{DigestAuthenticationBackend, HttpURLConnectionBackend, basicRequest}
@@ -163,36 +164,34 @@ object RdfConversions {
163164

164165
def readModel(data: Array[Byte], lang: Lang, context: Option[util.Context]): Try[(Model, List[Warning])] = Try {
165166
val model = ModelFactory.createDefaultModel()
167+
val eh = newErrorHandlerWithWarnings
166168
val dataStream = new ByteArrayInputStream(data)
167169
val dest = StreamRDFLib.graph(model.getGraph)
168-
val parser = RDFParserBuilder.create()
170+
171+
val parser = RDFParser.create()
169172
.source(dataStream)
170-
.base(null)
173+
.errorHandler(eh)
171174
.lang(lang)
172175

173-
val eh = newErrorHandlerWithWarnings
174-
parser.errorHandler(eh)
175-
176176
context.foreach(cs =>
177177
parser.context(cs))
178178

179179
parser.parse(dest)
180180
(model, eh.warningsList)
181181
}
182182

183-
def graphToBytes(model: Graph, outputLang: Lang, context: Option[URL]): Try[Array[Byte]] = Try {
183+
def graphToBytes(model: Graph, outputLang: Lang, context: Option[Context], contextURL: Option[URL]): Try[Array[Byte]] = Try {
184184
val str = new ByteArrayOutputStream()
185-
val builder = RDFWriter.create.format(langToFormat(outputLang))
185+
val builder = RDFWriter.create()
186186
.source(model)
187+
.format(langToFormat(outputLang))
187188

188-
context.foreach(ctx => {
189-
val jctx = jenaContext(CachingContext.parse(ctx.toString))
190-
builder.context(jctx)
191-
builder.set(JsonLD10Writer.JSONLD_CONTEXT_SUBSTITUTION, new JsonString(ctx.toString))
192-
})
189+
context.foreach(ctx =>
190+
builder.context(ctx))
191+
contextURL.foreach(ctx =>
192+
builder.set(JsonLD10Writer.JSONLD_CONTEXT_SUBSTITUTION, new JsonString(ctx.toString)))
193193

194194
builder
195-
.build()
196195
.output(str)
197196
str.toByteArray
198197
}
@@ -224,8 +223,8 @@ object RdfConversions {
224223
def langToFormat(lang: Lang): RDFFormat = lang match {
225224
case RDFLanguages.TURTLE => RDFFormat.TURTLE_PRETTY
226225
case RDFLanguages.TTL => RDFFormat.TTL
227-
case RDFLanguages.JSONLD => RDFFormat.JSONLD10
228-
case RDFLanguages.JSONLD10 => RDFFormat.JSONLD10
226+
case RDFLanguages.JSONLD => RDFFormat.JSONLD10_COMPACT_PRETTY
227+
case RDFLanguages.JSONLD10 => RDFFormat.JSONLD10_COMPACT_PRETTY
229228
case RDFLanguages.JSONLD11 => RDFFormat.JSONLD11
230229
case RDFLanguages.TRIG => RDFFormat.TRIG_PRETTY
231230
case RDFLanguages.RDFXML => RDFFormat.RDFXML_PRETTY
@@ -313,8 +312,17 @@ object RdfConversions {
313312
None
314313
}
315314

316-
def jenaJsonLdContextWithFallbackForLocalhost(jsonLdContextUrl: URL, requestHost: String): Try[util.Context] =
315+
def jenaJsonLdContextWithFallbackForLocalhost(jsonLdContextUrl: URL, requestHost: String, baseUrl: Option[String]): Try[util.Context] =
317316
jsonLdContextWithFallbackForLocalhost(jsonLdContextUrl, requestHost)
317+
.map(ctx =>
318+
baseUrl
319+
.map(bu => {
320+
val c = ctx.clone()
321+
c.put("@base", bu)
322+
c
323+
})
324+
.getOrElse(ctx)
325+
)
318326
.map(jenaContext)
319327

320328
private def jsonLdContextUrl(data: Array[Byte]): Try[Option[URL]] =
@@ -359,12 +367,14 @@ object RdfConversions {
359367
ctx
360368
}
361369

362-
private def initCachingContext() = {
363-
val opts = new JsonLdOptions(null)
370+
def defaultJsonLdOpts(base: String) = {
371+
val opts = new JsonLdOptions(base)
364372
opts.useNamespaces = true
365-
new CachingJsonldContext(30, opts)
373+
opts
366374
}
367375

376+
private def initCachingContext() = new CachingJsonldContext(30, defaultJsonLdOpts(null))
377+
368378
private def escapeString(s: String) = {
369379
val sb = new StringBuilder(s.length())
370380
val slen = s.length()

‎src/test/resources/test-relative.jsonld‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"@context" : "https://raw.githubusercontent.com/dbpedia/databus-moss/dev/devenv/context2.jsonld",
3+
"@id" : "#mod",
4+
"wasGeneratedBy" : {
5+
"@id" : "#layer",
6+
"@type" : "DatabusMetadataLayer",
7+
"version" : "1.0.0",
8+
"name": "simple",
9+
"created" : "2024-03-01 14:37:32",
10+
"used" : "https://databus.dbpedia.org/lrec2024/linguistics/wordnet/2023#wordnet_lang=en.ttl.gz"
11+
},
12+
"subject" : [ "oeo:OEO_00020033" ]
13+
}

‎src/test/scala/org/dbpedia/databus/DatabusScalatraTest.scala‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package org.dbpedia.databus
33

44
import org.apache.jena.iri.ViolationCodes
55

6-
import java.io.ByteArrayInputStream
6+
import java.io.{ByteArrayInputStream}
77
import java.nio.file.{Files, Paths}
88
import org.apache.jena.rdf.model.ModelFactory
99
import org.apache.jena.riot.{Lang, RDFDataMgr}
@@ -98,6 +98,25 @@ class DatabusScalatraTest extends ScalatraFlatSpec with BeforeAndAfter {
9898

9999
}
100100

101+
"File save" should "save and retrieve jsonlds with relative uris" in {
102+
103+
val file = "test-relative.jsonld"
104+
val bytes = Files.readAllBytes(Paths.get(getClass.getClassLoader.getResource(file).getFile))
105+
106+
post("/databus/graph/save?repo=kuckuck&path=pa/rel_test.jsonld", bytes) {
107+
status should equal(200)
108+
}
109+
110+
get("/databus/graph/read?repo=kuckuck&path=pa/rel_test.jsonld") {
111+
status should equal(200)
112+
val respCtx = RdfConversions.contextUrl(bodyBytes, Lang.JSONLD10)
113+
respCtx should equal(RdfConversions.contextUrl(bytes, Lang.JSONLD10))
114+
respCtx.get.toString.nonEmpty should equal(true)
115+
body.contains(" \"generated\" : \"#mod\",") should equal(true)
116+
}
117+
118+
}
119+
101120
"File save" should "report problems in input" in {
102121

103122

‎src/test/scala/org/dbpedia/databus/ValidationTest.scala‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ class ValidationTest extends FlatSpec with Matchers with BeforeAndAfter {
2323
val bytes = Files.readAllBytes(Paths.get(getClass.getClassLoader.getResource(file).getFile))
2424

2525
val ctxU = contextUrl(bytes, lang)
26-
val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random").get)
26+
val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random", None).get)
2727

2828
val re = RdfConversions.validateWithShacl(bytes, ctx, shacl, lang)
2929
re.get.conforms() should be(true)
@@ -35,7 +35,7 @@ class ValidationTest extends FlatSpec with Matchers with BeforeAndAfter {
3535
val bytes = Files.readAllBytes(Paths.get(getClass.getClassLoader.getResource(file).getFile))
3636

3737
val ctxU = contextUrl(bytes, lang)
38-
val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random").get)
38+
val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random", None).get)
3939

4040
val re = RdfConversions.validateWithShacl(bytes, ctx, shacl, lang)
4141
re.get.conforms() should be(false)
@@ -47,7 +47,7 @@ class ValidationTest extends FlatSpec with Matchers with BeforeAndAfter {
4747
val bytes = Files.readAllBytes(Paths.get(getClass.getClassLoader.getResource(file).getFile))
4848

4949
val ctxU = contextUrl(bytes, lang)
50-
val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random").get)
50+
val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random", None).get)
5151

5252
val re = RdfConversions.validateWithShacl(bytes, ctx, shacl, lang)
5353
re.get.conforms() should be(true)
@@ -59,7 +59,7 @@ class ValidationTest extends FlatSpec with Matchers with BeforeAndAfter {
5959
val bytes = Files.readAllBytes(Paths.get(getClass.getClassLoader.getResource(file).getFile))
6060

6161
val ctxU = contextUrl(bytes, lang)
62-
val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random").get)
62+
val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random", None).get)
6363

6464
val re = RdfConversions.validateWithShacl(bytes, ctx, shacl, lang)
6565
re.get.conforms() should be(true)
@@ -72,10 +72,10 @@ class ValidationTest extends FlatSpec with Matchers with BeforeAndAfter {
7272
val bytes = Files.readAllBytes(Paths.get(getClass.getClassLoader.getResource(file).getFile))
7373

7474
val ctxU = contextUrl(bytes, lang)
75-
val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random").get)
75+
val ctx = ctxU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random", None).get)
7676

7777
val shaclU = contextUrl(shacl, RdfConversions.DefaultShaclLang)
78-
val shaclCtx = shaclU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random").get)
78+
val shaclCtx = shaclU.map(cu => jenaJsonLdContextWithFallbackForLocalhost(cu, "random", None).get)
7979

8080
val re = RdfConversions.validateWithShacl(bytes, shacl, ctx, shaclCtx, lang)
8181
re.get.conforms() should be(true)

‎swagger.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,12 @@ paths:
140140
type: string
141141
required: true
142142
example: "testgroup/test.jsonld"
143+
- in: query
144+
name: prefix
145+
description: "Prefix for graphid URL if it is not default. NOTE! should be without slash in the end."
146+
type: string
147+
required: false
148+
example: "http://foreighhost/api"
143149
responses:
144150
"200":
145151
description: "A file data."

0 commit comments

Comments
 (0)