diff --git a/.github/workflows/gh-pages-workflow.yml b/.github/workflows/gh-pages-workflow.yml deleted file mode 100644 index 243e0b0..0000000 --- a/.github/workflows/gh-pages-workflow.yml +++ /dev/null @@ -1,42 +0,0 @@ -# This is a basic workflow to help you get started with Actions - -name: CI - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the main branch - push: - branches: [ master ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - # This workflow contains a single job called "build" - build: - # The type of runner that the job will run on - runs-on: ubuntu-latest - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v3 - - - name: Publish Bikeshed document - # You may pin to the exact commit or the version. - # uses: netwerk-digitaal-erfgoed/bikeshed-action@9058e7fb3f90044967e63c0d2e6add0514c5ae83 - #uses: netwerk-digitaal-erfgoed/bikeshed-action@v1 - uses: w3c/spec-prod@v2 - with: - TOOLCHAIN: bikeshed - - # Modify as appropriate - GH_PAGES_BRANCH: gh-pages - # The source file - SOURCE: eventstreams.bs # - - # output filename defaults to your input - # with .html extension instead, - # but if you want to customize it: - DESTINATION: index.html diff --git a/.github/workflows/ldes-server-primer-build.yml b/.github/workflows/ldes-server-primer-build.yml new file mode 100644 index 0000000..7d81ade --- /dev/null +++ b/.github/workflows/ldes-server-primer-build.yml @@ -0,0 +1,63 @@ +name: LDES Server Primer Build + +on: + pull_request: + paths: ["server-primer.bs"] + push: + branches: [main] + paths: ["server-primer.bs"] + tags: ["*"] + + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Publish Bikeshed document + uses: w3c/spec-prod@v2 + with: + TOOLCHAIN: bikeshed + + # Modify as appropriate + GH_PAGES_BRANCH: gh-pages + # The source file + SOURCE: server-primer.bs + + # output filename defaults to your input + # with .html extension instead, + # but if you want to customize it: + DESTINATION: server-primer.html + + copy-tag-version: + name: Copy tag snapshot + needs: build + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + permissions: + contents: write + steps: + - name: Checkout gh-pages + uses: actions/checkout@v3 + with: + ref: gh-pages + fetch-depth: 0 + + - name: Copy server-primer snapshot + run: | + set -euo pipefail + tag="${GITHUB_REF_NAME}" + dest="versions/${tag}" + mkdir -p "${dest}" + cp server-primer.html "${dest}/server-primer.html" + if git status --porcelain | grep . >/dev/null; then + git config user.email "semic-eu+github-actions[bot]@users.noreply.github.com" + git config user.name "SEMIC EU archival [bot]" + git add "${dest}/server-primer.html" + git commit -m "Add server primer snapshot for ${tag}" + git push + else + echo "No changes to commit." diff --git a/.github/workflows/ldes-specification-build.yml b/.github/workflows/ldes-specification-build.yml new file mode 100644 index 0000000..1fc230e --- /dev/null +++ b/.github/workflows/ldes-specification-build.yml @@ -0,0 +1,69 @@ +name: LDES Specification Build + +on: + pull_request: + paths: ["eventstreams.bs"] + push: + branches: [main] + paths: ["eventstreams.bs"] + tags: ["*"] + + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v3 + + - name: Publish Bikeshed document + # You may pin to the exact commit or the version. + uses: w3c/spec-prod@v2 + with: + TOOLCHAIN: bikeshed + + # Modify as appropriate + GH_PAGES_BRANCH: gh-pages + # The source file + SOURCE: eventstreams.bs + + # output filename defaults to your input + # with .html extension instead, + # but if you want to customize it: + DESTINATION: index.html + + copy-tag-version: + name: Copy tag snapshot + needs: build + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + permissions: + contents: write + steps: + - name: Checkout gh-pages + uses: actions/checkout@v3 + with: + ref: gh-pages + fetch-depth: 0 + + - name: Copy spec snapshot + run: | + set -euo pipefail + tag="${GITHUB_REF_NAME}" + dest="versions/${tag}" + mkdir -p "${dest}" + cp index.html "${dest}/index.html" + if git status --porcelain | grep . >/dev/null; then + git config user.email "semic-eu+github-actions[bot]@users.noreply.github.com" + git config user.name "SEMIC EU archival [bot]" + git add "${dest}/index.html" + git commit -m "Add spec snapshot for ${tag}" + git push + else + echo "No changes to commit." \ No newline at end of file diff --git a/.github/workflows/ldes-vocabulary-build.yml b/.github/workflows/ldes-vocabulary-build.yml new file mode 100644 index 0000000..3ebadac --- /dev/null +++ b/.github/workflows/ldes-vocabulary-build.yml @@ -0,0 +1,63 @@ +name: LDES Vocabulary Build + +on: + pull_request: + paths: ["vocabulary.bs"] + push: + branches: [main] + paths: ["vocabulary.bs"] + tags: ["*"] + + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Publish Bikeshed document + uses: w3c/spec-prod@v2 + with: + TOOLCHAIN: bikeshed + + # Modify as appropriate + GH_PAGES_BRANCH: gh-pages + # The source file + SOURCE: vocabulary.bs + + # output filename defaults to your input + # with .html extension instead, + # but if you want to customize it: + DESTINATION: vocabulary.html + + copy-tag-version: + name: Copy tag snapshot + needs: build + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + permissions: + contents: write + steps: + - name: Checkout gh-pages + uses: actions/checkout@v3 + with: + ref: gh-pages + fetch-depth: 0 + + - name: Copy vocabulary snapshot + run: | + set -euo pipefail + tag="${GITHUB_REF_NAME}" + dest="versions/${tag}" + mkdir -p "${dest}" + cp vocabulary.html "${dest}/vocabulary.html" + if git status --porcelain | grep . >/dev/null; then + git config user.email "semic-eu+github-actions[bot]@users.noreply.github.com" + git config user.name "SEMIC EU archival [bot]" + git add "${dest}/vocabulary.html" + git commit -m "Add vocabulary snapshot for ${tag}" + git push + else + echo "No changes to commit." \ No newline at end of file diff --git a/README.md b/README.md index 38ac9c8..101395f 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,14 @@ # The Linked Data Event Streams specification -A Linked Data Event Stream is a collection of immutable objects (such as version objects, sensor observations or archived representation). Each object is described in RDF. +Linked Data Event Streams (LDES) is an initiative to, as a data publisher, find a balance between publishing your data using a as-complete-as-possible set of querying APIs and a data dump. We propose an event stream as the base API, and want to make it as light-weight as possible to host one. -The objective of a Linked Data Event Stream is to allow consumers to replicate all of its items and to stay in sync when items are added. +LDES includes: + * A [vocabulary](https://w3id.org/ldes) that introduces terms to talk about an `ldes:EventStream` + * An example JSON-LD context that can be accessed from [here](https://w3id.org/ldes/context). It includes the JSON labels we recommend to use in JSON-LD documents. This document may however change over time and does not come with any waranties cfr. uptime. When building for a production environment, do thus not use this URL as an external context. + * A [consumer oriented specification](https://w3id.org/specification). You can use this spec if you want to implement an LDES client or consumer pipeline. + * A [server primer](https://w3id.org/ldes/server-primer) that you can consult when building an LDES as a data provider. -The HTML specification can be accessed from [here](https://w3id.org/ldes/specification). - -An example JSON-LD context can be accessed from [here](https://w3id.org/ldes/context). It includes the JSON labels we recommend to use in JSON-LD documents. This document may however change over time and does not come with any waranties cfr. uptime. When building for a production environment, do thus not use this URL as an external context. - -This specification uses [TREE](https://w3id.org/tree/specification) for its collection and fragmentation features, which in its turn is compatible to other specifications such as Activity Streams, DCAT-AP, LDP or Shape Trees. - -If you are new to the concept of Linked Data Event Stream or Linked Data, [this short training](https://academy.europa.eu/courses/publishing-data-with-linked-data-event-streams-why-and-how) introduces the main concepts. +The LDES specification uses the W3C [TREE hypermedia specification](https://w3id.org/tree/specification) for creating paginated search tree of event stream pages. ## Build the spec @@ -24,7 +22,8 @@ Install [bikeshed](https://tabatkins.github.io/bikeshed/) and then run `bikeshed ## Changelog -In 2025 the LDES specification is being rewritten from a consumer oriented perspective, while adding functionalities. Check the releases for changelogs. + * 2025-11-25: v1.0 launched: redesignd retention policies, a client algorithm, a server primer, etc. + * 2021-03-15: v0.1 launched: an initial design of the vocabulary with retention policies ## Acknowledgements diff --git a/eventstreams.bs b/eventstreams.bs index b514230..ada0e39 100644 --- a/eventstreams.bs +++ b/eventstreams.bs @@ -7,53 +7,57 @@ URL: https://w3id.org/ldes/specification Markup Shorthands: markdown yes Editor: Pieter Colpaert, https://pietercolpaert.be Repository: https://github.com/SEMICeu/LinkedDataEventStreams -Abstract: A Linked Data Event Stream (LDES) is an append-only collection of members described using the Resource Description Framework (RDF). The specification explains how a client can replicate the history of an event stream, and how it can then remain synchronized as new members are published. +Abstract: A Linked Data Event Stream (LDES) is an append-only collection of members described using the Resource Description Framework (RDF). The specification says how a client must replicate the history of an event stream, and how it can then remain synchronized as new members are published. # Introduction # {#introduction} -An **LDES client** is a piece of software used by a *consumer* that accepts the URL to an entry point, and returns a stream of members of the corresponding `ldes:EventStream`. -The data stream emits the history that is available from this entry point, and once the consumer has caught up with the stream, it remains synchronized as new members are published. -The specification describes two modes for the client: either it fetches all updates since the previous synchronization run in an unordered fashion, either it does this while maintaining the chronological order. +Linked Data Event Streams (LDES) is an initiative designed to help data publishers strike a balance between offering rich, queryable APIs and providing static data dumps. By proposing an event stream as the foundational API, LDES aims to make it as lightweight and straightforward as possible to host and maintain such a stream. + +LDES provides several key components: -In LDES, we extend [the vocabulary of the W3C TREE hypermedia](https://w3id.org/tree/specification) with specialized terms for dealing with append-only collections, referred to as *event streams*. -For example, one can indicate what time-based property in the member is used for indicating the order of the event stream, indicate the retention policy as a promise from the producer to the consumer, or detail how to deal with version-based members. + 1. A [consumer-oriented specification](https://w3id.org/ldes/specification) (this document) for implementing LDES clients and processors in a consumer pipeline. + 2. A [vocabulary](https://w3id.org/ldes) that introduces terms for describing an `ldes:EventStream`, such as for indicating the chronological order, retention policies and version-based create-update-delete semantics. + 3. An example [JSON-LD context](https://w3id.org/ldes/context), which includes recommended JSON labels for use in JSON-LD documents. Note that this context may change over time and is not guaranteed for uptime or stability; for production environments, avoid referencing this URL as an external context. + 4. A [server primer](server-primer) to guide data providers in building and publishing LDES-compliant streams. -# Overview # {#overview} +The document you’re reading now is the main specification that focuses on the consumer side, detailing how clients can efficiently replicate and synchronize with an event stream. -An overview of the LDES specification. +# Overview and terminology # {#overview} -A Linked Data Event Stream (LDES) (`ldes:EventStream`) is a collection (`rdfs:subClassOf tree:Collection`) of members that cannot be updated or removed once they are published, with each member being a set of RDF quads ([[!rdf-primer]]). -This way, the collection of members becomes an append-only log or *event stream*. +A **Linked Data Event Stream (LDES)** (`ldes:EventStream`) is a collection of **members** that cannot be updated or removed once they are published, with each member being a set of RDF quads ([[!rdf-primer]]). +This way, the collection of members becomes an append-only log or **event stream**. -Following the [TREE specification](https://w3id.org/tree/specification), this event stream is published using one or more HTTP resources. -When more resources are used, these pages, or `tree:Node`s, will be structured according to a search tree. -Therefore we will use the terms *root node* for the first page, and *subsequent node* for every next page in the structure. +An **LDES client** is a piece of software used by a **consumer** that accepts the URL to an entry point, and returns a stream of members of the corresponding `ldes:EventStream`. +The data stream emits the history that is available from this entry point, and once the consumer has caught up with the stream, it remains synchronized as new members are published. +The client can be used in a **consumer pipeline** with other **processors** in the pipeline that can benefit from the **context information** provided by the client. -In the **root node**, the client will expect these properties to be described on the `ldes:EventStream` entity: - * `ldes:timestampPath`: this is a [SHACL property path](https://www.w3.org/TR/shacl/#property-paths) (all further *Path properties are as well) that sets the chronological time with a `xsd:dateTime` literal within each member. This timestamp determines the chronological order in which members of the event stream are added. When `ldes:timestampPath` is set, no member can be added to the LDES with a timestamp earlier than the latest published member. - * `ldes:sequencePath`: when the LDES producer wants to make clear what the ordering is within members with the same timestamp for the `ldes:timestampPath`, this property defines, based on the [[!xpath-functions-31]] [comparison operator](https://www.w3.org/TR/xpath-functions-31/#func-compare), what xsd-literals define the order of processing. When no `ldes:timestampPath` has been set, the `ldes:sequencePath` defines the sequence for all members in the LDES. - * `ldes:versionOfPath`: when your entities are versioned, this property points at the object that tells you what entity it is a version of (e.g., `dcterms:isVersionOf`). - * `ldes:versionTimestampPath`: when those versions are not published chronologically, LDES providers have the possibility to indicate a different version timestamp to establish the latest version. - * `ldes:versionSequencePath`: when the versions do not follow the order in `ldes:timestampPath` and `ldes:sequencePath`, and for when `ldes:versionTimestampPath` are the same for multiple members, or for when this property is not set. E.g., for out of order publishing of `1` → `2`, in which `2` may have been published by the server before `1`. - * `tree:shape`: a [[!SHACL]] shape that can be used to select a search tree in the discovery phase, as well as to validate the members in the event stream. The `sh:NodeShape` linked validates each target of the `tree:member` property. A validator in a consumer pipeline MUST ignore other targets. - * `tree:view`: connects the collection to the current page, or points to one specific root node after dereferencing the `ldes:EventStream` identifier. +An **LDES server** is an HTTP server with a view of the members that can be consumed by an LDES client. +A **producer** can choose to do this by hosting static pages as well as hosting a dynamic server application. -In the **root node**, the current node identified by the URL of the page (a provider can achieve this simply by using a relative IRI `<>`) will be further described using these properties: - * `ldes:retentionPolicy`: indicates a retention policy (see next the dedicated chapter [](#retention)). - * `tree:viewDescription`: can as well contain the retention policy, or other context data about this view of the LDES (e.g., the `dcat:Distribution`, the `tree:SearchTree`, or the `ldes:EventSource`) as a named entity. This is useful for example if a producer would like to disambiguate the IRI for the `ldes:EventSource` from the root `tree:Node`. By default, the `tree:viewDescription` points at the root node. +The meaning of the words consumer, client, producer, server, etc. +An LDES is published using one or more HTTP resources, reusing the concepts from the [W3C TREE hypermedia specification](https://w3id.org/tree/specification). +When more resources are used, these pages, or **nodes** (`tree:Node`), will be structured according to a **search tree**. +Therefore, we use the terms **root node** for the first page and **subsequent node** for each next page in the structure. -In **any node**, root node or subsequent node, the client expects to find - * members (0 or more) of the `ldes:EventStream` using the `tree:member` property. The subject is the event stream instance and the object is the root focus node of a member. - * relations (0 or more) of the `tree:Node` the client is currently on, using the `tree:relation` properties, containing a description of the `tree:Relation`s from this node to subsequent nodes. +A **synchronization run** is one complete invocation of the client’s traversal logic, visiting all nodes that are relevant given the current state. During this synchronization run, the client emits the newly found members. -Note: In an `ldes:EventStream`, the object of the `tree:member` triple can only be an IRI as this IRI will be used in the state to check whether the member has already been emitted or not. +A **root node** will contain all [context information](#context-information). +The **root node** and any **subsequent node** will contain members and relations to other nodes. + +A `tree:Node` is considered **immutable** when re-fetching it does not result in new members. + +A search tree visualization + +An LDES has a **chronological order** that is the order of the members as they appear in the log. This is also the default order followed by the versions. +However, a more specific **version order** can be set, in which versions will not appear in the same order as their intended meaning (for example, version 2 might be published chronologically before version 1).
-An example record from a sensor observation dataset in the [[!turtle]] format: +An example root node with one member from a sensor observation dataset in the [[!turtle]] format: ```turtle ex:Observations a ldes:EventStream ; + # defines the chronological order ldes:timestampPath sosa:resultTime ; ldes:pollingInterval 60; # Each minute, new results are expected tree:shape ex:shape1.shacl ; @@ -66,8 +70,18 @@ ex:Observation1 a sosa:Observation ; ```
+ +A **view** is a specific publication of the members of the LDES. Multiple views can exist. The property `tree:view` connects the collection to the current page, or points to one specific root node after dereferencing the `ldes:EventStream` identifier. + +A [**retention policy**](#retention) can be documented on the root node that indicates not all members are being published in this view, but only a documented subset. + +Root node and subsequent nodes can contain [**relations**](#traversing-search-tree) to other nodes (using `tree:relation`) of the search tree. +They can also contain **members** using the `tree:member` property, pointing to a [**focus node**](https://www.w3.org/TR/shacl/#focusNodes) from which the full set of quads for this member can be found. The term focus node is borrowed from [[!SHACL]]. + +Note: In an `ldes:EventStream`, the object of the `tree:member` triple can only be an IRI as this IRI will be used in the state to check whether the member has already been emitted or not. +
-An example record from a base registry of addresses in the [[!trig]] format: +An example root node including 1 member from a base registry of addresses in the [[!trig]] format: ```turtle ex:AddressRecords a ldes:EventStream ; ldes:pollingInterval 86400; # Each day, new addresses are expected @@ -87,8 +101,6 @@ ex:AddressRecord1-activity1 { ```
-Issue: More specific server documentation should be found in a Server Primer (to do), such as containing a [link to the JSON-LD context](https://github.com/SEMICeu/LinkedDataEventStreams/issues/59), [official SHACL shapes for LDES](https://github.com/SEMICeu/LinkedDataEventStreams/issues/70) to validate your pages, best practices for publishing an LDES for reaching an optimal performance, best practices for enveloping your data using named graphs, how to build a status log for the use case of an aggregator or harvester, etc. - # Synchronization algorithm # {#synchronization-algorithm} There are multiple modes in which a client MAY operate. @@ -96,11 +108,9 @@ The client MUST have an unordered mode and/or an ordered ascending mode. It MAY also have any other mode not specified in this document. Ordered modes are only possible with `ldes:EventStreams` that have a `ldes:timestampPath` and/or `ldes:sequencePath`. -A *synchronization run* is one complete invocation of the client’s traversal logic, visiting all nodes that are relevant given the current state. During this synchronization run it emits the newly found members. - A client SHOULD check whether an `ldes:pollingInterval` was set on the LDES. If it is, the client SHOULD use this amount of seconds (`xsd:integer`) to set the time to keep between synchronization runs. -Note: Unordered will be straightforward to implement, while ordered modes will be more challenging due to the need for a more precise interpretation of relations and paths. Nevertheless, this will come with more functionality. It is up to a developer of a client to decide what functionalities they decide to offer. +Note: Unordered will be straightforward to implement, while ordered modes will be more challenging due to the need for a more precise interpretation of relations and paths. Nevertheless, this comes with more functionality. It is up to a client developer to decide which functionalities to offer. A client MUST have a way to indicate to further processors in a consumption pipeline that a synchronization run has been finalized. In order to prevent inconsistencies when reusing the result of the pipeline when not in ordered ascending mode, a consumer pipeline SHOULD wait for this finalization flag before committing those members at once into their system. @@ -112,10 +122,10 @@ In case there is no state yet, a client MUST perform an initialization run. ## Initialization run ## {#initialization} -The client MUST dereference `I` (see [HTTP Requests - Responses](#http-requests-responses)). +The client MUST dereference `I` (see [HTTP requests and responses](#http-requests-responses)). After dereferencing the IRI, the client MUST look for the patterns `?s tree:view <>` with `<>` the base IRI (after redirect). If this pattern was matched exactly once, `<>` is to be considered the root node, and `?s` is to be considered the `ldes:EventStream` IRI. In case it was matched multiple times, an error MUST be returned. If this pattern is not found, then it MUST look for the pattern `I tree:view ?o` instead. If this pattern matches exactly once, then `I` is to be considered the `ldes:EventStream` IRI and `?o` the root node. In this case, the IRI bound to `?o` MUST be dereferenced. In case multiple or no matches were found, an error SHOULD be returned. -The client’s aforementioned IRI dereferencing step MAY be extended with a source selection mechanism. +The client’s aforementioned IRI dereferencing step MAY be extended with a source selection mechanism. After processing the root node, the client MUST initiate a state object (see [state management](#state-management)) with the context information (see [context information](#context-information)) as found in the root node. @@ -131,21 +141,18 @@ A client MUST ensure a member is only emitted once. Note: Keeping a list of all emitted members forever will become problematic for large LDESs and slow down emitting the members. Instead, a client in unordered mode can assume that members found on immutable pages can safely be removed from the state after the run is finished. A client in ordered ascending mode can simply use the timestamp and/or sequence number of the last emitted member for that purpose. Mind that still the members that have exactly this timestamp and/or sequence number will still need to be kept in the state. -A `tree:Node` is considered *immutable*, when re-fetching it will not result in new members. - For every `tree:Node`, a client SHOULD check whether it is immutable by first checking whether 1. the triple `<> ldes:immutable true .` is set; then whether - 2. the cache-control HTTP response header is set to immutable; and finally + 2. the `Cache-Control` HTTP response header is set to `immutable`; and finally 3. a client MAY check whether the `tree:Relation` with a `tree:path` equal to the `ldes:timestampPath` that pointed us to the `tree:Node` had an upper bound that is earlier than the time of the latest processed member. A client SHOULD ensure an immutable `tree:Node` is not fetched more than once. - Note: Keeping a list of all immutable pages forever will become problematic for large LDESs. A client MUST ensure it can resume from a previous run. It SHOULD do so by keeping a frontier of pages that are not (yet) immutable. -In ordered mode, it MAY also use timestamp and sequence path of the last member as a bookmark. +In ordered mode, it MAY also use the timestamp and/or sequence path of the last member as a bookmark. A client MUST keep context information such as the identifier of the event stream and the root node, the SHACL shape of the event stream, or the retention policy of the root node, cf. the chapter on [Context Information](#context-information). @@ -153,19 +160,19 @@ A client SHOULD keep statistics such as the number of members emitted and the da A client MUST have a mechanism to communicate this context information and statistics to other processors in the pipeline. -When a `tree:Node` is not immutable, the `etag` SHOULD be kept if this is set in the response. +When a `tree:Node` is not immutable, the `ETag` SHOULD be kept if this is set in the response. ## HTTP requests and responses ## {#http-requests-responses} -A client MUST support HTTP responses in at least [[!n-quads]], [[!n-triples]], [[!trig]], [[!turtle]] and [[!json-ld]]. For JSON-LD external contexts, the client SHOULD implement HTTP caching. +A client MUST support HTTP responses in at least [[!n-quads]], [[!n-triples]], [[!trig]], [[!turtle]], and [[!json-ld]]. For JSON-LD external contexts, the client SHOULD implement HTTP caching. An `Accept` request header MUST be set. -A client SHOULD inspect the `cache-control` header to see whether it is set to `immutable`. +A client SHOULD inspect the `Cache-Control` header to see whether it is set to `immutable`. A client MUST follow redirects. -A client SHOULD support `If-None-Match` request header using the etags stored in the state, and process the `304 Not Modified` response accordingly. +A client SHOULD support the `If-None-Match` request header, using the ETags stored in the state, and process the `304 Not Modified` response accordingly. For the following status codes, the client MUST implement a retry mechanism with a back-off strategy: * `408 Request Timeout` @@ -180,18 +187,18 @@ A client MAY implement authorization and respond to a code like `401` with an au A client MUST process `410 Gone` as a page with an empty set of relations and an empty set of members. -A client MUST abort and throw an error on any other `4xx` or `5xx` range status codes. +A client MUST abort and throw an error on any other 4xx or 5xx status codes. ## Emitting members ## {#members} In unordered mode, the client SHOULD emit a member as soon as it is extracted. -In ordered mode, the client MUST ensure no other member can be still discovered that could precede the member that is to be emitted. +In ordered mode, the client MUST ensure no other member can still be discovered that could precede the member that is to be emitted. Extra conditions as documented in the next section MUST be checked before emitting it. A client MAY implement support for more specialized content types and profiles. For example, the [TREE profile](https://w3id.org/tree/specification/profile) specification promises to a parser that the member quads are going to be grouped together, and delimited by the `tree:member` quad. -Additionally to this specification an LDES client can assume the members will be in chronological ascending order and does not have to sort them anymore. +In addition to this specification, an LDES client can assume the members will be in chronological ascending order and does not need to sort them anymore. Without a specialized profile or content type that can indicate a “grouping of quads”/a “message”/a “frame”, a client MUST extract a description of the members as follows: @@ -201,7 +208,7 @@ For each focus node, a client MUST look up the subject-based star pattern (` For each match where `o` is a blank node, the algorithm is to be repeated recursively with `o` being the new focus node. A client MUST ensure a blank node is not processed twice. -A client in ordered mode that reads data from a `tree:Node` without a specialized profile or content type, MUST order the members according to the `ldes:timestampPath` and/or `ldes:sequencePath` itself. +A client in ordered mode that reads data from a `tree:Node` without a specialized profile or content type MUST order the members according to the `ldes:timestampPath` and/or `ldes:sequencePath`.
An example member that will be fully extracted thanks to the algorithm: @@ -239,11 +246,11 @@ Each distinct `n` MUST be further dereferenced and processed. ### Ordered ### {#ordered-traversal} -A client in ordered mode MUST be able to find the matching objects from a SHACL property path as this functionality is required for interpretting the paths in the TREE/LDES and SHACL specifications. +A client in ordered mode MUST be able to evaluate [SHACL property paths](https://www.w3.org/TR/shacl/#property-paths) to find the matching objects, as this functionality is required for interpreting the paths in the TREE/LDES and SHACL specifications. -The client in ordered mode MUST in the initialization phase check whether there is an `ldes:timestampPath` and/or `ldes:sequencePath` set. If not, it MUST return an error as order cannot be guaranteed. +The client in ordered mode MUST check, during the initialization phase, whether `ldes:timestampPath` and/or `ldes:sequencePath` is set. If not, it MUST return an error, as order cannot be guaranteed. -A client SHOULD implement a priority queue of next links to follow by interpretting these `tree:Relation` subclasses related to time literals: +A client SHOULD implement a priority queue of next links to follow by interpreting these `tree:Relation` subclasses related to time literals: * `tree:GreaterThanRelation`: later in time * `tree:GreaterThanOrEqualToRelation`: later in or at the same time * `tree:LessThanRelation`: earlier in time @@ -267,29 +274,67 @@ _:b1 a tree:LessThanRelation ; A client MUST combine multiple relations to the same node using a logical AND. A client MUST check whether the `ldes:timestampPath` is used in the `tree:path`. -Only then the relation can be used for ordering. +Only then can the relation be used for ordering. -Note: A link to a `tree:Node` with only a relation that is not supported (e.g. a `tree:GeospatiallyContainsRelation`) will have to be prioritized right away, as following this link may result in members that are earlier than any other member found elsewhere. +Note: A link to a `tree:Node` with only a relation that is not supported (e.g., a `tree:GeospatiallyContainsRelation`) will have to be prioritized right away, as following this link may result in members that are earlier than any other member found elsewhere. -As an addition to transactions text in the next chapter, the client in ordered ascending mode MUST ensure that the member that finalizes the transaction is emitted as the last member when there are multiple members on the same `ldes:timestampPath` and/or `ldes:sequencePath`. +In addition to the transactions text in the next chapter, the client in ordered ascending mode MUST ensure that the member that finalizes the transaction is emitted as the last member when there are multiple members with the same `ldes:timestampPath` and/or `ldes:sequencePath`. # Context information # {#context-information} -In this chapter we extend the context information described in the overview above with more advanced features and more details where needed. +A client MUST extract the context information from the **root node** and have a way to communicate the context information to processors further in the consumer pipeline. + +The client MUST extract context about the LDES, as well as about the service that is publishing the LDES. The former is attached to the LDES entity; the latter through the current page (`<>`) or from the entities linked using `tree:viewDescription`. + +
+A client must be able to handle context information on the LDES, on the root node, or in a view description: +```turtle +# event stream level context information + a ldes:EventStream ; + ldes:timestampPath dcterms:created ; + tree:view <> . +# Using a view description is optional for producers +<> tree:viewDescription <#LatestView> . +# view-level context information +<#LatestView> ldes:retentionPolicy [ + # ... see example of retention policies below +] . +# page-level context information +<> ldes:immutable true . +``` +
+ +## The chronological order of the stream ## {#chronological-order} + +When a consumer, such as the client in chronological mode, wants to establish the chronological order, it MUST derive this from the following two properties on the LDES (if set): + * `ldes:timestampPath`: this is a SHACL property path that sets the chronological time with an `xsd:dateTime` literal within each member. This timestamp determines the chronological order in which members of the event stream are added. When `ldes:timestampPath` is set, no member can be added to the LDES with a timestamp earlier than the latest published member. + * `ldes:sequencePath`: when the LDES producer wants to make clear what the ordering is within members with the same timestamp for the `ldes:timestampPath`, this property defines, based on the [[!xpath-functions-31]] [comparison operator](https://www.w3.org/TR/xpath-functions-31/#func-compare), which XSD literals define the order of processing. When no `ldes:timestampPath` has been set, the `ldes:sequencePath` defines the sequence for all members in the LDES. + +## The member’s SHACL shape ## {#shape} + +Using the property `tree:shape` on the LDES, a [[!SHACL]] `sh:NodeShape` can be linked that communicates an intention of the data provider to respect the shape for every member in the LDES. + +Note: This can be used by a client looking for specific members across multiple LDESs that wants to extend the initialization phase with a discovery or source selection phase. + +When building a processor to validate the members of an LDES, the processor MUST pass each `tree:member` object as the target for the given `sh:NodeShape` to the SHACL validator that is being used. + +Note: Multiple NodeShapes can be provided using [SHACL logical constraint components](https://www.w3.org/TR/shacl/#core-components-logical). + +Providing multiple `tree:shape` statements MUST be interpreted as a `sh:and` logical constraint component. ## Versions and transactions ## {#versions-transactions} Consumers can use the LDES version properties to decide what action to take. -E.g., when the consumer understands the members are versioned, it can upsert the members on each update. -If it understands something was created instead of updated, it can just add it into the store without removing statements first, and when a deletion comes in, it knows it can remove the statements associated with the previous insert or upsert. -To that extent, on the `ldes:EventStream` entity, these properties can be used, which are further explained in the vocabulary. - * [`ldes:versionOfPath`](#voc-versionOfPath) - such as `dcterms:isVersionOf` or `as:object` - * [`ldes:versionDeleteObject`](#voc-versionDeleteObject) - such as `as:Delete` - * [`ldes:versionCreateObject`](#voc-versionCreateObject) - such as `as:Create` - * [`ldes:versionUpdateObject`](#voc-versionUpdateObject) - such as `as:Update` - * [`ldes:versionDeletePath`](#voc-versionDeletePath) - defaults to `rdf:type` - * [`ldes:versionCreatePath`](#voc-versionCreatePath) - defaults to `rdf:type` - * [`ldes:versionUpdatePath`](#voc-versionUpdatePath) - defaults to `rdf:type` +For example, when the consumer understands the members are versioned, it can upsert the members on each update. +If it understands something was created instead of updated, it can add it into the store without removing statements first, and when a deletion comes in, it knows it can remove the statements associated with the previous insert or upsert. +To that extent, on the `ldes:EventStream` entity, these properties can be used and are further explained in the vocabulary. + * [`ldes:versionOfPath`](https://w3id.org/ldes#versionOfPath): such as `dcterms:isVersionOf` or `as:object` + * [`ldes:versionDeleteObject`](https://w3id.org/ldes#versionDeleteObject): such as `as:Delete` + * [`ldes:versionCreateObject`](https://w3id.org/ldes#versionCreateObject): such as `as:Create` + * [`ldes:versionUpdateObject`](https://w3id.org/ldes#versionUpdateObject): such as `as:Update` + * [`ldes:versionDeletePath`](https://w3id.org/ldes#versionDeletePath): defaults to `rdf:type` + * [`ldes:versionCreatePath`](https://w3id.org/ldes#versionCreatePath): defaults to `rdf:type` + * [`ldes:versionUpdatePath`](https://w3id.org/ldes#versionUpdatePath): defaults to `rdf:type`
Example: Versioned members using `ldes:versionOfPath`, `ldes:versionCreateObject`, `ldes:versionUpdateObject`, and `ldes:versionDeleteObject`: @@ -307,10 +352,15 @@ ex:AddressRecords a ldes:EventStream ; ```
-A consumer can also understand how to process the event stream in a way that the resulting knowledge graph will be consistent, by interpreting transactions using these properties: - * [`ldes:transactionPath`](#voc-transactionPath) - points at an identifier for the transaction. The result of evaluating the path can be a literal or a IRI. - * [`ldes:transactionFinalizedPath`](#voc-transactionFinalizedPath) - points at the object - * [`ldes:transactionFinalizedObject`](#voc-transactionFinalizedObject) - the value that the object needs to be in order to be finalized. Defaults to the `"true"^^xsd:boolean`. +Versions can also be published out of order. +A consumer that needs to interpret versions and select the latest MUST use these properties: + * [`ldes:versionTimestampPath`](https://w3id.org/ldes#versionTimestampPath): similar to `ldes:timestampPath`, but used when versioned entities are not published chronologically. + * [`ldes:versionSequencePath`](https://w3id.org/ldes#versionSequencePath): used when versions do not follow the order in `ldes:timestampPath` and `ldes:sequencePath`, or when `ldes:versionTimestampPath` is the same for multiple members, or when `ldes:versionTimestampPath` is not set. For example, for out-of-order publishing of `1` → `2`, `2` may have been published by the server before `1`. + +A consumer can also process the event stream in a way that ensures the resulting knowledge graph is consistent by interpreting transactions using these properties: + * [`ldes:transactionPath`](https://w3id.org/ldes#transactionPath): points to an identifier for the transaction. The result of evaluating the path can be a literal or an IRI. + * [`ldes:transactionFinalizedPath`](https://w3id.org/ldes#transactionFinalizedPath): points to the property whose value indicates whether the transaction has been finalized. + * [`ldes:transactionFinalizedObject`](https://w3id.org/ldes#transactionFinalizedObject): the value that the object must have in order to be considered finalized. Defaults to `"true"^^xsd:boolean`.
Example: Using `ldes:transactionPath`, `ldes:transactionFinalizedPath`, and `ldes:transactionFinalizedObject` to indicate transactions in an event stream: @@ -336,15 +386,15 @@ ex:Observation2 a sosa:Observation ;
When the IRI in the object of the `tree:member` triple is also used as a named graph, an LDES consumer MAY assume the payload of the upsert is in the named graph. -A consumer MUST implement a way to find back this group of triples in case an update or deletion comes in. +A consumer MUST implement a way to find this group of triples again in case an update or deletion comes in. ## Retention policies ## {#retention} The goal of a retention policy is to indicate in what way a specific view will not be able to provide a complete history of the event stream to the consumer. -This can help a consumer in the discovery phase to pick an specific LDES view, or help the consumer to detect non-viable synchronization set-ups. +This can help a consumer in the discovery phase to pick a specific LDES view, or help the consumer detect non-viable synchronization setups. -When no retention policy is provided in the root node, the consumer MUST assume that all members, that have once been added to the `ldes:EventStream`, are still available from this root node. -When a retention policy is provided however, a consumer MUST assume it will not be able to find members outside of the retention policy. +When no retention policy is provided in the root node, the consumer MUST assume that all members that have been added to the `ldes:EventStream` are still available from this root node. +When a retention policy is provided, however, a consumer MUST assume it will not be able to find members outside of the retention policy. An overview of the existing retention policies in LDES @@ -373,200 +423,23 @@ The root node itself can contain this information using the property `ldes:reten When the client is processing the root node, it MUST look for a retention policy in both ways. In the example above, the retention policy has been set on the root node (double typed as the `ldes:EventSource`). -When the [`ldes:retentionPolicy`](#voc-retentionpolicyproperty) would refer to an entity without further statements in the current page, the client MUST assume this view keeps no members at all. +When the [`ldes:retentionPolicy`](https://w3id.org/ldes#retentionPolicy) would refer to an entity without further statements in the current page, the client MUST assume this view keeps no members at all. Multiple properties can then be added to make the scope of members that are kept larger: - * [`ldes:startingFrom`](#voc-startingfrom): this view only retaines members starting from this `xsd:dateTime` with timezone. In combination with other retention policies, this property only enforces the period before the timestamp for which the view will not retain any member. - * [`ldes:fullLogDuration`](#voc-fulllogduration): the duration from current time from which all members are retained. Only in combination with `ldes:startingFrom`, and when the `ldes:startingFrom` timestamp is within this window, not all members within the member are retained. No other properties can influence this property. - * [`ldes:versionAmount`](#voc-versionamount): the amount of versions to keep. - * [`ldes:versionDuration`](#voc-versionduration): the duration from current time from which an amount of version are kept, to be used together with `ldes:versionAmount`. Defaults the duration of the full event stream. - * [`ldes:versionDeleteDuration`](#voc-versiondeleteduration): the period of time from current time the deletions in the event stream are retained. Before this period, deletions are not retained, regardless of `ldes:versionAmount` or `ldes:versionDuration`. + * [`ldes:startingFrom`](https://w3id.org/ldes#startingFrom): this view only retains members starting from this `xsd:dateTime` with timezone. In combination with other retention policies, this property only enforces the period before the timestamp for which the view will not retain any member. + * [`ldes:fullLogDuration`](https://w3id.org/ldes#fullLogDuration): the duration, from the current time, for which all members are retained. Only in combination with `ldes:startingFrom`, and when the `ldes:startingFrom` timestamp is within this window, not all members within the window are retained. No other properties can influence this property. + * [`ldes:versionAmount`](https://w3id.org/ldes#versionAmount): the number of versions to keep. + * [`ldes:versionDuration`](https://w3id.org/ldes#versionDuration): the duration, from the current time, for which a number of versions are kept, to be used together with `ldes:versionAmount`. Defaults to the duration of the full event stream. + * [`ldes:versionDeleteDuration`](https://w3id.org/ldes#versionDeleteDuration): the period of time, from the current time, for which deletions in the event stream are retained. Before this period, deletions are not retained, regardless of `ldes:versionAmount` or `ldes:versionDuration`. When using the current time in calculations, the consumer MUST take into account a safe buffer to mitigate clock inaccuracies. -The `ldes:timestampPath` points at the timestamp in the member that can be compared with the current time minus the durations. -When the `ldes:versionTimestampPath` has been set, the two version durations are to be compared with this timestamp. +The `ldes:timestampPath` points to the timestamp in the member that can be compared with the current time minus the durations. +When the `ldes:versionTimestampPath` has been set, the two version durations must be compared with this timestamp. -Historically, there are more specific type of retention policies that MUST remain supported although their use is discouraged in favor of the just introduced retention policy design. -These retention policies types are: +Historically, there are more specific types of retention policies that MUST remain supported, although their use is discouraged in favor of the retention policy design just introduced. +These retention policy types are: 1. `ldes:DurationAgoPolicy`: a time-based retention policy in which data generated before a specified duration is not retained. 2. `ldes:LatestVersionSubset`: a version subset based on the latest versions of an entity in the stream. 3. `ldes:PointInTimePolicy`: a point-in-time retention policy in which data generated before a specific time is not retained. -A `ldes:LatestVersionSubset` uses the property `ldes:amount` with as range an `xsd:integer` datatype, indicating the number of versions to keep. By default, this value is set to 1. -A `ldes:PointInTimePolicy` uses a `ldes:pointInTime` with an `xsd:dateTime`-typed literal to indicate the point in time on or after which data is kept when compared to a member’s timestamp. - -# Vocabulary # {#vocabulary} - -Next to re-using terms from the `tree:` vocabulary, the `ldes:` namespace introduced in this document provides a couple of new terms. -The base IRI for LDES is `https://w3id.org/ldes#`, and the preferred prefix is `ldes:`. -There is a Turtle version available at [https://w3id.org/ldes#Vocabulary](https://w3id.org/ldes). - -## ldes:EventStream ## {#voc-eventstream} - -The class `ldes:EventStream` is a subclass of `tree:Collection`. The specialization being that all members are immutable, and thus that this `tree:Collection` is append-only. - -## ldes:timestampPath ## {#voc-timestamppath} - -The path to the `xsd:dateTime` literal in each member that defines the order of the event stream. - -**Domain:** `ldes:EventStream` - -**Range:** a [SHACL property path](https://www.w3.org/TR/shacl/#property-paths) - -## ldes:sequencePath ## {#voc-sequencePath} - -The path to an xsd literal in each member that defines the order of the event stream in addition to the `ldes:timestampPath`. - -**Domain:** `ldes:EventStream` - -**Range:** a [SHACL property path](https://www.w3.org/TR/shacl/#property-paths) - -## ldes:pollingInterval ## {#voc-pollingInterval} - -The amount of seconds the client should keep in-between synchronization runs. - -**Domain:** `ldes:EventStream` - -**Range:** `xsd:integer` - -## ldes:versionOfPath ## {#voc-versionOfPath} - -The path to the IRI in each member that defines the entity of which this member is a version. - -**Domain:** `ldes:EventStream` - -**Range:** a [SHACL property path](https://www.w3.org/TR/shacl/#property-paths) - - -## ldes:versionTimestampPath ## {#voc-versionTimestampPath} - -For out of order event streams, this defines the path to the `xsd:dateTime` literal in each member that defines the order of versioned members. - -Only relevant when the `ldes:versionOfPath` has been set. - -**Domain:** `ldes:EventStream` - -**Range:** a [SHACL property path](https://www.w3.org/TR/shacl/#property-paths) - -## ldes:versionSequencePath ## {#voc-versionSequencePath} - -For out of order event streams, this defines the path to an xsd literal in each member that defines the order of the event stream in addition to the `ldes:versionTimestampPath`. - -**Domain:** `ldes:EventStream` - -**Range:** a [SHACL property path](https://www.w3.org/TR/shacl/#property-paths) - -## ldes:EventSource ## {#voc-eventsource} - -The class `ldes:EventSource` is a subclass of `dcat:Distribution`, the specialization being that this is a feed that uses a chronological search tree to make available a Linked Data Event Stream in order. - -An `ldes:EventSource` can *only* be published on LDESs that have a `ldes:timestampPath` set, and thus will publish their entities in this chronological order. - -## ldes:immutable ## {#voc-immutable} - -Re-fetching this tree:Node will not result in new members. - -**Domain:**: `tree:Node` - -**Range:**: `xsd:boolean` - -## ldes:retentionPolicy ## {#voc-retentionpolicyproperty} - -Links to a retention policy. - -**Domain:** Preferably the root node. Alternatively it can occur on any type of entity that is linked from the root node using `tree:viewDescription`. - -**Range:** `ldes:RetentionPolicy` - -## ldes:RetentionPolicy ## {#voc-retentionpolicy} - -The class for a retention policy that indicates how long members are preserved in this search tree. - -### ldes:startingFrom ### {#voc-startingfrom} - -The search tree only keeps members starting a certain timestamp. - -**Domain:** `ldes:RetentionPolicy` - -**Range:** `xsd:dateTime` with a timezone - -### ldes:versionDuration ### {#voc-versionduration} - -The search tree only keeps its versions, for which an `ldes:versionAmount` MUST have been set, only during a specific window. - -**Domain:** `ldes:RetentionPolicy` - -**Range:** `xsd:duration` - -### ldes:versionAmount ### {#voc-versionamount} - -The number of versions to keep. This MUST be a number greater than 0. - -**Domain:** `ldes:RetentionPolicy` - -**Range:** `xsd:integer` > 0 - -### ldes:versionDeleteDuration ### {#voc-versiondeleteduration} - -The search tree only keeps its deletions for a certain duration. - -**Domain:** `ldes:RetentionPolicy` - -**Range:** `xsd:duration` - -### ldes:fullLogDuration ### {#voc-fulllogduration} - -The search tree keeps its full log for a certain duration. - -**Domain:** `ldes:RetentionPolicy` - -**Range:** `xsd:duration` - -### Former retention policies terms ### {#voc-former-retentionpolicies} - - * `ldes:DurationAgoPolicy`: A retention policy class that uses an `xsd:duration` literal to document a sliding window of data. - * `ldes:LatestVersionSubset`: A retention policy class that select an amount of versions based on the versionOfPath. - * `ldes:amount`: The number of versions to keep. This MUST be a number greater than 0. - - Domain: `ldes:LatestVersionSubset` - - Range: `xsd:integer` - * `ldes:PointInTimePolicy`: A retention policy class that indicates members are kept starting on a certain point in time. - * `ldes:pointInTime`: The point in time from which members will be available starting from this root node. - - Domain: `ldes:PointInTimePolicy` - - Range: `xsd:dateTime` including an explicit timezone - -## Terms for versioning and transactions on top of ldes:EventStream ## {#voc-versioning-transactions} - -### ldes:versionCreatePath ### {#voc-versionCreatePath} - -Path indicating where you can do an object check on whether the member represents a create. Defaults to rdf:type. - -### ldes:versionUpdatePath ### {#voc-versionUpdatePath} - -Path indicating where you can do an object check on whether the member represents an update. Defaults to rdf:type. - -### ldes:versionDeletePath ### {#voc-versionDeletePath} - -Path indicating where you can do an object check on whether the member represents a delete. Defaults to rdf:type. - -### ldes:versionCreateObject ### {#voc-versionCreateObject} - -If the RDF object matches the object in the version create path, the member represents a create. - -### ldes:versionUpdateObject ### {#voc-versionUpdateObject} - -If the RDF object matches the object in the version update path, the member represents an update. - -### ldes:versionDeleteObject ### {#voc-versionDeleteObject} - -If the RDF object matches the object in the version delete path, the member represents a delete. - -### ldes:transactionPath ### {#voc-transactionPath} - -Path indicating how a member indicates whether it is part of a transaction. - -### ldes:transactionFinalizedPath ### {#voc-transactionFinalizedPath} - -Path indicating whether the transaction has been finalized. - -### ldes:transactionFinalizedObject ### {#voc-transactionFinalizedObject} - -If the RDF object matches the object in the transaction finalized path, the member indicates the transaction has been finalized. +An `ldes:LatestVersionSubset` uses the property `ldes:amount` with range `xsd:integer`, indicating the number of versions to keep. By default, this value is set to 1. +An `ldes:PointInTimePolicy` uses the property `ldes:pointInTime` with an `xsd:dateTime`-typed literal to indicate the point in time on or after which data is kept when compared to a member’s timestamp. \ No newline at end of file diff --git a/provider-consumer.svg b/provider-consumer.svg new file mode 100644 index 0000000..d42196b --- /dev/null +++ b/provider-consumer.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/searchtree.svg b/searchtree.svg new file mode 100644 index 0000000..21c826e --- /dev/null +++ b/searchtree.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/server-primer.bs b/server-primer.bs new file mode 100644 index 0000000..e186b5a --- /dev/null +++ b/server-primer.bs @@ -0,0 +1,265 @@ +
+Title: LDES Server Primer
+Shortname: LDES-SERVER
+Level: 1
+Status: LD
+URL: https://w3id.org/ldes/server-primer
+Markup Shorthands: markdown yes
+Editor: Pieter Colpaert, https://pietercolpaert.be
+Repository: https://github.com/SEMICeu/LinkedDataEventStreams
+Abstract: This Server Primer for Linked Data Event Streams (LDES) provides practical guidance for data publishers on implementing and hosting an LDES server. LDES aims to help publishers balance between offering rich querying APIs and simple data dumps by proposing an event stream as the base API. This primer focuses on lightweight, scalable approaches and best practices for setting up and maintaining an LDES server.
+
+ +# Introduction # {#introduction} + +This server primer is a living document of derived normative rules based on [the main consumer-oriented LDES specification](https://w3id.org/ldes/specification) and the [W3C TREE hypermedia specification](https://w3id.org/tree/specification). + +A Linked Data Event Stream (LDES) is an append-only log consisting of *immutable* members. The term “member” could also be interpreted as “event”, “activity”, “observation”, “record”, or “immutable entity”. For example, “an observation states that at this timestamp a specific sensor observed 5°C”. However, since LDES extends the W3C TREE hypermedia specification, we use the term “member” for consistency. + +An overview of the LDES specification. + +# Serializations and HTTP Responses # {#serializations} + +A server MUST provide data in either [[!n-quads]], [[!n-triples]], [[!trig]], [[!turtle]] or [[!json-ld]]. +It MAY also provide multiple serializations using content negotiation. + +Note: When using content negotiation, set `Vary: Accept`. + +It SHOULD provide an `ETag` header on responses. If the page is immutable, it SHOULD provide a `Cache-Control: immutable` header. + +If [[!json-ld]] is used, there is an example context at https://w3id.org/ldes/context. +Do not reference this URL directly in production; copy it into your project. +If you host an external context yourself, ensure robust caching with the `ETag` and/or `Cache-Control` max-age headers. + +A provider SHOULD implement the [TREE Profile specification](https://w3id.org/tree/specification/profile) for performance. In this case, you MUST order members chronologically in the page (i.e., append the members to the file as you go). + +Issue: We will try to generalize this in the future so that we can also integrate with [Jelly](https://jelly-rdf.github.io/dev). This binary serialization has the potential to raise performance drastically. + +If the server is overloaded, it MUST provide a `429 Too Many Requests`. The client will then retry later. + +# Context Information # {#context-information} + +On the first page (root node), you MUST include context information about the LDES and this particular root node of the LDES. +For features and how a client would interpret them, see [the main spec](https://w3id.org/ldes/specification). + +Using `tree:viewDescription` on the root node, you MAY also link to an entity (embedded in the same page) that contains the retention policy, or other context data about this view of the LDES (e.g., the `dcat:Distribution`, the `tree:SearchTree`, or the `ldes:EventSource`) as a named entity. This is useful, for example, if a producer would like to disambiguate the IRI for the `ldes:EventSource` from the root `tree:Node`. + +Recommended context properties on the `ldes:EventStream`: +- `ldes:timestampPath`:chronological order +- `ldes:sequencePath`: tie-breaking or alternative order +- `ldes:pollingInterval`: expected seconds between new members +- `tree:shape`: SHACL node shape for members. If the `sh:NodeShape` changes, it should remain backwards compatible to avoid breaking existing consumers. If a backwards-incompatible change is needed (such as changing your dataset to a different vocabulary), it should be published as a new event stream, ensuring older streams continue to function as expected. +- Versioning: `ldes:versionOfPath`, optional create/update/delete paths and objects +- Out-of-order versioning: `ldes:versionTimestampPath`, `ldes:versionSequencePath` +- Transactions (optional): `ldes:transactionPath`, `ldes:transactionFinalizedPath`, `ldes:transactionFinalizedObject` + +# Paginating Your Event Stream # {#paginating} + +Instead of a one- or two-dimensional pagination scheme, TREE/LDES lets you describe the relations you want and build the search tree you need. We recommend the following: + * You SHOULD set the chronological order of your event stream using `ldes:timestampPath`. Other properties such as `ldes:sequencePath` MAY be used as an addition, or as an alternative. In the latter case, the ordering will be incremental based on the ordering of the XSD literal. + * You SHOULD use the same `tree:path` in your relations as in your `ldes:timestampPath`. This way, a client knows you structured your search tree according to chronological order. + * Use two relations towards one node, one with the lower bound and another with the upper bound of the time interval it directs to. + * Start with 1 root node that contains links to member pages. If that gets too large, you can introduce another level. + * All `xsd:dateTime` literals you publish SHOULD come with a timezone. + * You SHOULD use relative IRIs when referring to other pages. + +Issue: we should still add examples of how to paginate here. + +Every `tree:Node` MAY contain zero or more members and MAY contain zero or more relations. + +## Entry Points and Discovery ## {#entry-points} + +Publish a stable entry point for clients. Expose either: +- A page where the `ldes:EventStream` IRI `S` links with `tree:view <>` to the current page; or +- An `ldes:EventStream` IRI `S` that has exactly one `tree:view` triple pointing to the root node `R`. + +Avoid ambiguity by ensuring there is exactly one `tree:view` for the entry point. If you rotate the root node over time, keep `R` stable or use redirects. + +Issue: Discovery is yet to be further explained. More input from existing implementations is appreciated through the issue tracker. + +## Members ## {#members} + +Members MUST be linked from the event stream identifier using `tree:member`. For example: + +
+```turtle +@prefix ldes: . +@prefix tree: . +@prefix ex: . + +ex:eventstream a ldes:EventStream ; + tree:view <> ; + tree:member ex:member1, ex:member2 ; + ldes:timestampPath ex:createdAt ; + ldes:versionOfPath ex:versionOf . +``` +
+ +The object of `tree:member` MUST be an IRI that identifies an immutable concept. + +Note: To ensure immutability, the IRI should reference a resource that cannot change over time. A common approach is to include a timestamp, hash or version identifier in the IRI, so that each IRI corresponds to a specific, unalterable state or event. + +If you add a member to multiple pages, this MUST be done atomically. This ensures that a client’s synchronization run is reliable: members emitted in the current run will not be newly encountered in future runs. This atomicity is a precondition for clients to safely forget parts of the log, as those members cannot be encountered again once the pages the members were encountered in become immutable. + +If you reuse the member IRI as a named graph, clients MAY assume the payload of the upsert is in that named graph. Publish consistently so consumers can locate the triples for updates and deletions. + +## Transactions ## {#transactions} + +If you want to flag that certain members must be processed together (e.g., a large deletion operation), you can model transactions: +- Set `ldes:transactionPath` to identify the transaction (literal or IRI). +- Set `ldes:transactionFinalizedPath` whose object indicates the transaction is finalized. +- Optionally set `ldes:transactionFinalizedObject` to the value that denotes finalization (defaults to `"true"^^xsd:boolean`). + +Producers SHOULD ensure the member that finalizes the transaction has an equal or later `ldes:timestampPath`/`ldes:sequencePath` than preceding transaction members so ordered clients can emit it last. + +# Scaling # {#scaling} + +Next to optimizations such as using a binary format such as [Jelly](https://jelly-rdf.github.io), or manually creating an aggregated summary LDES as a derived LDES, there are also two other tools one can use for scaling up. + +## Compacting your log with a retention policy ## {#log-compaction} + +Retention policies enable servers to compact their logs while keeping client expectations clear. By declaring a retention policy on the root node (or via a `tree:viewDescription` entity linked from the root), producers communicate what portion of the event history is still available from this view. Clients will assume they cannot retrieve members outside the declared policy window. + +Where to publish and cardinality +- Publish the retention policy at the root node using `ldes:retentionPolicy` (0..1), or at an entity referenced from the root via `tree:viewDescription` with its own `ldes:retentionPolicy`. +- If `ldes:retentionPolicy` points to an IRI with no further statements in the current page, clients will assume no members are retained from this view. + +Supported policy properties +- `ldes:startingFrom` (`xsd:dateTime` with timezone): Earliest timestamp of retained members for this view. +- `ldes:fullLogDuration` (`xsd:duration`): Duration from the current time for which all members are retained. Used to express a sliding full-history window. +- `ldes:versionAmount` (`xsd:integer` > 0): Number of versions retained per entity. +- `ldes:versionDuration` (`xsd:duration`): Duration from the current time for which up to `ldes:versionAmount` versions are retained. +- `ldes:versionDeleteDuration` (`xsd:duration`): Duration from the current time for which delete events are retained. + +Computation and time base +- Use the member timestamp indicated by `ldes:timestampPath` to compare with the current time minus durations. +- If `ldes:versionTimestampPath` is set, evaluate `ldes:versionDuration` and `ldes:versionDeleteDuration` against that version timestamp. +- Servers SHOULD account for small clock skew by using a safety buffer when computing which members fall outside the window. + +Publishing changes and server behavior +- When compaction removes members or whole nodes from a view, update the search tree so that no relations point to removed nodes. +- For nodes that are no longer available, respond with `410 Gone`. Clients will treat such a page as having no members and no relations. +- Do not modify the content of immutable pages; instead, stop linking to them, redirect, or make them `410 Gone`. +- Keep relation semantics consistent: if you publish lower/upper bounds to a node, ensure the window described by the relations still matches the members after compaction. + +
+Sliding full history for one year, plus version constraints +```turtle +@prefix ldes: . +@prefix tree: . +@prefix dcterms: . +@prefix xsd: . + +<> a ldes:EventSource ; + ldes:retentionPolicy [ + ldes:fullLogDuration "P1Y"^^xsd:duration ; + ldes:versionAmount 1 ; + ldes:versionDeleteDuration "P1Y"^^xsd:duration + ] . +``` +
+ +
+Point-in-time start and version window +```turtle +@prefix ldes: . +@prefix tree: . +@prefix xsd: . + +<> a ldes:EventSource ; + ldes:retentionPolicy [ + ldes:startingFrom "2026-01-01T00:00:00Z"^^xsd:dateTime ; + ldes:versionAmount 3 ; + ldes:versionDuration "P90D"^^xsd:duration + ] . +``` +
+ +Notes +- Changing a retention policy affects client expectations; keep the policy in sync with the actual availability of members. +- Historical, more specific policy classes (`ldes:DurationAgoPolicy`, `ldes:LatestVersionSubset`, `ldes:PointInTimePolicy`) SHOULD remain supported for backward compatibility but are discouraged in favor of `ldes:retentionPolicy` with the properties above. + +## Rebalancing the search tree ## {#rebalancing} + +Rebalancing a search tree of an LDES is interesting for old immutable pages as most clients are going to be interested in the full history anyway. Compression becomes much more efficient on bigger pages, and thus less data will need to be transferred over the wire, saving bandwidth. + +Rebalancing is tricky though, because a client might get stuck in edge cases when it is just replicating the datasets while the rebalancing is happening, and also a server cache might still have a copy of all or some of your immutable pages. + +As a running example, imagine a client is synchronizing a day-page `2022-05-02` but then all pages under `2022` are getting merged into one. + +A server MUST, in that case, provide redirects to a new IRI, such as `2022-rebalanced`, from all old pages, including the page `2022` to `2022-rebalanced`. + +Note: the semantics of `ldes:immutable` are that the members on this page and the relations should not be processed again. The page MAY still be rebalanced later on, or the page can become unavailable on disk (`410 Gone`). + +# Validating the pages # {#validating} + +This section includes the rules to validate an implementation of a root node and any subsequent node. + +Issue: we still need to build the SHACL shapes here. + +Issue: we still need an UML image here. + +## For the Root Node ## {#rootnode} + +A root node MUST link the event stream to the view using the `tree:view` property. + +A root node MUST contain context information about the LDES. All these properties in the domain of the event stream have a cardinality of 0 or 1: +- `ldes:timestampPath`: points to a SHACL property path (an rdf:List or an IRI) +- `ldes:sequencePath`: points to a SHACL property path (an rdf:List or an IRI) +- `ldes:versionOfPath`: points to a SHACL property path (an rdf:List or an IRI) +- `ldes:versionTimestampPath`: points to a SHACL property path (an rdf:List or an IRI) +- `ldes:versionSequencePath` : points to a SHACL property path (an rdf:List or an IRI) +- `tree:shape`: point to a `sh:NodeShape`. +- `ldes:versionCreatePath`: points to a SHACL property path (an rdf:List or an IRI) +- `ldes:versionUpdatePath`: points to a SHACL property path (an rdf:List or an IRI) +- `ldes:versionDeletePath`: points to a SHACL property path (an rdf:List or an IRI) +- `ldes:versionCreateObject` +- `ldes:versionUpdateObject` +- `ldes:versionDeleteObject` +- `ldes:transactionPath`: points to a SHACL property path (an rdf:List or an IRI) +- `ldes:transactionFinalizedPath`: points to a SHACL property path (an rdf:List or an IRI) +- `ldes:transactionFinalizedObject` + +A root node MUST contain context information about this particular entry point: +- `ldes:retentionPolicy` 0 or 1. Although in older versions of the spec multiple were allowed. +- `tree:viewDescription` 0 or 1. + +### For the Retention policies ### {#retention-policies} + +A root node MUST contain at most one `ldes:retentionPolicy` property (cardinality: 0..1). +The value of `ldes:retentionPolicy` MUST be an IRI referring to a retention policy description. + +A retention policy description MAY contain the following properties, each with cardinality 0 or 1: + +- `ldes:startingFrom` (0..1) If present, this property MUST be a `xsd:dateTime` literal indicating the earliest timestamp for retained members. + +- `ldes:fullLogDuration` (0..1) If present, this property MUST be a duration literal specifying the time window for which all members are retained. + +- `ldes:versionAmount` (0..1) If present, this property MUST be an integer literal specifying the number of versions to retain per entity. + +- `ldes:versionDuration` (0..1) If present, this property MUST be a duration literal specifying the time window for which versions are retained. + +- `ldes:versionDeleteDuration` (0..1) If present, this property MUST be a duration literal specifying the time window for which deletions are retained. + +## Root Node and Subsequent Nodes ## {#nodes} + +On the event stream, 0 or more `tree:member` triples are provided. The objects MUST be IRIs. + +A `tree:view` triple MAY be present on the event stream to the current page `<>`. + +A current page `<>` has 0 or more `tree:relation` properties to relations. + +This page MAY also have `ldes:immutable true` attached to it. +The default value is false. If it is not immutable, this SHOULD NOT be made explicit using a `false` value. + +## Relations ## {#relations} + +Relations in LDES are used to describe how pages or nodes are connected within the event stream. Each relation is represented using the `tree:relation` property and SHOULD specify its type and relevant properties. + +On all relations, exactly one `tree:node` MUST be present. The object MUST be an IRI. + +In case it is typed a `tree:GreaterThanRelation`, `tree:LessThanRelation`, `tree:EqualToRelation`, `tree:LessThanOrEqualToRelation`, or `tree:GreaterThanOrEqualToRelation`, each of these relations MUST specify exactly one `tree:path` (a [[!SHACL]] path) and `tree:value`. + +For chronological views, you SHOULD use the same `tree:path` as the `ldes:timestampPath`. For time windows, publish both lower- and upper-bound relations to the same node; clients combine relations to the same node using logical AND. Avoid orphan relations and overlapping intervals that cause ambiguous traversal. + +If a relation type isn’t understood by clients (e.g., a geospatial relation), provide an ordering-compatible path elsewhere so ordered clients can still discover early members. diff --git a/vocabulary.bs b/vocabulary.bs new file mode 100644 index 0000000..64d9889 --- /dev/null +++ b/vocabulary.bs @@ -0,0 +1,205 @@ +
+Title: Linked Data Event Streams Vocabulary
+Shortname: LDES Vocabulary
+Level: 1
+Status: LS
+URL: https://w3id.org/ldes
+Markup Shorthands: markdown yes
+Editor: Pieter Colpaert, https://pietercolpaert.be
+Repository: https://github.com/SEMICeu/LinkedDataEventStreams
+Abstract:  This document defines the Linked Data Event Streams (LDES) Vocabulary, which provides terms for describing append-only, immutable collections of events as Linked Data. The vocabulary extends existing standards such as the TREE and DCAT vocabularies to support event streams, versioning, retention policies, and transactional semantics.
+
+ +# Introduction # {#introduction} + +This specification introduces terms used to describe event streams. +While these terms can be used independently of the LDES specification, +the main LDES specification is available at [https://w3id.org/ldes/specification](https://w3id.org/ldes/specification). + +# Terms # {#terms} + +This document introduces terms in the namespace `https://w3id.org/ldes#` with the preferred prefix `ldes:`. +There is also a [Turtle version available](https://w3id.org/ldes.ttl). + +Used prefixes are: + +```turtle +@prefix ldes: . +@prefix tree: . +@prefix dct: . +@prefix rdf: . +@prefix rdfs: . +@prefix xsd: . +``` + +## ldes:EventStream ## {#EventStream} + +The class `ldes:EventStream` is a subclass of `tree:Collection`, which in turn is a subclass of `dcat:Dataset`, +specialized such that all members are immutable, making the collection append-only. + +## ldes:timestampPath ## {#timestampPath} + +The path to the `xsd:dateTime` literal in each member that defines the order of the event stream. + +**Domain:** `ldes:EventStream` + +**Range:** a [SHACL property path](https://www.w3.org/TR/shacl/#property-paths) + +## ldes:sequencePath ## {#sequencePath} + +The path to an `xsd`-typed literal in each member that defines the order of the event stream in addition to the `ldes:timestampPath`. + +**Domain:** `ldes:EventStream` + +**Range:** a [SHACL property path](https://www.w3.org/TR/shacl/#property-paths) + + +## ldes:versionOfPath ## {#versionOfPath} + +The path to the IRI in each member that defines the entity of which this member is a version. + +**Domain:** `ldes:EventStream` + +**Range:** a [SHACL property path](https://www.w3.org/TR/shacl/#property-paths) + + +## ldes:versionTimestampPath ## {#versionTimestampPath} + +For out-of-order event streams, this defines the path to the `xsd:dateTime` literal in each member that defines the order of versioned members. + +Only relevant when the `ldes:versionOfPath` has been set. + +**Domain:** `ldes:EventStream` + +**Range:** a [SHACL property path](https://www.w3.org/TR/shacl/#property-paths) + +## ldes:versionSequencePath ## {#versionSequencePath} + +For out-of-order event streams, this defines the path to an `xsd`-typed literal in each member that defines the order of the event stream in addition to the `ldes:versionTimestampPath`. + +**Domain:** `ldes:EventStream` + +**Range:** a [SHACL property path](https://www.w3.org/TR/shacl/#property-paths) + +## ldes:pollingInterval ## {#pollingInterval} + +The number of seconds the client should keep between synchronization run calls. + +**Domain:** `ldes:EventStream` +**Range:** `xsd:integer` + +## ldes:immutable ## {#immutable} + +If the node is not going to contain new relations leading to new members, or new members itself, it can be set to immutable. + +**Domain:** `tree:Node` +**Range:** `xsd:boolean` + +## ldes:EventSource ## {#EventSource} + +The class `ldes:EventSource` is a subclass of `dcat:Distribution`, +specialized to represent a feed that uses a chronological search tree to make a Linked Data Event Stream available in order. + +An `ldes:EventSource` can only be published for LDESs that have `ldes:timestampPath` set, and thus publishes its entities in chronological order. + +## ldes:retentionPolicy ## {#retentionPolicy} + +Links to a retention policy. + +**Domain:** Preferably on the root node. Alternatively, it can occur on any entity linked from the root node using `tree:viewDescription`. + +**Range:** `ldes:RetentionPolicy` + +## ldes:RetentionPolicy ## {#RetentionPolicy} + +The class for a retention policy that indicates how long members are preserved in this view. + +### ldes:startingFrom ### {#startingFrom} + +The view only keeps members starting from a certain timestamp. + +**Domain:** `ldes:RetentionPolicy` + +**Range:** `xsd:dateTime` with a timezone + +### ldes:versionDuration ### {#versionDuration} + +The view keeps versions (for which `ldes:versionAmount` MUST be set) only for a specific time window. + +**Domain:** `ldes:RetentionPolicy` + +**Range:** `xsd:duration` + +### ldes:versionAmount ### {#versionAmount} + +The number of versions to keep. This MUST be greater than 0. + +**Domain:** `ldes:RetentionPolicy` + +**Range:** `xsd:integer` + +### ldes:versionDeleteDuration ### {#versionDeleteDuration} + +The view only keeps its deletions for a certain duration. + +**Domain:** `ldes:RetentionPolicy` + +**Range:** `xsd:duration` + +### ldes:fullLogDuration ### {#fullLogDuration} + +The view keeps its full log for a certain duration. + +**Domain:** `ldes:RetentionPolicy` + +**Range:** `xsd:duration` + +### Former retention policy terms ### {#former-retentionpolicies} + + * `ldes:DurationAgoPolicy`: A retention policy class that uses an `xsd:duration` literal to document a sliding window of data. + * `ldes:LatestVersionSubset`: A retention policy class that selects a number of versions based on `ldes:versionOfPath`. + * `ldes:amount`: The number of versions to keep. This MUST be a number greater than 0. + - Domain: `ldes:LatestVersionSubset` + - Range: `xsd:integer` + * `ldes:PointInTimePolicy`: A retention policy class that indicates members are kept starting from a certain point in time. + * `ldes:pointInTime`: The point in time from which members will be available starting from this root node. + - Domain: `ldes:PointInTimePolicy` + - Range: `xsd:dateTime` including an explicit timezone + +## Terms for versioning and transactions on top of ldes:EventStream ## {#versioning-transactions} + +### ldes:versionCreatePath ### {#versionCreatePath} + +Path whose object indicates that the member represents a create. Defaults to `rdf:type`. + +### ldes:versionUpdatePath ### {#versionUpdatePath} + +Path whose object indicates that the member represents an update. Defaults to `rdf:type`. + +### ldes:versionDeletePath ### {#versionDeletePath} + +Path whose object indicates that the member represents a delete. Defaults to `rdf:type`. + +### ldes:versionCreateObject ### {#versionCreateObject} + +If the object at `ldes:versionCreatePath` equals this value, the member represents a create. + +### ldes:versionUpdateObject ### {#versionUpdateObject} + +If the object at `ldes:versionUpdatePath` equals this value, the member represents an update. + +### ldes:versionDeleteObject ### {#versionDeleteObject} + +If the object at `ldes:versionDeletePath` equals this value, the member represents a delete. + +### ldes:transactionPath ### {#transactionPath} + +Path whose object indicates whether the member is part of a transaction. + +### ldes:transactionFinalizedPath ### {#transactionFinalizedPath} + +Path indicating whether the transaction has been finalized. + +### ldes:transactionFinalizedObject ### {#transactionFinalizedObject} + +If the object at `ldes:transactionFinalizedPath` equals this value, the member indicates the transaction has been finalized. diff --git a/vocabulary.ttl b/vocabulary.ttl index 43839da..7935191 100644 --- a/vocabulary.ttl +++ b/vocabulary.ttl @@ -165,4 +165,10 @@ ldes:immutable a rdf:Property ; rdfs:label "Immutable"@en ; rdfs:comment "Re-fetching this tree:Node will not result in new members."@en ; rdfs:domain tree:Node ; - rdfs:range xsd:boolean . \ No newline at end of file + rdfs:range xsd:boolean . + +ldes:pollingInterval a rdf:Property ; + rdfs:label "Polling Interval"@en ; + rdfs:comment "The number of seconds the client should keep between synchronization run calls."@en ; + rdfs:domain ldes:EventStream ; + rdfs:range xsd:integer . \ No newline at end of file