Add sparql-playground (#41)

paul121 · web-flow · commit 848cfd3ac6d2 · 2025-07-24T13:04:43.000-06:00
* Add sparql-playground

* Add schema/data/playground WIP

* Add script to gen-rdf

* Add script to update-graph

* Update C01 example project

* Add sparql-playground github action

* Update README

* Remove sparql-playground dir

* Add primary impact

* Add GRAPH_STORE_URL env and secret

* TEMP use endpoints.dev for graph store

* Check that dataset exists

* Do not add / to url

* Add -f to fail requests

* Add graph store auth

* Clear graph before adding data

* Add curl silent mode

* Add status check with link to yasgui

* Add permission for statuses write

* Remove test data
diff --git a/.github/workflows/sparql-playground.yaml b/.github/workflows/sparql-playground.yaml
@@ -0,0 +1,87 @@
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      deployments: write
+      statuses: write
+    name: Validate and Convert data
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+      - name: Install python dependencies
+        run: pip install -r requirements.txt
+      - name: Generate RDF
+        run: make -C schema gen-rdf
+      - name: Check if dataset exists
+        id: check-dataset
+        env:
+          GRAPH_STORE_URL: ${{ vars.GRAPH_STORE_URL }}
+          GRAPH_STORE_USER: ${{ secrets.GRAPH_STORE_USER }}
+          GRAPH_STORE_PW: ${{ secrets.GRAPH_STORE_PW }}
+          DATASET: ${{ github.event_name == 'pull_request' && format('PR-{0}', github.event.number) || 'main' }}
+        run: |
+          URL="${GRAPH_STORE_URL}${DATASET}"
+          echo "Using dataset: ${DATASET}"
+          echo "Full URL: ${URL}"
+          echo "url=${URL}" >> $GITHUB_OUTPUT
+          
+          GRAPH_STORE_AUTH="${GRAPH_STORE_USER}:${GRAPH_STORE_PW}"
+          echo "auth=${GRAPH_STORE_AUTH}" >> $GITHUB_OUTPUT
+          
+          echo "Checking if dataset exists at: ${URL}"
+          if curl -s -f -o /dev/null --user "${GRAPH_STORE_AUTH}" "${URL}"; then
+            echo "✅ Dataset exists at ${URL}"
+            echo "exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "❌ Dataset does not exist at ${URL}"
+            echo "⚠️  Skipping graph update step"
+            echo "exists=false" >> $GITHUB_OUTPUT
+          fi 
+
+      - name: Update graph
+        if: steps.check-dataset.outputs.exists == 'true'
+        env:
+          GRAPH_STORE_URL: ${{ steps.check-dataset.outputs.url }}
+          GRAPH_STORE_AUTH: ${{ steps.check-dataset.outputs.auth }}
+        run: make -C schema update-graph
+      - name: Create SPARQL Playground Status
+        if: steps.check-dataset.outputs.exists == 'true'
+        env:
+          GRAPH_STORE_URL: ${{ steps.check-dataset.outputs.url }}
+          GH_TOKEN: ${{ github.token }}
+        run: |  
+          # Define the SPARQL query in plain text
+          SPARQL_QUERY="
+          PREFIX rfs: <https://framework.regen.network/schema/>
+            
+            SELECT * WHERE {
+            ?sub ?pred ?obj .
+          } LIMIT 1000"
+          
+          # URL encode the SPARQL query
+          QUERY_ENDPOINT="${GRAPH_STORE_URL}/query"
+          ENCODED_QUERY=$(printf '%s' "${SPARQL_QUERY}" | jq -sRr @uri)
+          
+          # URL encode the query endpoint
+          ENCODED_ENDPOINT=$(printf '%s' "${QUERY_ENDPOINT}" | jq -sRr @uri)
+          
+          # Create the YASGUI URL
+          YASGUI_URL="https://yasgui.triply.cc/#query=${ENCODED_QUERY}&endpoint=${ENCODED_ENDPOINT}&requestMethod=POST&tabTitle=Query&headers=%7B%7D&contentTypeConstruct=application%2Fn-triples%2C*%2F*%3Bq%3D0.9&contentTypeSelect=application%2Fsparql-results%2Bjson%2C*%2F*%3Bq%3D0.9&outputFormat=table&outputSettings=%7B%22isEllipsed%22%3Afalse%2C%22compact%22%3Afalse%7D"
+          
+          # Create commit status
+          gh api repos/${{ github.repository }}/statuses/${{ github.sha }} \
+            --method POST \
+            --field state=success \
+            --field target_url="${YASGUI_URL}" \
+            --field description="Click to explore RDF data in SPARQL playground" \
+            --field context="sparql-playground/${DATASET}"
diff --git a/schema/Makefile b/schema/Makefile
@@ -7,10 +7,10 @@ lint:
 gen-doc:
 	gen-doc src/schema.yaml --directory generated --diagram-type er_diagram -v
 
-gen-jsonld:
-	linkml-convert --input-format yaml --output-format json-ld --schema src/schema.yaml data/BT01ProjectInfoData.yaml	--output data/BT01ProjectInfoData.jsonld --target-class BT01ProjectInfo
+gen-rdf:
+	scripts/gen-rdf.sh
 
-validate:
-	linkml-validate -s src/schema.yaml --target-class BT01ProjectInfo data/BT01ProjectInfoData.yaml
+update-graph:
+	scripts/update-graph.sh
 
-all: gen-taxonomy lint gen-doc gen-jsonld validate
+all: gen-taxonomy lint gen-doc gen-rdf update-graph
diff --git a/schema/README.md b/schema/README.md
@@ -12,10 +12,22 @@
 - Create separate schema files for each logical schema class and `import` into the root `schemas.yaml` file.
 - Generated markdown from schemas:
     ```shell
-    gen-doc schema/src/schemas.yaml --directory schema/generated --diagram-type er_diagram
     make gen-doc
     ```
 - Generate linkml enums for taxonomy terms:
     ```shell
     make gen-taxonomy 
-    ```
+    ```
+
+## Playground
+
+- Datasets are provided inside `schema/data/playground/{Credit-Class}/{LinkML-Class}/*.yaml` files
+- Validate and generate RDF data. This creates `.ttl` and `.jsonld` files for each dataset.
+    ```shell
+    make gen-rdf
+    ```
+- Add data to graph. This adds all `.ttl` files to the graph. Override the `GRAPH_STORE_URL` env var:
+  ```shell
+  export GRAPH_STORE_URL=http://localhost:7878/store
+  make update-graph
+  ```
diff --git a/schema/data/playground/.gitignore b/schema/data/playground/.gitignore
@@ -0,0 +1,2 @@
+*/*/*.jsonld
+*/*/*.ttl
diff --git a/schema/scripts/gen-rdf.sh b/schema/scripts/gen-rdf.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Root directory for data
+DATA_DIR="data/playground"
+SCHEMA_PATH="src/schema.yaml"
+
+# Use globbing to iterate through the nested structure
+shopt -s nullglob # Handle cases where no files match pattern
+
+# Initialize counters
+failed_count=0
+total_count=0
+
+# Loop through nested directories of linkml classes
+for linkml_class_dir in "$DATA_DIR"/*/*/; do
+
+    # Convert yaml files to RDF using linkml class schema
+    linkml_class=$(basename "$linkml_class_dir")
+    for yaml_file in "$linkml_class_dir"*.yaml; do
+
+        ((total_count++))
+        # Create output filename by replacing .yaml extension with .jsonld
+        output_file="${yaml_file%.yaml}.jsonld"
+        if ! linkml-convert -s "$SCHEMA_PATH" --validate --input-format yaml --output-format json-ld --target-class "$linkml_class" --output "$output_file" "$yaml_file" ; then
+            echo "❌ JSON-LD conversion failed for: $yaml_file"
+            ((failed_count++))
+        else
+            echo "✅ JSON-LD conversion passed for: $yaml_file"
+        fi
+
+        # Create output filename by replacing .yaml extension with .ttl
+        output_file="${yaml_file%.yaml}.ttl"
+        ((total_count++))
+        if ! linkml-convert -s "$SCHEMA_PATH" --validate --input-format yaml --output-format ttl --target-class "$linkml_class" --output "$output_file" "$yaml_file" ; then
+            echo "❌ TTL conversion failed for: $yaml_file"
+            ((failed_count++))
+        else
+            echo "✅ TTL conversion passed for: $yaml_file"
+        fi
+    done
+done
+
+echo "RDF conversion complete: $((total_count - failed_count)) passed, $failed_count failed out of $total_count total files"
+
+# Exit with failure if any validations failed
+[[ $failed_count -eq 0 ]] || exit 1
diff --git a/schema/scripts/update-graph.sh b/schema/scripts/update-graph.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# Root directory for data
+DATA_DIR="data/playground"
+
+# Only set default values if environment variables are not set
+: "${GRAPH_STORE_URL:=http://localhost:7878/store}"
+: "${GRAPH:=default}"
+
+# Build auth variable if GRAPH_STORE_AUTH is present.
+# GRAPH_STORE_AUTH should be in format user:pass
+if [ -n "$GRAPH_STORE_AUTH" ]; then
+    AUTH="--user $GRAPH_STORE_AUTH"
+else
+    AUTH=""
+fi
+
+# Set METHOD and GRAPH_PARAM based on GRAPH value
+# Use POST to append to default graph, and PUT to replace named graphs
+if [ "$GRAPH" = "default" ]; then
+    METHOD=POST
+    GRAPH_PARAM="?default"
+else
+    METHOD=PUT
+    GRAPH_PARAM="?graph=$GRAPH"
+fi
+
+# First, clear the graph.
+if ! curl -s -X DELETE -f $AUTH "$GRAPH_STORE_URL$GRAPH_PARAM" ; then
+    echo "❌ Failed to delete content in graph: $GRAPH"
+else
+    echo "✅ Deleted content in graph: $GRAPH with $file"
+fi
+
+# Use globbing to iterate through the nested structure
+shopt -s nullglob # Handle cases where no files match pattern
+
+# Initialize counters
+failed_count=0
+total_count=0
+
+# Loop through turtle files
+for file in "$DATA_DIR"/*/*/*.ttl; do
+    ((total_count++))
+
+    if ! curl -s -X $METHOD -f -H 'Content-Type: text/turtle' -T "$file" $AUTH "$GRAPH_STORE_URL$GRAPH_PARAM" ; then
+      echo "❌ Failed to update graph: $GRAPH with $file"
+        ((failed_count++))
+    else
+        echo "✅ Updated graph: $GRAPH with $file"
+    fi
+done
+
+echo "Updating graph complete: $((total_count - failed_count)) passed, $failed_count failed out of $total_count total files"
+
+# Exit with failure if any validations failed
+[[ $failed_count -eq 0 ]] || exit 1
