Update common files and create CI action.

gkellogg · gkellogg · commit ee53b08ec13d · 2021-02-03T16:22:02.000-08:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,40 @@
+# This workflow validates the document for markup and examples.
+name: CI
+
+on:
+  push:
+    branches: [ '**' ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  tests:
+    name: Build and Validate
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.7
+    - name: Install dependencies
+      run: bundle install
+
+    - name: "Verify common files are consistent"
+      run: |
+        git remote add -f b https://github.com/w3c/json-ld-wg.git
+        git remote update
+        git diff --exit-code remotes/b/main -- common
+
+    - name: Verify examples are consistent
+      run: bundle exec rake test
+
+    - name: Checks HTML5
+      run: bundle exec rake check_html
+
+    # See https://github.com/w3c/spec-prod/blob/main/docs/examples.md
+    #- name: ReSpec Checker
+    #  uses: w3c/spec-prod@v1
+    #  with:
+    #    VALIDATE_LINKS: true
+    #    VALIDATE_MARKUP: true
diff --git a/common/common.js b/common/common.js
@@ -9,7 +9,7 @@ require(["core/pubsubhub"], (respecEvents) => {
 
   respecEvents.sub('end-all', (documentElement) => {
     // remove data-cite on where the citation is to ourselves.
-    const selfDfns = document.querySelectorAll("dfn[data-cite^='__SPEC__#']");
+    const selfDfns = Array.from(document.querySelectorAll("dfn[data-cite^='__SPEC__#']"));
     for (const dfn of selfDfns) {
       const anchor = dfn.querySelector('a');
       if (anchor) {
@@ -27,6 +27,51 @@ require(["core/pubsubhub"], (respecEvents) => {
       delete anchor.dataset.cite;
     }
 
+    //
+    // Remove/hide definitions which are  unused
+    // 1. Find all definitions in a termlist which are not preserved, indexed by data-cite
+    // 2. Find all references to definitions not in termlist
+    // 4. Hide definitions which are unreferenced
+    //
+    const remoteDfns = [];
+    document.querySelectorAll(".termlist dfn:not(.preserve)")
+      .forEach((item, index) => {
+        if (!selfDfns.includes(item)) {
+          remoteDfns[item.dataset["cite"]] = item;
+        }
+      });
+
+    // termlist internal references to definitions
+    const internalRefs = Array.from(document.querySelectorAll(".termlist a[data-cite]"));
+
+    // all references to definitions which are not internal refs
+    const allRefs = Array.from(document.querySelectorAll("a[data-cite]"))
+      .filter(e => !internalRefs.includes(e));
+
+    // Remove terms which are referenced
+    for (const item of allRefs) {
+      const cite = item.dataset["cite"];
+      // Delete this from remoteDfns, as it is referenced
+      delete remoteDfns[cite];
+    }
+
+    // Now remoteDfns only contains unreferenced terms
+    for (const item of Object.values(remoteDfns)) {
+      const dt = item.closest("dt");
+      if(dt) {
+        const dd = dt.nextElementSibling;
+        // Note, removing messes up some ReSpec references, so hiding instead
+        // dt.parentNode.removeChild(dt);
+        // dd.parentNode.removeChild(dd);
+        dt.hidden = true;
+        dd.hidden = true;
+      }
+    }
+    
+    //
+    // Playground
+    //
+
     // Add playground links
     for (const link of document.querySelectorAll("a.playground")) {
       let pre;
diff --git a/common/extract-examples.rb b/common/extract-examples.rb
@@ -11,6 +11,8 @@
 require 'getoptlong'
 require 'json'
 require 'json/ld/preloaded'
+require 'rdf/isomorphic'
+require 'rdf/vocab'
 require 'nokogiri'
 require 'linkeddata'
 require 'fileutils'
@@ -304,6 +306,17 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
     args = []
     content = ex[:content]
 
+    options = ex[:options].to_s.split(',').inject({}) do |memo, pair|
+      k, v = pair.split('=')
+      v = case v
+      when 'true' then true
+      when 'false' then false
+      else v
+      end
+      memo.merge(k.to_sym => v)
+    end
+    options[:validate] = true unless options.key?(:validate)
+
     $stderr.puts "example #{ex[:number]}: #{ex.select{|k,v| k != :content}.to_json(JSON::LD::JSON_STATE)}" if verbose
     $stderr.puts "content: #{ex[:content]}" if verbose
 
@@ -332,6 +345,7 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
       rescue JSON::ParserError => exception
         errors << "Example #{ex[:number]} at line #{ex[:line]} parse error: #{exception.message}"
         $stdout.write "F".colorize(:red)
+        $stderr.puts exception.backtrace.join("\n") if verbose
         next
       end
     when 'html'
@@ -355,30 +369,33 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
       rescue Nokogiri::XML::SyntaxError => exception
         errors << "Example #{ex[:number]} at line #{ex[:line]} parse error: #{exception.message}"
         $stdout.write "F".colorize(:red)
+        $stderr.puts exception.backtrace.join("\n") if verbose
         next
       end
     when 'table'
       content = Nokogiri::HTML.parse(content)
     when 'ttl', 'trig'
       begin
         reader_errors = []
-        RDF::Repository.new << RDF::TriG::Reader.new(content, validate: true, logger: reader_errors)
+        RDF::Repository.new << RDF::TriG::Reader.new(content, logger: reader_errors, **options)
       rescue
         reader_errors.each do |er|
           errors << "Example #{ex[:number]} at line #{ex[:line]} parse error: #{er}"
         end
         $stdout.write "F".colorize(:red)
+        $stderr.puts $!.backtrace.join("\n") if verbose
         next
       end
     when 'nq'
       begin
         reader_errors = []
-        RDF::Repository.new << RDF::NQuads::Reader.new(content, validate: true, logger: reader_errors)
+        RDF::Repository.new << RDF::NQuads::Reader.new(content, logger: reader_errors, **options)
       rescue
         reader_errors.each do |er|
           errors << "Example #{ex[:number]} at line #{ex[:line]} parse error: #{er}"
         end
         $stdout.write "F".colorize(:red)
+        $stderr.puts $!.backtrace.join("\n") if verbose
         next
       end
     end
@@ -389,17 +406,6 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
       content.define_singleton_method(:content_type) {ex[:content_type]} if ex[:content_type]
     end
 
-    options = ex[:options].to_s.split(',').inject({}) do |memo, pair|
-      k, v = pair.split('=')
-      v = case v
-      when 'true' then true
-      when 'false' then false
-      else v
-      end
-      memo.merge(k.to_sym => v)
-    end
-    options[:validate] = true
-
     # Set API to use
     method = case
     when ex[:compact]        then :compact
@@ -549,22 +555,26 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
       result = case method
       when nil then nil
       when :fromRdf
-        args[0] = RDF::Reader.for(file_extension: ext).new(args[0])
-        JSON::LD::API.fromRdf(*args)
+        args[0] = RDF::Reader.for(file_extension: ext).new(args[0], **options)
+        opts = args.last.is_a?(Hash) ? args.pop : {}
+        JSON::LD::API.fromRdf(*args, **opts)
       when :toRdf
-        RDF::Dataset.new statements: JSON::LD::API.toRdf(*args)
+        opts = args.last.is_a?(Hash) ? args.pop : {}
+        RDF::Repository.new << JSON::LD::API.toRdf(*args, **opts)
       else
-        JSON::LD::API.method(method).call(*args)
+        opts = args.last.is_a?(Hash) ? args.pop : {}
+        JSON::LD::API.method(method).call(*args, **opts)
       end
     rescue
       errors << "Example #{ex[:number]} at line #{ex[:line]} parse error generating result: #{$!}"
       $stdout.write "F".colorize(:red)
+      $stderr.puts $!.backtrace.join("\n") if verbose
       next
     end
 
     if verbose
-      if result.is_a?(RDF::Dataset)
-        $stderr.puts "result:\n" + result.to_trig
+      if result.is_a?(RDF::Enumerable)
+        $stderr.puts "result:\n" + result.to_nquads
       else
         $stderr.puts "result:\n" + result.to_json(JSON::LD::JSON_STATE)
       end
@@ -575,24 +585,26 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
         # Compare to expected to result
         case ex[:ext]
         when 'ttl', 'trig', 'nq', 'html'
-          reader = RDF::Reader.for(file_extension: ex[:ext]).new(content)
-          expected = RDF::Dataset.new(statements: reader)
-          $stderr.puts "expected:\n" + expected.to_trig if verbose
+          reader = RDF::Reader.for(file_extension: ex[:ext]).new(content, **options)
+          expected = RDF::Repository.new << reader
+          $stderr.puts "expected:\n" + expected.to_nquads if verbose
         when 'table'
           expected = begin
             table_to_dataset(content.xpath('/html/body/table'))
           rescue
             errors << "Example #{ex[:number]} at line #{ex[:line]} raised error reading table: #{$!}"
-            RDF::Dataset.new
+            $stderr.puts $!.backtrace.join("\n") if verbose
+            RDF::Repository.new
           end
             
           if verbose
-            $stderr.puts "expected:\n" + expected.to_trig
+            $stderr.puts "expected:\n" + expected.to_nquads
             $stderr.puts "result table:\n" + begin
               dataset_to_table(result)
             rescue
               errors << "Example #{ex[:number]} at line #{ex[:line]} raised error turning into table: #{$!}"
               ""
+              $stderr.puts $!.backtrace.join("\n") if verbose
             end
           end
         else
@@ -601,14 +613,8 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
         end
 
         # Perform appropriate comparsion
-        if expected.is_a?(RDF::Dataset)
-          expected_norm = RDF::Normalize.new(expected).map(&:to_nquads)
-          result_norm = RDF::Normalize.new(result).map(&:to_nquads)
-          if expected_norm.sort != result_norm.sort
-            if verbose
-              $stderr.puts "expected_norm:\n" + expected_norm.sort.join("")
-              $stderr.puts "result_norm:\n" + result_norm.sort.join("")
-            end
+        if expected.is_a?(RDF::Enumerable)
+          if !expected.isomorphic_with?(result)
             errors << "Example #{ex[:number]} at line #{ex[:line]} not isomorphic with #{examples[ex[:result_for]][:number]}"
             $stdout.write "F".colorize(:red)
             next
@@ -637,6 +643,7 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
     rescue
       errors << "Example #{ex[:number]} at line #{ex[:line]} parse error comparing result: #{$!}"
       $stdout.write "F".colorize(:red)
+      $stderr.puts $!.backtrace.join("\n") if verbose
       next
     end
 
