diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 97394937..5eb9ffb9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -16,7 +16,7 @@ jobs: build: # The type of runner that the job will run on runs-on: ubuntu-latest - container: obolibrary/odkfull:v1.2.32 + container: obolibrary/odkfull:v1.3.0 # Steps represent a sequence of tasks that will be executed as part of the job steps: diff --git a/src/ontology/Makefile b/src/ontology/Makefile index 22c9d9cb..d27fc94a 100644 --- a/src/ontology/Makefile +++ b/src/ontology/Makefile @@ -1,7 +1,7 @@ # ---------------------------------------- # Makefile for bdso # Generated using ontology-development-kit -# ODK Version: v1.2.32 +# ODK Version: v1.3.0 # ---------------------------------------- # IMPORTANT: DO NOT EDIT THIS FILE. To override default make goals, use bdso.Makefile instead @@ -28,6 +28,9 @@ RELEASEDIR= ../.. REPORTDIR= reports TEMPLATEDIR= ../templates TMPDIR= tmp +MIRRORDIR= mirror +IMPORTDIR= imports +SUBSETDIR= subsets SCRIPTSDIR= ../scripts SPARQLDIR = ../sparql COMPONENTSDIR = components @@ -35,9 +38,9 @@ REPORT_FAIL_ON = None REPORT_LABEL = -l true REPORT_PROFILE_OPTS = OBO_FORMAT_OPTIONS = -SPARQL_VALIDATION_CHECKS = equivalent-classes owldef-self-reference -SPARQL_EXPORTS = basic-report class-count-by-prefix edges xrefs obsoletes synonyms -ODK_VERSION_MAKEFILE = v1.2.32 +SPARQL_VALIDATION_CHECKS = owldef-self-reference iri-range label-with-iri +SPARQL_EXPORTS = basic-report class-count-by-prefix edges xrefs obsoletes synonyms +ODK_VERSION_MAKEFILE = v1.3.0 TODAY ?= $(shell date +%Y-%m-%d) OBODATE ?= $(shell date +'%d:%m:%Y %H:%M') @@ -46,8 +49,7 @@ ANNOTATE_ONTOLOGY_VERSION = annotate -V $(ONTBASE)/releases/$(VERSION)/$@ --anno OTHER_SRC = $(PATTERNDIR)/definitions.owl $(COMPONENTSDIR)/all_templates.owl ONTOLOGYTERMS = $(TMPDIR)/ontologyterms.txt PATTERNDIR= ../patterns -DOSDP_SCHEMA= http:// # change to PURL when ready. -PATTERN_TESTER= simple_pattern_tester.py +PATTERN_TESTER= dosdp validate -i DOSDPT= dosdp-tools PATTERN_RELEASE_FILES= $(PATTERNDIR)/definitions.owl $(PATTERNDIR)/pattern.owl @@ -64,10 +66,10 @@ RELEASE_ARTEFACTS = $(sort $(ONT)-base $(ONT)-simple-non-classified $(ONT)-full .PHONY: .FORCE .PHONY: all -all: odkversion all_imports patterns all_main all_subsets sparql_test all_reports all_assets +all: odkversion test all_assets .PHONY: test -test: odkversion sparql_test all_reports $(REPORTDIR)/validate_profile_owl2dl_$(ONT).owl.txt +test: odkversion dosdp_validation sparql_test robot_reports $(REPORTDIR)/validate_profile_owl2dl_$(ONT).owl.txt $(ROBOT) reason --input $(SRC) --reasoner ELK --equivalent-classes-allowed asserted-only --exclude-tautologies structural --output test.owl && rm test.owl && echo "Success" .PHONY: odkversion @@ -75,28 +77,29 @@ odkversion: echo "ODK Makefile version: $(ODK_VERSION_MAKEFILE) (this is the version of the ODK with which this Makefile was generated, not the version of the ODK you are running)" &&\ echo "ROBOT version (ODK): " && $(ROBOT) --version -$(TMPDIR) $(REPORTDIR) : +$(TMPDIR) $(REPORTDIR) $(MIRRORDIR) $(IMPORTDIR) $(COMPONENTSDIR) $(SUBSETDIR): mkdir -p $@ -## -- main targets -- -## -## By default this is the cross-product of {ont, ont-base} x FORMATS +# ---------------------------------------- +# Release assets +# ---------------------------------------- MAIN_PRODUCTS = $(sort $(foreach r,$(RELEASE_ARTEFACTS), $(r)) $(ONT)) -MAIN_GZIPPED = -MAIN_FILES = $(foreach n,$(MAIN_PRODUCTS), $(foreach f,$(FORMATS), $(n).$(f))) $(MAIN_GZIPPED) +MAIN_GZIPPED = +MAIN_FILES = $(foreach n,$(MAIN_PRODUCTS), $(foreach f,$(FORMATS), $(n).$(f))) $(MAIN_GZIPPED) +SRCMERGED = $(TMPDIR)/merged-$(SRC) .PHONY: all_main all_main: $(MAIN_FILES) -## -- import targets -- -## -## By default this is the cross-product of IMPORT_MODULES x FORMATS +# ---------------------------------------- +# Import assets +# ---------------------------------------- IMPORTS = pr go ro uberon pato cl ensmusg ncbitaxon -IMPORT_ROOTS = $(patsubst %, imports/%_import, $(IMPORTS)) +IMPORT_ROOTS = $(IMPORTDIR)/merged_import IMPORT_OWL_FILES = $(foreach n,$(IMPORT_ROOTS), $(n).owl) IMPORT_FILES = $(IMPORT_OWL_FILES) @@ -104,20 +107,23 @@ IMPORT_FILES = $(IMPORT_OWL_FILES) .PHONY: all_imports all_imports: $(IMPORT_FILES) -## -- subset targets -- -## -## By default this is the cross-product of SUBSETS x FORMATS -## Note we also include TSV as a format +# ---------------------------------------- +# Subset assets +# ---------------------------------------- SUBSETS = -SUBSET_ROOTS = $(patsubst %, subsets/%, $(SUBSETS)) +SUBSET_ROOTS = $(patsubst %, $(SUBSETDIR)/%, $(SUBSETS)) SUBSET_FILES = $(foreach n,$(SUBSET_ROOTS), $(foreach f,$(FORMATS_INCL_TSV), $(n).$(f))) .PHONY: all_subsets all_subsets: $(SUBSET_FILES) +# ---------------------------------------- +# QC Reports & Utilities +# ---------------------------------------- + OBO_REPORT = $(SRC)-obo-report REPORTS = $(OBO_REPORT) REPORT_FILES = $(patsubst %, $(REPORTDIR)/%.tsv, $(REPORTS)) @@ -126,20 +132,53 @@ REPORT_FILES = $(patsubst %, $(REPORTDIR)/%.tsv, $(REPORTS)) robot_reports: $(REPORT_FILES) .PHONY: all_reports -all_reports: all_reports_onestep $(REPORT_FILES) +all_reports: custom_reports robot_reports + +# ---------------------------------------- +# ROBOT OWL Profile checking +# ---------------------------------------- -$(REPORTDIR)/validate_profile_owl2dl_%.txt: % | $(REPORTDIR) - $(ROBOT) validate-profile --profile DL -i $< -o $@ +# The conversion to functional syntax is necessary to avoid undeclared entity violations. +$(REPORTDIR)/validate_profile_owl2dl_%.txt: % | $(REPORTDIR) $(TMPDIR) + $(ROBOT) merge -i $< convert -f ofn -o $(TMPDIR)/validate.ofn + $(ROBOT) validate-profile --profile DL -i $(TMPDIR)/validate.ofn -o $@ || { cat $@ && exit 1; } .PRECIOUS: $(REPORTDIR)/validate_profile_owl2dl_%.txt -.PHONY: validate_profile_% validate_profile_%: $(REPORTDIR)/validate_profile_owl2dl_%.txt + echo "$* profile validation completed." + +# ---------------------------------------- +# Sparql queries: Q/C +# ---------------------------------------- -## -- all files/assets -- +# these live in the ../sparql directory, and have suffix -violation.sparql +# adding the name here will make the violation check live. + +SPARQL_VALIDATION_QUERIES = $(foreach V,$(SPARQL_VALIDATION_CHECKS),$(SPARQLDIR)/$(V)-violation.sparql) + +sparql_test: $(SRC) catalog-v001.xml | $(REPORTDIR) +ifneq ($(SPARQL_VALIDATION_QUERIES),) + $(ROBOT) verify --catalog catalog-v001.xml -i $< --queries $(SPARQL_VALIDATION_QUERIES) -O $(REPORTDIR) +endif + +# ---------------------------------------- +# ROBOT report +# ---------------------------------------- + +$(REPORTDIR)/$(SRC)-obo-report.tsv: $(SRCMERGED) | $(REPORTDIR) + $(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) --print 5 -o $@ + +$(REPORTDIR)/%-obo-report.tsv: % | $(REPORTDIR) + $(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) --print 5 -o $@ + +# ---------------------------------------- +# Release assets +# ---------------------------------------- ASSETS = \ $(IMPORT_FILES) \ $(MAIN_FILES) \ + $(PATTERN_RELEASE_FILES) \ $(REPORT_FILES) \ $(SUBSET_FILES) @@ -150,13 +189,11 @@ RELEASE_ASSETS = \ .PHONY: all_assets all_assets: $(ASSETS) - .PHONY: show_assets show_assets: echo $(ASSETS) du -sh $(ASSETS) - # ---------------------------------------- # Release Management # ---------------------------------------- @@ -166,33 +203,28 @@ CLEANFILES=$(MAIN_FILES) $(SRCMERGED) # It will ensure that all assets/files are fresh, and will copy to release folder .PHONY: prepare_release -prepare_release: $(ASSETS) $(PATTERN_RELEASE_FILES) +prepare_release: all rsync -R $(RELEASE_ASSETS) $(RELEASEDIR) &&\ - mkdir -p $(RELEASEDIR)/patterns &&\ - cp $(PATTERN_RELEASE_FILES) $(RELEASEDIR)/patterns &&\ - rm -f $(CLEANFILES) &&\ - echo "Release files are now in $(RELEASEDIR) - now you should commit, push and make a release on your git hosting site such as GitHub or GitLab" + mkdir -p $(RELEASEDIR)/patterns && cp -rf $(PATTERN_RELEASE_FILES) $(RELEASEDIR)/patterns &&\ + rm -f $(CLEANFILES) &&\ + echo "Release files are now in $(RELEASEDIR) - now you should commit, push and make a release on your git hosting site such as GitHub or GitLab" .PHONY: prepare_initial_release -prepare_initial_release: prepare_release +prepare_initial_release: all_assets + rsync -R $(RELEASE_ASSETS) $(RELEASEDIR) &&\ + mkdir -p $(RELEASEDIR)/patterns && cp -rf $(PATTERN_RELEASE_FILES) $(RELEASEDIR)/patterns &&\ + rm -f $(patsubst %, ./%, $(CLEANFILES)) &&\ cd $(RELEASEDIR) && git add $(RELEASE_ASSETS) -# ---------------------------------------- -# Import modules -# ---------------------------------------- -# Most ontologies are modularly constructed using portions of other ontologies -# These live in the imports/ folder - # ------------------------ # Imports: Seeding system # ------------------------ # seed.txt contains all referenced entities IMPORTSEED=$(TMPDIR)/seed.txt -SRCMERGED=$(TMPDIR)/merged-$(SRC) PRESEED=$(TMPDIR)/pre_seed.txt -$(SRCMERGED): $(SRC) +$(SRCMERGED): $(SRC) $(OTHER_SRC) $(ROBOT) remove --input $< --select imports --trim false \ merge $(patsubst %, -i %, $(OTHER_SRC)) -o $@ @@ -209,51 +241,53 @@ $(SIMPLESEED): $(SRCMERGED) $(ONTOLOGYTERMS) echo "http://www.geneontology.org/formats/oboInOwl#SynonymTypeProperty" >> $@ -ALLSEED = $(PRESEED) $(PATTERNDIR)/all_pattern_terms.txt \ +ALLSEED = $(PRESEED) $(TMPDIR)/all_pattern_terms.txt \ -$(IMPORTSEED): prepare_patterns $(ALLSEED) +$(IMPORTSEED): $(ALLSEED) | $(TMPDIR) if [ $(IMP) = true ]; then cat $(ALLSEED) | sort | uniq > $@; fi - ANNOTATION_PROPERTIES=rdfs:label IAO:0000115 -# -- Generate Import Modules -- -# +# ---------------------------------------- +# Import modules +# ---------------------------------------- +# Most ontologies are modularly constructed using portions of other ontologies +# These live in the imports/ folder # This pattern uses ROBOT to generate an import module -# Generate terms.txt for each import. (Assume OBO-style Possibly hacky step?) + # Should be able to drop this if robot can just take a big messy list of terms as input. -imports/%_terms_combined.txt: $(IMPORTSEED) imports/%_terms.txt +$(IMPORTDIR)/%_terms_combined.txt: $(IMPORTSEED) $(IMPORTDIR)/%_terms.txt if [ $(IMP) = true ]; then cat $^ | grep -v ^# | sort | uniq > $@; fi -ALL_TERMS_COMBINED = $(patsubst %, imports/%_terms_combined.txt, $(IMPORTS)) -imports/merged_terms_combined.txt: $(ALL_TERMS_COMBINED) +ALL_TERMS_COMBINED = $(patsubst %, $(IMPORTDIR)/%_terms_combined.txt, $(IMPORTS)) +$(IMPORTDIR)/merged_terms_combined.txt: $(ALL_TERMS_COMBINED) if [ $(IMP) = true ]; then cat $^ | grep -v ^# | sort | uniq > $@; fi -imports/merged_import.owl: mirror/merged.owl imports/merged_terms_combined.txt +$(IMPORTDIR)/merged_import.owl: $(MIRRORDIR)/merged.owl $(IMPORTDIR)/merged_terms_combined.txt if [ $(IMP) = true ]; then $(ROBOT) merge -i $< \ remove --select "" remove --select "" remove --select "" remove --select "" remove --select "" remove --select "" remove --select "" remove --select "" \ - extract -T imports/merged_terms_combined.txt --force true --copy-ontology-annotations true --individuals exclude --method BOT \ + extract -T $(IMPORTDIR)/merged_terms_combined.txt --force true --copy-ontology-annotations true --individuals exclude --method BOT \ query --update ../sparql/inject-subset-declaration.ru --update ../sparql/postprocess-module.ru \ annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --output $@.tmp.owl && mv $@.tmp.owl $@; fi -imports/%_import.owl: mirror/merged.owl imports/%_terms_combined.txt +$(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/merged.owl $(IMPORTDIR)/%_terms_combined.txt if [ $(IMP) = true ]; then $(ROBOT) query -i $< --update ../sparql/preprocess-module.ru \ - extract -T imports/$*_terms_combined.txt --force true --copy-ontology-annotations true --individuals exclude --method BOT \ + extract -T $(IMPORTDIR)/$*_terms_combined.txt --force true --copy-ontology-annotations true --individuals exclude --method BOT \ query --update ../sparql/inject-subset-declaration.ru --update ../sparql/postprocess-module.ru \ annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --output $@.tmp.owl && mv $@.tmp.owl $@; fi -.PRECIOUS: imports/%_import.owl +.PRECIOUS: $(IMPORTDIR)/%_import.owl ## Module for ontology: pr -imports/pr_import.owl: mirror/merged.owl imports/pr_terms_combined.txt - if [ $(IMP) = true ] && [ $(IMP_LARGE) = true ]; then $(ROBOT) extract -i $< -T imports/pr_terms_combined.txt --force true --individuals exclude --method BOT \ +$(IMPORTDIR)/pr_import.owl: $(MIRRORDIR)/merged.owl $(IMPORTDIR)/pr_terms_combined.txt + if [ $(IMP) = true ] && [ $(IMP_LARGE) = true ]; then $(ROBOT) extract -i $< -T $(IMPORTDIR)/pr_terms_combined.txt --force true --individuals exclude --method BOT \ query --update ../sparql/inject-subset-declaration.ru --update ../sparql/postprocess-module.ru \ annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --output $@.tmp.owl && mv $@.tmp.owl $@; fi @@ -265,18 +299,17 @@ refresh-imports: no-mirror-refresh-imports: make IMP=true MIR=false PAT=false IMP_LARGE=true all_imports -B - .PHONY: refresh-imports-excluding-large refresh-imports-excluding-large: make IMP=true MIR=true PAT=false IMP_LARGE=false all_imports -B .PHONY: refresh-% refresh-%: - make IMP=true IMP_LARGE=true MIR=true PAT=false imports/$*_import.owl -B + make IMP=true IMP_LARGE=true MIR=true PAT=false $(IMPORTDIR)/$*_import.owl -B .PHONY: no-mirror-refresh-% no-mirror-refresh-%: - make IMP=true IMP_LARGE=true MIR=false PAT=false imports/$*_import.owl -B + make IMP=true IMP_LARGE=true MIR=false PAT=false $(IMPORTDIR)/$*_import.owl -B # ---------------------------------------- @@ -284,7 +317,9 @@ no-mirror-refresh-%: # ---------------------------------------- # Some ontologies contain external and internal components. A component is included in the ontology in its entirety. -$(COMPONENTSDIR)/%: +COMP=true # Global parameter to bypass component generation + +$(COMPONENTSDIR)/%: | $(COMPONENTSDIR) touch $@ .PRECIOUS: $(COMPONENTSDIR)/% @@ -292,97 +327,83 @@ $(COMPONENTSDIR)/%: - - # ---------------------------------------- # Mirroring upstream ontologies # ---------------------------------------- -# IMP=true # Global parameter to bypass import generation MIR=true # Global parameter to bypass mirror generation IMP_LARGE=true # Global parameter to bypass handling of large imports -PAT=true # Global parameter to bypass pattern generation -## ONTOLOGY: pr -## Copy of pr is re-downloaded whenever source changes -mirror/pr.trigger: $(SRC) -mirror/pr.owl: mirror/pr.trigger + +## ONTOLOGY: pr +.PHONY: mirror-pr +.PRECIOUS: $(MIRRORDIR)/pr.owl +mirror-pr: | $(TMPDIR) if [ $(MIR) = true ] && [ $(IMP) = true ] && [ $(IMP_LARGE) = true ]; then $(ROBOT) convert -I https://raw.githubusercontent.com/obophenotype/pro_obo_slim/master/pr_slim.owl -o $@.tmp.owl && \ - $(ROBOT) remove -i $@.tmp.owl --base-iri $(URIBASE)/PR --axioms external --preserve-structure false --trim false -o $@.tmp.owl && mv $@.tmp.owl $@; fi -.PRECIOUS: mirror/pr.owl + $(ROBOT) remove -i $@.tmp.owl --base-iri $(URIBASE)/PR --axioms external --preserve-structure false --trim false -o $@.tmp.owl && mv $@.tmp.owl $(TMPDIR)/$@.owl; fi ## ONTOLOGY: go -## Copy of go is re-downloaded whenever source changes -mirror/go.trigger: $(SRC) - -mirror/go.owl: mirror/go.trigger - if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/go/go-base.owl --create-dirs -o mirror/go.owl --retry 4 --max-time 200 && $(ROBOT) convert -i mirror/go.owl -o $@.tmp.owl && mv $@.tmp.owl $@; fi -.PRECIOUS: mirror/go.owl +.PHONY: mirror-go +.PRECIOUS: $(MIRRORDIR)/go.owl +mirror-go: + if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/go/go-base.owl --create-dirs -o $(MIRRORDIR)/go.owl --retry 4 --max-time 200 && $(ROBOT) convert -i $(MIRRORDIR)/go.owl -o $@.tmp.owl && mv $@.tmp.owl $(TMPDIR)/$@.owl; fi ## ONTOLOGY: ro -## Copy of ro is re-downloaded whenever source changes -mirror/ro.trigger: $(SRC) - -mirror/ro.owl: mirror/ro.trigger - if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/ro/ro-base.owl --create-dirs -o mirror/ro.owl --retry 4 --max-time 200 && $(ROBOT) convert -i mirror/ro.owl -o $@.tmp.owl && mv $@.tmp.owl $@; fi -.PRECIOUS: mirror/ro.owl +.PHONY: mirror-ro +.PRECIOUS: $(MIRRORDIR)/ro.owl +mirror-ro: + if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/ro/ro-base.owl --create-dirs -o $(MIRRORDIR)/ro.owl --retry 4 --max-time 200 && $(ROBOT) convert -i $(MIRRORDIR)/ro.owl -o $@.tmp.owl && mv $@.tmp.owl $(TMPDIR)/$@.owl; fi ## ONTOLOGY: uberon -## Copy of uberon is re-downloaded whenever source changes -mirror/uberon.trigger: $(SRC) - -mirror/uberon.owl: mirror/uberon.trigger - if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/uberon/uberon-base.owl --create-dirs -o mirror/uberon.owl --retry 4 --max-time 200 && $(ROBOT) convert -i mirror/uberon.owl -o $@.tmp.owl && mv $@.tmp.owl $@; fi -.PRECIOUS: mirror/uberon.owl +.PHONY: mirror-uberon +.PRECIOUS: $(MIRRORDIR)/uberon.owl +mirror-uberon: + if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/uberon/uberon-base.owl --create-dirs -o $(MIRRORDIR)/uberon.owl --retry 4 --max-time 200 && $(ROBOT) convert -i $(MIRRORDIR)/uberon.owl -o $@.tmp.owl && mv $@.tmp.owl $(TMPDIR)/$@.owl; fi ## ONTOLOGY: pato -## Copy of pato is re-downloaded whenever source changes -mirror/pato.trigger: $(SRC) - -mirror/pato.owl: mirror/pato.trigger - if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/pato/pato-base.owl --create-dirs -o mirror/pato.owl --retry 4 --max-time 200 && $(ROBOT) convert -i mirror/pato.owl -o $@.tmp.owl && mv $@.tmp.owl $@; fi -.PRECIOUS: mirror/pato.owl +.PHONY: mirror-pato +.PRECIOUS: $(MIRRORDIR)/pato.owl +mirror-pato: + if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/pato/pato-base.owl --create-dirs -o $(MIRRORDIR)/pato.owl --retry 4 --max-time 200 && $(ROBOT) convert -i $(MIRRORDIR)/pato.owl -o $@.tmp.owl && mv $@.tmp.owl $(TMPDIR)/$@.owl; fi ## ONTOLOGY: cl -## Copy of cl is re-downloaded whenever source changes -mirror/cl.trigger: $(SRC) - -mirror/cl.owl: mirror/cl.trigger - if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/cl/cl-base.owl --create-dirs -o mirror/cl.owl --retry 4 --max-time 200 && $(ROBOT) convert -i mirror/cl.owl -o $@.tmp.owl && mv $@.tmp.owl $@; fi -.PRECIOUS: mirror/cl.owl +.PHONY: mirror-cl +.PRECIOUS: $(MIRRORDIR)/cl.owl +mirror-cl: + if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/cl/cl-base.owl --create-dirs -o $(MIRRORDIR)/cl.owl --retry 4 --max-time 200 && $(ROBOT) convert -i $(MIRRORDIR)/cl.owl -o $@.tmp.owl && mv $@.tmp.owl $(TMPDIR)/$@.owl; fi ## ONTOLOGY: ensmusg -## Copy of ensmusg is re-downloaded whenever source changes -mirror/ensmusg.trigger: $(SRC) - -mirror/ensmusg.owl: mirror/ensmusg.trigger - if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/ensmusg.owl --create-dirs -o mirror/ensmusg.owl --retry 4 --max-time 200 && $(ROBOT) convert -i mirror/ensmusg.owl -o $@.tmp.owl && \ - $(ROBOT) remove -i $@.tmp.owl --base-iri $(URIBASE)/ENSMUSG --axioms external --preserve-structure false --trim false -o $@.tmp.owl && mv $@.tmp.owl $@; fi -.PRECIOUS: mirror/ensmusg.owl +.PHONY: mirror-ensmusg +.PRECIOUS: $(MIRRORDIR)/ensmusg.owl +mirror-ensmusg: | $(TMPDIR) + if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(URIBASE)/ensmusg.owl --create-dirs -o $(MIRRORDIR)/ensmusg.owl --retry 4 --max-time 200 && $(ROBOT) convert -i $(MIRRORDIR)/ensmusg.owl -o $@.tmp.owl && \ + $(ROBOT) remove -i $@.tmp.owl --base-iri $(URIBASE)/ENSMUSG --axioms external --preserve-structure false --trim false -o $@.tmp.owl && mv $@.tmp.owl $(TMPDIR)/$@.owl; fi ## ONTOLOGY: ncbitaxon -## Copy of ncbitaxon is re-downloaded whenever source changes -mirror/ncbitaxon.trigger: $(SRC) - -mirror/ncbitaxon.owl: mirror/ncbitaxon.trigger - if [ $(MIR) = true ] && [ $(IMP) = true ]; then $(ROBOT) convert -I http://purl.obolibrary.org/obo/ncbitaxon/subsets/taxslim.owl -o $@.tmp.owl && mv $@.tmp.owl $@; fi -.PRECIOUS: mirror/ncbitaxon.owl +.PHONY: mirror-ncbitaxon +.PRECIOUS: $(MIRRORDIR)/ncbitaxon.owl +mirror-ncbitaxon: | $(TMPDIR) + if [ $(MIR) = true ] && [ $(IMP) = true ]; then $(ROBOT) convert -I http://purl.obolibrary.org/obo/ncbitaxon/subsets/taxslim.owl -o $@.tmp.owl && mv $@.tmp.owl $(TMPDIR)/$@.owl; fi -ALL_MIRRORS = $(patsubst %, mirror/%.owl, $(IMPORTS)) +ALL_MIRRORS = $(patsubst %, $(MIRRORDIR)/%.owl, $(IMPORTS)) MERGE_MIRRORS = true -mirror/merged.owl: $(ALL_MIRRORS) +$(MIRRORDIR)/merged.owl: $(ALL_MIRRORS) if [ $(IMP) = true ] && [ $(MERGE_MIRRORS) = true ]; then $(ROBOT) merge $(patsubst %, -i %, $^) -o $@; fi -.PRECIOUS: mirror/merged.owl +.PRECIOUS: $(MIRRORDIR)/merged.owl + + +$(MIRRORDIR)/%.owl: mirror-% | $(MIRRORDIR) + if [ $(IMP) = true ] && [ $(MIR) = true ]; then if cmp -s $(TMPDIR)/mirror-$*.owl $@ ; then echo "Mirror identical, ignoring."; else echo "Mirrors different, updating." && cp $(TMPDIR)/mirror-$*.owl $@; fi; fi @@ -390,132 +411,127 @@ mirror/merged.owl: $(ALL_MIRRORS) # ---------------------------------------- # Subsets # ---------------------------------------- -subsets/%.tsv: subsets/%.owl +$(SUBSETDIR)/%.tsv: $(SUBSETDIR)/%.owl $(ROBOT) query -f tsv -i $< -s ../sparql/labels.sparql $@ -.PRECIOUS: subsets/%.tsv +.PRECIOUS: $(SUBSETDIR)/%.tsv -subsets/%.owl: $(ONT).owl +$(SUBSETDIR)/%.owl: $(ONT).owl | $(SUBSETDIR) $(OWLTOOLS) $< --extract-ontology-subset --fill-gaps --subset $* -o $@.tmp.owl && mv $@.tmp.owl $@ &&\ $(ROBOT) annotate --input $@ --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $@.tmp.owl && mv $@.tmp.owl $@ -.PRECIOUS: subsets/%.owl +.PRECIOUS: $(SUBSETDIR)/%.owl -subsets/%.obo: subsets/%.owl +$(SUBSETDIR)/%.obo: $(SUBSETDIR)/%.owl $(ROBOT) convert --input $< --check false -f obo $(OBO_FORMAT_OPTIONS) -o $@.tmp.obo && grep -v ^owl-axioms $@.tmp.obo > $@ && rm $@.tmp.obo -subsets/%.json: subsets/%.owl +$(SUBSETDIR)/%.json: $(SUBSETDIR)/%.owl $(ROBOT) convert --input $< --check false -f json -o $@.tmp.json &&\ jq -S 'walk(if type == "array" then sort else . end)' $@.tmp.json > $@ && rm $@.tmp.json -# ---------------------------------------- -# Release -# ---------------------------------------- -# copy from staging area (this directory) to top-level -.PHONY: release -release: $(ONT).owl $(ONT).obo - cp $^ $(RELEASEDIR) && cp imports/* $(RELEASEDIR)/imports - -# ---------------------------------------- -# Sparql queries: Q/C -# ---------------------------------------- - -# these live in the ../sparql directory, and have suffix -violation.sparql -# adding the name here will make the violation check live. -# NOTE: these will soon be phased out and replaced by robot-report - -# run all violation checks -SPARQL_VALIDATION_QUERIES = $(foreach V,$(SPARQL_VALIDATION_CHECKS),$(SPARQLDIR)/$(V)-violation.sparql) -sparql_test: $(SRC) catalog-v001.xml | $(REPORTDIR) -ifneq ($(SPARQL_VALIDATION_QUERIES),) - $(ROBOT) verify --catalog catalog-v001.xml -i $< --queries $(SPARQL_VALIDATION_QUERIES) -O $(REPORTDIR) -endif - -# ---------------------------------------- -# ROBOT report -# ---------------------------------------- - -$(REPORTDIR)/$(SRC)-obo-report.tsv: $(SRCMERGED) | $(REPORTDIR) - $(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) --print 5 -o $@ - -$(REPORTDIR)/%-obo-report.tsv: % | $(REPORTDIR) - $(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) --print 5 -o $@ - -# ---------------------------------------- -# Sparql queries: Exports -# ---------------------------------------- +# --------------------------------------------- +# Sparql queries: Table exports / Query Reports +# --------------------------------------------- SPARQL_EXPORTS_ARGS = $(foreach V,$(SPARQL_EXPORTS),-s $(SPARQLDIR)/$(V).sparql $(REPORTDIR)/$(V).tsv) # This combines all into one single command -.PHONY: all_reports_onestep -all_reports_onestep: $(SRC) | $(REPORTDIR) +.PHONY: custom_reports +custom_reports: $(SRC) | $(REPORTDIR) ifneq ($(SPARQL_EXPORTS_ARGS),) $(ROBOT) query -f tsv -i $< $(SPARQL_EXPORTS_ARGS) endif # ---------------------------------------- -# Patterns (experimental) +# DOSDP Templates/Patterns # ---------------------------------------- -# Test patterns for schema compliance: +PAT=true # Global parameter to bypass pattern generation +ALL_PATTERN_FILES=$(wildcard $(PATTERNDIR)/dosdp-patterns/*.yaml) +ALL_PATTERN_NAMES=$(strip $(patsubst %.yaml,%, $(notdir $(wildcard $(PATTERNDIR)/dosdp-patterns/*.yaml)))) -.PHONY: patterns -patterns: all_imports $(PATTERNDIR)/pattern.owl $(PATTERNDIR)/definitions.owl +PATTERN_CLEAN_FILES=../patterns/all_pattern_terms.txt \ + $(DOSDP_OWL_FILES_DEFAULT) $(DOSDP_TERM_FILES_DEFAULT) \ + +# Note to future generations: prepending ./ is a safety measure to ensure that +# the environment does not malicously set `PATTERN_CLEAN_FILES` to `\`. .PHONY: pattern_clean pattern_clean: - echo "Not implemented" + rm -f $(PATTERN_CLEAN_FILES) + +.PHONY: patterns +patterns dosdp: + echo "Validating all DOSDP templates" + make dosdp_validation + echo "Building $(PATTERNDIR)/definitions.owl" + make $(PATTERNDIR)/pattern.owl $(PATTERNDIR)/definitions.owl + +# DOSDP Template Validation + +$(TMPDIR)/pattern_schema_checks: $(ALL_PATTERN_FILES) | $(TMPDIR) + $(PATTERN_TESTER) $(PATTERNDIR)/dosdp-patterns/ && touch $@ .PHONY: pattern_schema_checks -pattern_schema_checks: update_patterns - $(PATTERN_TESTER) $(PATTERNDIR)/dosdp-patterns/ +pattern_schema_checks dosdp_validation: $(TMPDIR)/pattern_schema_checks -#This command is a workaround for the absence of -N and -i in wget of alpine (the one ODK depend on now). It downloads all patterns specified in external.txt .PHONY: update_patterns -update_patterns: .FORCE - if [ $(PAT) = true ]; then rm -f $(PATTERNDIR)/dosdp-patterns/*.yaml.1 || true; fi - if [ $(PAT) = true ] && [ -s $(PATTERNDIR)/dosdp-patterns/external.txt ]; then wget -i $(PATTERNDIR)/dosdp-patterns/external.txt --backups=1 -P $(PATTERNDIR)/dosdp-patterns; fi - if [ $(PAT) = true ]; then rm -f $(PATTERNDIR)/dosdp-patterns/*.yaml.1 || true; fi +update_patterns: download_patterns + cp -r $(TMPDIR)/dosdp/*.yaml $(PATTERNDIR)/dosdp-patterns +# This command is a workaround for the absence of -N and -i in wget of alpine (the one ODK depend on now). +# It downloads all patterns specified in external.txt +.PHONY: download_patterns +download_patterns: + if [ $(PAT) = true ]; then rm -f $(TMPDIR)/dosdp/*.yaml.1 || true; fi + if [ $(PAT) = true ] && [ -s $(PATTERNDIR)/dosdp-patterns/external.txt ]; then wget -i $(PATTERNDIR)/dosdp-patterns/external.txt --backups=1 -P $(TMPDIR)/dosdp; fi + if [ $(PAT) = true ]; then rm -f $(TMPDIR)/dosdp/*.yaml.1 || true; fi -$(PATTERNDIR)/pattern.owl: pattern_schema_checks update_patterns - if [ $(PAT) = true ]; then $(DOSDPT) prototype --obo-prefixes true --template=$(PATTERNDIR)/dosdp-patterns --outfile=$@; fi +$(PATTERNDIR)/dospd-patterns/%.yml: download_patterns + if [ $(PAT) = true ] ; then if cmp -s $(TMPDIR)/dosdp-$*.yml $@ ; then echo "DOSDP templates identical."; else echo "DOSDP templates different, updating." && cp $(TMPDIR)/dosdp-$*.yml $@; fi; fi -individual_patterns_default := $(patsubst %.tsv, $(PATTERNDIR)/data/default/%.ofn, $(notdir $(wildcard $(PATTERNDIR)/data/default/*.tsv))) -pattern_term_lists_default := $(patsubst %.tsv, $(PATTERNDIR)/data/default/%.txt, $(notdir $(wildcard $(PATTERNDIR)/data/default/*.tsv))) +# DOSDP Template: Pipelines +# Each pipeline gets its own directory structure +# DOSDP default pipeline +DOSDP_TSV_FILES_DEFAULT = $(wildcard $(PATTERNDIR)/data/default/*.tsv) +DOSDP_OWL_FILES_DEFAULT = $(patsubst %.tsv, $(PATTERNDIR)/data/default/%.ofn, $(notdir $(wildcard $(PATTERNDIR)/data/default/*.tsv))) +DOSDP_TERM_FILES_DEFAULT = $(patsubst %.tsv, $(PATTERNDIR)/data/default/%.txt, $(notdir $(wildcard $(PATTERNDIR)/data/default/*.tsv))) +DOSDP_PATTERN_NAMES_DEFAULT = $(strip $(patsubst %.tsv,%, $(notdir $(wildcard $(PATTERNDIR)/data/default/*.tsv)))) +$(DOSDP_OWL_FILES_DEFAULT): $(SRC) $(DOSDP_TSV_FILES_DEFAULT) $(ALL_PATTERN_FILES) + if [ $(PAT) = true ] && [ "${DOSDP_PATTERN_NAMES_DEFAULT}" ]; then $(DOSDPT) generate --catalog=catalog-v001.xml --infile=$(PATTERNDIR)/data/default/ --template=$(PATTERNDIR)/dosdp-patterns --batch-patterns="$(DOSDP_PATTERN_NAMES_DEFAULT)" --ontology=$< --obo-prefixes=true --outfile=$(PATTERNDIR)/data/default; fi -# Generating the individual pattern modules and merging them into definitions.owl -$(PATTERNDIR)/definitions.owl: prepare_patterns update_patterns dosdp_patterns_default - if [ $(PAT) = true ] && [ "${individual_patterns_names_default}" ] && [ $(PAT) = true ]; then $(ROBOT) merge $(addprefix -i , $(individual_patterns_default)) annotate --ontology-iri $(ONTBASE)/patterns/definitions.owl --version-iri $(ONTBASE)/releases/$(TODAY)/patterns/definitions.owl --annotation owl:versionInfo $(VERSION) -o definitions.ofn && mv definitions.ofn $@; fi -individual_patterns_names_default := $(strip $(patsubst %.tsv,%, $(notdir $(wildcard $(PATTERNDIR)/data/default/*.tsv)))) -dosdp_patterns_default: $(SRC) all_imports .FORCE - if [ $(PAT) = true ] && [ "${individual_patterns_names_default}" ]; then $(DOSDPT) generate --catalog=catalog-v001.xml --infile=$(PATTERNDIR)/data/default/ --template=$(PATTERNDIR)/dosdp-patterns --batch-patterns="$(individual_patterns_names_default)" --ontology=$< --obo-prefixes=true --outfile=$(PATTERNDIR)/data/default; fi +# Generate template file seeds + +## Generate template file seeds +$(PATTERNDIR)/data/default/%.txt: $(PATTERNDIR)/dosdp-patterns/%.yaml $(PATTERNDIR)/data/default/%.tsv + if [ $(PAT) = true ]; then $(DOSDPT) terms --infile=$(word 2, $^) --template=$< --obo-prefixes=true --outfile=$@; fi + # Generating the seed file from all the TSVs. If Pattern generation is deactivated, we still extract a seed from definitions.owl -$(PATTERNDIR)/all_pattern_terms.txt: $(pattern_term_lists_default) $(PATTERNDIR)/pattern_owl_seed.txt - if [ $(PAT) = true ]; then cat $^ | sort | uniq > $@; else $(ROBOT) query --use-graphs true -f csv -i ../patterns/definitions.owl --query ../sparql/terms.sparql $@; fi +$(TMPDIR)/all_pattern_terms.txt: $(DOSDP_TERM_FILES_DEFAULT) $(TMPDIR)/pattern_owl_seed.txt + if [ $(PAT) = true ]; then cat $^ | sort | uniq > $@; else $(ROBOT) query --use-graphs true -f csv -i $(PATTERNDIR)/definitions.owl --query ../sparql/terms.sparql $@; fi -$(PATTERNDIR)/pattern_owl_seed.txt: $(PATTERNDIR)/pattern.owl +$(TMPDIR)/pattern_owl_seed.txt: $(PATTERNDIR)/pattern.owl if [ $(PAT) = true ]; then $(ROBOT) query --use-graphs true -f csv -i $< --query ../sparql/terms.sparql $@; fi -$(PATTERNDIR)/data/default/%.txt: $(PATTERNDIR)/dosdp-patterns/%.yaml $(PATTERNDIR)/data/default/%.tsv .FORCE - if [ $(PAT) = true ]; then $(DOSDPT) terms --infile=$(word 2, $^) --template=$< --obo-prefixes=true --outfile=$@; fi - -.PHONY: prepare_patterns -prepare_patterns: - if [ $(PAT) = true ]; then touch $(PATTERNDIR)/data $(pattern_term_lists_default) ; fi - if [ $(PAT) = true ]; then touch $(PATTERNDIR)/data $(individual_patterns_default) ; fi +# Pattern pipeline main targets: the generated OWL files +# Create pattern.owl, an ontology of all DOSDP patterns +$(PATTERNDIR)/pattern.owl: $(ALL_PATTERN_FILES) + if [ $(PAT) = true ]; then $(DOSDPT) prototype --obo-prefixes true --template=$(PATTERNDIR)/dosdp-patterns --outfile=$@; fi +# Generating the individual pattern modules and merging them into definitions.owl +$(PATTERNDIR)/definitions.owl: $(DOSDP_OWL_FILES_DEFAULT) + if [ $(PAT) = true ] && [ "${DOSDP_PATTERN_NAMES_DEFAULT}" ] && [ $(PAT) = true ]; then $(ROBOT) merge $(addprefix -i , $^) \ + annotate --ontology-iri $(ONTBASE)/patterns/definitions.owl --version-iri $(ONTBASE)/releases/$(TODAY)/patterns/definitions.owl --annotation owl:versionInfo $(VERSION) -o definitions.ofn && mv definitions.ofn $@; fi @@ -591,7 +607,7 @@ $(ONT)-base.owl: $(SRC) $(OTHER_SRC) --output $@.tmp.owl && mv $@.tmp.owl $@ # Full: The full artefacts with imports merged, reasoned -$(ONT)-full.owl: $(SRC) $(OTHER_SRC) +$(ONT)-full.owl: $(SRC) $(OTHER_SRC) $(IMPORT_FILES) $(ROBOT) merge --input $< \ reason --reasoner ELK --equivalent-classes-allowed asserted-only --exclude-tautologies structural \ relax \ @@ -602,7 +618,7 @@ $(ONT)-full.owl: $(SRC) $(OTHER_SRC) # drop every axiom: filter --term-file keep_terms.txt --trim true # remove --select imports --trim false \ -$(ONT)-simple.owl: $(SRC) $(OTHER_SRC) $(SIMPLESEED) +$(ONT)-simple.owl: $(SRC) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILES) $(ROBOT) merge --input $< $(patsubst %, -i %, $(OTHER_SRC)) \ reason --reasoner ELK --equivalent-classes-allowed asserted-only --exclude-tautologies structural \ relax \ @@ -616,7 +632,7 @@ $(ONT)-simple.owl: $(SRC) $(OTHER_SRC) $(SIMPLESEED) # Should this be the non-classified ontology with the drop foreign axiom filter? # Consider adding remove --term "http://www.geneontology.org/formats/oboInOwl#hasOBONamespace" -$(ONT)-simple-non-classified.owl: $(SRC) $(OTHER_SRC) $(ONTOLOGYTERMS) +$(ONT)-simple-non-classified.owl: $(SRC) $(OTHER_SRC) $(ONTOLOGYTERMS) $(IMPORT_FILES) $(ROBOT) remove --input $< --select imports --trim true \ merge $(patsubst %, -i %, $(OTHER_SRC)) \ remove --axioms equivalent \ @@ -632,12 +648,30 @@ explain_unsat: $(SRC) +# ---------------------------------------- +# General Validation +# ---------------------------------------- +TSV= +ALL_TSV_FILES=$(DOSDP_TSV_FILES_DEFAULT) + +validate-tsv: $(TSV) | $(TMPDIR) + for FILE in $< ; do \ + tsvalid $$FILE > $(TMPDIR)/validate.txt; \ + if [ -s $(TMPDIR)/validate.txt ]; then cat $(TMPDIR)/validate.txt && exit 1; fi ; \ + done + +validate-all-tsv: $(ALL_TSV_FILES) + make validate-tsv TSV="$^" + +# ---------------------------------------- +# Editors Utilities +# ---------------------------------------- + .PHONY: normalize_src normalize_src: $(SRC) - $(ROBOT) convert -i $< -f ofn -o tmp/normalise && mv tmp/normalise $< - + $(ROBOT) convert -i $< -f ofn -o $(TMPDIR)/normalise && mv $(TMPDIR)/normalise $< .PHONY: validate_idranges validate_idranges: @@ -649,4 +683,81 @@ update_repo: +# Note to future generations: prepending ./ is a safety measure to ensure that +# the environment does not malicously set `CLEANFILES` to `\`. +.PHONY: clean +clean: + make pattern_clean + [ -n "$(MIRRORDIR)" ] && [ $(MIRRORDIR) != "." ] && [ $(MIRRORDIR) != "/" ] && [ $(MIRRORDIR) != ".." ] && [ -d ./$(MIRRORDIR) ] && rm -rf ./$(MIRRORDIR)/* + [ -n "$(TMPDIR)" ] && [ $(TMPDIR) != "." ] && [ $(TMPDIR) != "/" ] && [ $(TMPDIR) != ".." ] && [ -d ./$(TMPDIR) ] && rm -rf ./$(TMPDIR)/* + rm -f $(CLEANFILES) + +.PHONY: help +help: + @echo "$$data" + +define data +Usage: [IMAGE=(odklite|odkfull)] [ODK_DEBUG=yes] sh run.sh make [(IMP|MIR|IMP_LARGE|PAT)=(false|true)] command + +---------------------------------------- + Command reference +---------------------------------------- + +Core commands: +* prepare_release: Run the entire release pipeline. Use make IMP=false prepare_release to avoid rerunning the imports +* update_repo: Update the ODK repository setup using the config file bdso-odk.yaml +* test: Running all validation tests +* odkversion: Show the current version of the ODK Makefile and ROBOT. +* clean: Delete all temporary files +* help: Print ODK Usage information + + +Imports management: +* refresh-imports: Refresh all imports and mirrors. +* no-mirror-refresh-imports: Refresh all imports without downloading mirrors. +* refresh-imports-excluding-large: Refresh all imports and mirrors, but skipping the ones labelled as 'is_large'. +* refresh-%: Refresh a single import, i.e. refresh-go will refresh 'imports/go_import.owl'. +* no-mirror-refresh-%: Refresh a single import without updating the mirror, i.e. refresh-go will refresh 'imports/go_import.owl'. +* mirror-%: Refresh a single mirror. + +DOSDP templates +* dosdp: Run the DOSDP patterns pipeline: Run tests, then build OWL files from the tables. +* patterns: Alias of the 'dosdp' command +* pattern_clean: Delete all temporary pattern files +* dosdp_validation: Run all validation checks on DOSDP template files and tables +* pattern_schema_checks: Alias of the 'dosdp_validation' command +* update_patterns: Pull updated patterns listed in dosdp-patterns/external.txt +* dosdp-matches-%: Run the DOSDP matches/query pipeline as configured in your bdso-odk.yaml file. + +Editor utilities: +* validate_idranges: Make sure your ID ranges file is formatted correctly +* normalize_src: Load and safe your bdso-edit file after you to make sure its serialised correctly +* explain_unsat: If you have unsatisfiable classes, this command will create a markdown file (tmp/explain_unsat.md) which will explain all your unsatisfiable classes +* validate-all-tsv: Check all your tsv files for possible problems in syntax. Use ALL_TSV_FILES variable to list files +* validate-tsv: Check a tsv file for syntactic problems with tsvalid. Use TSV variable to pass filepath, e.g. make TSV=../my.tsv validate-tsv. + +Additional build commands (advanced users) +* all: Run the entire pipeline (like prepare_release), but without copying the release files to the release directory. +* all_subsets: Build all subsets +* custom_reports: Generate all custom sparql reports you have configured in your bdso-odk.yaml file. +* all_assets: Build all assets +* show_assets: Print a list of all assets that would be build by the release pipeline + +Additional QC commands (advanced users) +* robot_reports: Run all configured ROBOT reports +* validate_profile_%: Run an OWL2 DL profile validation check, for example validate_profile_bdso-edit.owl. + +Examples: +* sh run.sh make IMP=false prepare_release +* sh run.sh make update_repo +* sh run.sh make test + +Tricks: +* Add -B to the end of your command to force re-running it even if nothing has changed +* Use the IMAGE parameter to the run.sh script to use a different image like odklite +* Use ODK_DEBUG=yes sh run.sh make ... to print information about timing and debugging + +endef +export data + include bdso.Makefile \ No newline at end of file diff --git a/src/ontology/bdso.Makefile b/src/ontology/bdso.Makefile index 3f0ca0b9..6222dac4 100644 --- a/src/ontology/bdso.Makefile +++ b/src/ontology/bdso.Makefile @@ -30,7 +30,7 @@ PCL_LEGACY_FILE = components/pcl-legacy.owl #TEMPLATE_CLASS_FILES = $(patsubst %, ../templates/_%class.tsv, $(JOBS)) # overriding to add prefixes -$(PATTERNDIR)/pattern.owl: pattern_schema_checks update_patterns +$(PATTERNDIR)/pattern.owl: if [ $(PAT) = true ]; then $(DOSDPT) prototype --prefixes=template_prefixes.yaml --obo-prefixes true --template=$(PATTERNDIR)/dosdp-patterns --outfile=$@; fi individual_patterns_names_default := $(strip $(patsubst %.tsv,%, $(notdir $(wildcard $(PATTERNDIR)/data/default/*.tsv)))) @@ -221,8 +221,8 @@ $(ONT)-pcl-comp.json: $(RELEASEDIR)/$(ONT)-pcl-comp.owl convert --check false -f json -o $@.tmp.json &&\ jq -S 'walk(if type == "array" then sort else . end)' $@.tmp.json > $(RELEASEDIR)/$@ && rm $@.tmp.json - -# skip schema checks for now, because odk using the wrong validator -.PHONY: pattern_schema_checks -pattern_schema_checks: update_patterns - if [ $(PAT) = "skip" ]; then $(PATTERN_TESTER) $(PATTERNDIR)/dosdp-patterns/ ; fi \ No newline at end of file +# New ODK should remove the need for this +## skip schema checks for now, because odk using the wrong validator +##.PHONY: pattern_schema_checks +##pattern_schema_checks: update_patterns +## if [ $(PAT) = "skip" ]; then $(PATTERN_TESTER) $(PATTERNDIR)/dosdp-patterns/ ; fi \ No newline at end of file diff --git a/src/sparql/bdso_terms.sparql b/src/sparql/bdso_terms.sparql index 6ecc01db..4a4b18e6 100644 --- a/src/sparql/bdso_terms.sparql +++ b/src/sparql/bdso_terms.sparql @@ -3,5 +3,5 @@ WHERE { { ?s1 ?p1 ?term . } UNION { ?term ?p2 ?o2 . } - FILTER(isIRI(?term) && (regex(str(?term), UCASE("bdso_")))) + FILTER(isIRI(?term) && (STRSTARTS(str(?term), "http://purl.obolibrary.org/obo/BDSO_"))) } diff --git a/src/sparql/iri-range-violation.sparql b/src/sparql/iri-range-violation.sparql new file mode 100644 index 00000000..796ffa64 --- /dev/null +++ b/src/sparql/iri-range-violation.sparql @@ -0,0 +1,19 @@ +PREFIX never_in_taxon: +PREFIX present_in_taxon: +PREFIX oboInOwl: +PREFIX dcterms: +PREFIX foaf: + +SELECT ?term ?property ?value +WHERE { + VALUES ?property { + never_in_taxon: + present_in_taxon: + foaf:depicted_by + oboInOwl:inSubset + dcterms:contributor } + ?term ?property ?value . + FILTER(isIRI(?term) && (STRSTARTS(str(?term), "http://purl.obolibrary.org/obo/BDSO_"))) + FILTER (!isIRI(?value)) +} + diff --git a/src/sparql/label-with-iri-violation.sparql b/src/sparql/label-with-iri-violation.sparql new file mode 100644 index 00000000..d4828076 --- /dev/null +++ b/src/sparql/label-with-iri-violation.sparql @@ -0,0 +1,9 @@ +PREFIX rdfs: + +SELECT ?term ?value +WHERE { + ?term rdfs:label ?value . + FILTER (REGEX(?value, "http[s]?[:]")) + FILTER(isIRI(?term) && (STRSTARTS(str(?term), "http://purl.obolibrary.org/obo/BDSO_"))) +} + diff --git a/src/sparql/owldef-self-reference-violation.sparql b/src/sparql/owldef-self-reference-violation.sparql index 63857fea..f2b28b1a 100644 --- a/src/sparql/owldef-self-reference-violation.sparql +++ b/src/sparql/owldef-self-reference-violation.sparql @@ -1,11 +1,12 @@ -prefix rdf: -prefix oio: -prefix owl: -prefix rdfs: +PREFIX rdf: +PREFIX oio: +PREFIX owl: +PREFIX rdfs: -SELECT ?c -WHERE { - { ?c owl:equivalentClass [ owl:intersectionOf [ rdf:rest*/rdf:first ?c ] ] } +SELECT ?term WHERE { + { ?term owl:equivalentClass [ owl:intersectionOf [ rdf:rest*/rdf:first ?term ] ] } UNION - { ?c owl:equivalentClass [ owl:intersectionOf [ rdf:rest*/rdf:first [ owl:someValuesFrom ?c ] ] ] } + { ?term owl:equivalentClass [ owl:intersectionOf [ rdf:rest*/rdf:first [ owl:someValuesFrom ?term ] ] ] } + FILTER(isIRI(?term) && (STRSTARTS(str(?term), "http://purl.obolibrary.org/obo/BDSO_"))) } +