Merge pull request #388 from INCATools/odk-1.2.26

ODK 1.2.26

Author: matentzn

Changes.md

@@ -1,5 +1,31 @@
-# unreleased
-- bumped fastobo validator to new version ([pull](https://github.com/INCATools/ontology-development-kit/pull/379)
+
+# v1.2.26 (2 February 2021)
+- New versions:
+  - fastobo validator to new version ([pull](https://github.com/INCATools/ontology-development-kit/pull/379)
+  - ROBOT 1.8.1 (lots of new changes, see: [Changelog](https://github.com/ontodev/robot/blob/master/CHANGELOG.md))
+  - DOSDPTOOLS 0.16 (lots of speed-up for bulk pattern generation)
+- New features:
+  - new python dependencies ([cogs](https://github.com/ontodev/cogs), a tool to directly manage tsv files in your repo on Google sheets)
+  - stable serialisation order for JSON files using jq's walk function. -> this decreases the size of the diff for git!
+  - Some improvements to logging when seeding a new repo, to make it easier to find errors
+  - A new method to validate the id-ranges file can be invoked using `sh run.sh make validate_idranges` (after update to latest ODK repo)
+  - modules are now annotated with a dc:source annotation to their original ontology (version)
+- New configuration options:
+  - module_type ([example slme](tests/test-module-star.yaml), [example minimal](tests/test-module-minimal.yaml), [example mireot](tests/test-module-star.yaml)). Direct support for MIREOT and a new module type, minimal.
+  - To encourage stable versions and releases, ODK, by default, merges imports into the release files. Previously, we continued to release the imports as well - which we do not recommend anymore. If you still wish to release your imports as usual, you can set a flag `release_imports` in the `import_group` section of your makefile (see [example](tests/test-release.yaml)).
+  - the same as the above applies for reports (see [example](tests/test-robot-report.yaml))
+  - The custom sparql checks, and the custom sparql exports, are now directly configurable
+    - `custom_sparql_checks` : Chose which additional sparql checks you want to run. The related sparql query must be named CHECKNAME-violation.sparql, and be placed in the src/sparql directory (see [example](tests/test-robot-report.yaml))
+    - `custom_sparql_exports` : Chose which additional sparql checks you want to run. The related sparql query must be named CHECKNAME-violation.sparql, and be placed in the src/sparql directory (see [example](tests/test-robot-report.yaml))
+    - `custom_sparql_exports` : Chose which custom reports to generate. The related sparql query must be named CHECKNAME.sparql, and be placed in the src/sparql directory (see [example](tests/test-robot-report.yaml))
+  - `git_main_branch` : The `main` branch for your repo, default `main`, or (now discouraged, previously) `master`.
+  - `ci`: continuous integration defaults; currently available: `travis`, `github_actions`
+  - `create_obo_metadata`: This is mainly for new OBO ontologies. If true, OBO Markdown and PURL configs are created.
+  - `export_project_yaml`: Default `False`. If set to `True`, project.yaml is created in the top level of the repo.
+- Removed a few files from the standard config. This is all part of an effort to slimming down the ODK to the least number of necessary files checked into version control: `src/ontology/Dockerfile`,`src/ontology/patterns.sh`, `src/ontology/release.sh`, `src/ontology/test.sh`, and some temporary files. The `patterns` directory and all its contents only appear now when `use_dosdps`=TRUE.
+- Technical:
+  - Refactored ODK Dockerfile (merged some layers)
+  - added jq 1.6 which is not available via apt-get (yet).
 
 # v1.2.25 (18 November 2020)
 - Updated ROBOT to new version 1.7.2, which includes some hotfixes for ROBOT report and update to whelk 1.0.4

Dockerfile

@@ -4,33 +4,51 @@ FROM ubuntu:18.04
 LABEL maintainer="obo-tools@googlegroups.com" 
 
 ### 2. Get Java, Python and all required system libraries (version control etc)
-ENV JAVA_HOME="/usr/lib/jvm/java-1.8-openjdk"
-#ENV JAVA_HOME /usr/lib/jvm/java-8-oracle
-
-RUN apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:swi-prolog/stable && apt-get upgrade -y \
- && apt-get install -y software-properties-common \
-  build-essential git \
-  openjdk-8-jre openjdk-8-jdk swi-prolog
-
-
-### 3. Python and all required system libraries (version control etc)
+ENV JAVA_HOME="/usr"
+WORKDIR /tools
+ENV PATH "/tools/:$PATH"
+COPY requirements.txt /tools/
+COPY scripts/obodash /tools/
+COPY odk/make-release-assets.py /tools/
 
-RUN apt-get update \
-  && apt-get install -y python3-pip python3-dev subversion make automake gcc g++ unzip rsync curl wget jq openssl git xlsx2csv \
-  && cd /usr/local/bin \
-  && ln -s /usr/bin/python3 python \
-  && pip3 install --upgrade pip setuptools \
-	&& if [ ! -e /usr/bin/pip ]; then ln -s pip3 /usr/bin/pip ; fi \
-	&& if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python3 /usr/bin/python; fi \
-	&& rm -r /root/.cache
+#ENV JAVA_HOME /usr/lib/jvm/java-8-oracle
+# LAYERSIZE ~1000MB
+RUN apt-get update &&\
+  apt-get install -y software-properties-common &&\
+  add-apt-repository ppa:swi-prolog/stable &&\
+  apt-get upgrade -y &&\
+  apt-get install -y build-essential \
+    git \
+    openjdk-8-jre \
+    openjdk-8-jdk \
+    swi-prolog \
+    maven \
+    python3-pip \
+    python3-dev \
+    subversion \
+    make \
+    automake \
+    gcc \
+    g++ \
+    unzip \
+    rsync \
+    curl \
+    wget \
+    jq \
+    openssl \
+    xlsx2csv &&\
+    cd /usr/local/bin \
+    && ln -s /usr/bin/python3 python \
+    && pip3 install --upgrade pip setuptools \
+    && pip3 install -r /tools/requirements.txt \
+  	&& if [ ! -e /usr/bin/pip ]; then ln -s pip3 /usr/bin/pip ; fi \
+  	&& if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python3 /usr/bin/python; fi \
+  	&& rm -r /root/.cache
 
 ### 4. Install custom tools
+#  scripts/droid
+
 
-###### Python libraries ######
-WORKDIR /tools
-ENV PATH "/tools/:$PATH"
-COPY requirements.txt /tools/
-RUN pip3 install -r requirements.txt
 
 ###### owltools & OORT ######
 # For now we get these from jenkins builds, but these should be obtained
@@ -44,20 +62,21 @@ RUN wget https://github.com/owlcollab/owltools/releases/download/$OWLTOOLS/owlto
     chmod +x /tools/owltools-oort-all.jar
 
 ###### Konclude, the DL reasoner ######
+# LAYERSIZE ~28MB
 RUN wget https://github.com/konclude/Konclude/releases/download/v0.6.2-845/Konclude-v0.6.2-845-LinuxAlpine-x64-GCC8.3.0-Static-Qt-5.13.zip -O /tools/konclude.zip && \
     unzip /tools/konclude.zip && \
-    mv /tools/Konclude-v0.6.2-845-LinuxAlpine-x64-GCC8.3.0-Static-Qt-5.13 /tools/konclude_reasoner && \ 
+    mv /tools/Konclude-v0.6.2-845-LinuxAlpine-x64-GCC8.3.0-Static-Qt-5.13 /tools/konclude_reasoner && \
     rm /tools/konclude.zip && \
     chmod +x /tools/konclude_reasoner/Binaries && \
     echo "#!/bin/bash" > /tools/Konclude && \
     echo '/tools/konclude_reasoner/Binaries/Konclude $*' >> /tools/Konclude && \
     chmod +x /tools/Konclude
 
 ###### ROBOT ######
-ENV ROBOT v1.7.2
+ENV ROBOT v1.8.1
 ARG ROBOT_JAR=https://github.com/ontodev/robot/releases/download/$ROBOT/robot.jar
 ENV ROBOT_JAR ${ROBOT_JAR}
-RUN pwd
+# LAYERSIZE ~66MB
 RUN wget $ROBOT_JAR -O /tools/robot.jar && \
     wget https://raw.githubusercontent.com/ontodev/robot/$ROBOT/bin/robot -O /tools/robot && \
     chmod +x /tools/robot && \
@@ -73,15 +92,18 @@ RUN wget https://dl.bintray.com/fastobo/fastobo-validator/$FASTOBO_VALIDATOR/fas
 && chmod +x /tools/fastobo-validator
 
 ##### Ammonite #####
+# LAYERSIZE ~31MB
 RUN (echo "#!/usr/bin/env sh" \
 && curl -L https://github.com/lihaoyi/Ammonite/releases/download/2.0.3/2.13-2.0.3) >/tools/amm \
 && chmod +x /tools/amm
 # Force precompile of ammonite files
+# LAYERSIZE ~67MB
 RUN amm /dev/null
 
 ###### DOSDPTOOLS ######
-ENV DOSDPVERSION=0.14
+ENV DOSDPVERSION=0.16
 ENV PATH "/tools/dosdp-tools/bin:$PATH"
+# LAYERSIZE ~200MB
 RUN wget -nv https://github.com/INCATools/dosdp-tools/releases/download/v$DOSDPVERSION/dosdp-tools-$DOSDPVERSION.tgz \
 && tar -zxvf dosdp-tools-$DOSDPVERSION.tgz \
 && mv dosdp-tools-$DOSDPVERSION /tools/dosdp-tools \
@@ -97,20 +119,34 @@ RUN swipl -g "pack_install(sparqlprog, [interactive(false)])" -g halt
 ENV PATH "/root/.local/share/swi-prolog/pack/sparqlprog/bin:$PATH"
 RUN ln -sf /root/.local/share/swi-prolog/pack/sparqlprog /tools/
 
-COPY scripts/obo-dash.sh /tools/obodash
-RUN cd /tools/ && chmod +x /tools/obodash && git clone https://github.com/OBOFoundry/OBO-Dashboard.git && \
-    cd OBO-Dashboard && git checkout docker-dash && echo "DOCKER DASH BRANCH CHECKED OUT" &&\
+RUN cd /tools/ && chmod +x /tools/obodash && git clone --depth 1 --branch docker-dash https://github.com/OBOFoundry/OBO-Dashboard.git && \
+    cd OBO-Dashboard && git checkout docker-dash && echo "Dashboard: using branch" &&\
     python3 -m pip install -r requirements.txt && echo " " >> Makefile &&\
     echo "build/robot.jar:" >> Makefile &&\
     echo "	echo 'skipped ROBOT jar download' && touch \$@" >> Makefile && echo "" >> Makefile
 
+########## DROID #########
+# LAYERSIZE ~18MB
+#RUN apt-get install -y leiningen
+#ENV DROID_JAR "droid-0.1.0-SNAPSHOT-standalone.jar"
+# LAYERSIZE: ~80MB
+#RUN cd /tools/ && mkdir droid_github && cd /tools/droid_github && git clone https://github.com/ontodev/droid &&\
+#    cd /tools/droid_github/droid && lein uberjar &&\
+#    mv /tools/droid_github/droid/target/uberjar/$DROID_JAR /tools/droid.jar && rm -rf /tools/droid_github &&\
+#    ls -l /tools/
+#RUN chmod +x /tools/droid
+
 ### 5. Install ODK
 ARG ODK_VERSION=0.0.0
 ENV ODK_VERSION=${ODK_VERSION}
 
+### TODO REVIEW THIS. As we speak, jq is official still stalled at 1.5, but for the walk function, we
+### Need 1.6 - once 1.6 is offcial, the following RUN can be removed.
+# LAYERSIZE: ~4MB
+RUN wget https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -O /tools/jq && chmod +x /tools/jq
 
+COPY odk/odk.py /tools/
 COPY template/ /tools/templates/
-COPY odk/ /tools/ 
 RUN chmod +x /tools/*.py
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8

Makefile

@@ -27,17 +27,18 @@ test_go_mini:
 
 TESTS = $(notdir $(wildcard tests/*.yaml))
 TEST_FILES = $(foreach n,$(TESTS), tests/$(n))
-test: custom_tests $(TEST_FILES)
+#TEST_FILES = tests/test-release.yaml
+test: $(TEST_FILES) custom_tests
 	echo "All tests passed successfully!"
-	
+
 tests/*.yaml: .FORCE
 	$(CMD) -c -C $@
 
 schema/project-schema.json:
 	./odk/odk.py dump-schema > $@
 
 # Building docker image
-VERSION = "v1.2.25"
+VERSION = "v1.2.26"
 IM=obolibrary/odkfull
 DEV=obolibrary/odkdev
 ROBOT_JAR="https://build.obolibrary.io/job/ontodev/job/robot/job/cmd-metrics/6/artifact/bin/robot.jar"

configs/omia-odk.yaml

@@ -0,0 +1,16 @@
+id: omia
+title: "Online Inheritance of Animals Ontology"
+github_org: monarch
+repo: omia-ontology
+report_fail_on: none
+use_dosdps: TRUE
+dosdp_tools_options: "--obo-prefixes=true"
+export_formats:
+  - owl
+  - obo
+  - json
+import_group:
+  products:
+    - id: ro
+robot_java_args: '-Xmx8G'
+allow_equivalents: asserted-only