Merge pull request #501 from NCEAS/develop

3.1.1 release

Author: jeanetteclark

.gitignore

@@ -6,3 +6,4 @@ test/**
 metadig-engine.iml
 secret
 demo/**
+.vscode/*

Docker/metadig-worker/Dockerfile

@@ -13,7 +13,7 @@ RUN mkdir -p /var/data/repos
 # Set the working directory 
 WORKDIR /var/lib/metadig
 
-ARG ENGINE_TAG=3.0.1
+ARG ENGINE_TAG=3.2.0-SNAPSHOT
 ENV ENGINE_TAG=${ENGINE_TAG}
 
 # The most recently built jar file is copied from the maven build directory to this dir by maven, so that
@@ -26,7 +26,7 @@ COPY target/metadig-engine-$ENGINE_TAG.jar metadig-engine-$ENGINE_TAG.jar
 COPY Docker/metadig-worker/solr solr/
 
 RUN apt-get update
-RUN apt-get install -y g++ git r-base r-base-dev r-base-core r-doc-html libc6-dev libssl-dev libxml2 libxml2-dev libcurl4-openssl-dev file libmagic-dev python3 python3-pip python3-dev openjdk-17-jdk
+RUN apt-get install -y g++ git r-base r-base-dev r-base-core r-doc-html libc6-dev libssl-dev libxml2 libxml2-dev libcurl4-openssl-dev file libjansson4 libmagic-mgc libmagic1 libyara8 yara python3 python3-pip python3-dev openjdk-17-jdk
 
 # Set Java environment variables
 ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
@@ -36,15 +36,22 @@ ENV PATH="$PATH:$JAVA_HOME/bin"
 RUN echo 'options(repos = c(CRAN = "http://cran.rstudio.com"))' >> /usr/lib/R/etc/Rprofile.site
 RUN Rscript -e "install.packages(c('remotes', 'stringr', 'jsonlite'))"
 RUN Rscript -e "remotes::install_github('NCEAS/metadig-r', ref = 'v.0.2.0')"
-RUN Rscript -e "remotes::install_github('NCEAS/metadig-rake', ref = 'develop')"
 
 # DataONE indexer prints copious error msgs if these files don't exist
 RUN mkdir -p /etc/dataone/index && touch /etc/dataone/index/d1client.properties && touch /etc/dataone/node.properties && touch /etc/dataone/index/cn-index-processor.properties
 
 COPY Docker/metadig-worker/requirements.txt /opt/local/metadig/
-RUN pip install cython meson meson-python ninja
+
 RUN pip install -r /opt/local/metadig/requirements.txt
 
+# get the list of yara rules
+ADD https://github.com/YARAHQ/yara-forge/releases/download/20250615/yara-forge-rules-core.zip .
+
+RUN unzip yara-forge-rules-core
+
+RUN python3 -c "import yara; r = yara.compile(filepath='/var/lib/metadig/packages/core/yara-rules-core.yar'); r.save('/var/lib/metadig/yara-compiled.yc')"
+RUN chmod o+rx /var/lib/metadig/yara-compiled.yc
+
 # Run the Worker process
 # Note: docker --buile-arg only allows one argument (one token only, multiple tokens inside quotes doesn't work, so have
 #   to specify java options directly on command line.

Docker/metadig-worker/requirements.txt

@@ -1,6 +1,13 @@
 numpy
-metadig @ git+https://github.com/NCEAS/metadig-py.git
 requests
 pandas
 python-magic
-tabulate
+tabulate
+chardet
+cython
+meson
+meson-python
+ninja
+yara-python
+xarray[io]
+metadig @ git+https://github.com/NCEAS/metadig-py.git

helm/metadig-controller/Chart.yaml

@@ -21,4 +21,4 @@ version: 1.0.7
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v.3.1.0"
+appVersion: "v.3.1.1"

helm/metadig-controller/config.dev/metadig.properties

@@ -39,9 +39,12 @@ quartz.monitor.run.limit = 10
 jep.path = /usr/local/lib/python3.10/dist-packages/jep
 # Path to the downloadsList.csv to retrieve web resources
 downloadsList = /opt/local/metadig/data/downloadsList.csv
-# HashStore properties - Manual for dev.nceas
+# HashStore properties
 store.store_type = HashStore
-store.store_path = /var/data/repos/dev/metacat/hashstore
+# Path for knb dev.nceas
+# store.store_path = /var/data/repos/dev/metacat/hashstore
+# Path for test adc
+store.store_path = /var/data/repos/arctic/metacat/hashstore
 store.store_depth = 3
 store.store_width = 2
 store.store_algorithm = SHA-256

helm/metadig-controller/config/metadig.properties

@@ -61,3 +61,9 @@ quartz.monitor.processing.time = 12
 quartz.monitor.run.limit = 10
 # jep library
 jep.path = /usr/local/lib/python3.10/dist-packages/jep
+# arctic hashstore
+store.store_path = /var/data/repos/arctic/metacat/hashstore
+store.store_depth = 3
+store.store_width = 2
+store.store_algorithm = SHA-256
+store.store_metadata_namespace = https://ns.dataone.org/service/types/v2.0#SystemMetadata

helm/metadig-scheduler/Chart.yaml

@@ -21,4 +21,4 @@ version: 1.0.7
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v.3.1.0"
+appVersion: "v.3.1.1"

helm/metadig-scheduler/config/taskList.csv

@@ -28,6 +28,7 @@ quality,quality-CA_OPC,metadig,25 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.
 quality,quality-sctld,metadig,30 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:SCTLD;2010-10-07T00:00:00.00Z;1;1000;"
 quality,quality-drp,metadig,35 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:DRP;2010-10-07T00:00:00.00Z;1;1000;"
 quality,quality-si,metadig,45 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:SI;2010-10-07T00:00:00.00Z;1;1000;"
+#quality,quality-data-suite,metadig,55 0/5 * * * ?,"^eml.*|^http.*eml.*;data-suite-0.1.0;urn:node:ARCTIC;1900-11-01T00:00:00.00Z;1;1000"
 #
 # Portal scoring tasks
 score,portal-KNB-FAIR,metadig,1 0/1 * * * ?,"*portals*;FAIR-suite-0.4.0;urn:node:KNB;2020-08-10T00:00:00.00Z;1;100;portal"