Skip to content

Commit a01bcb8

Browse files
Merge pull request #501 from NCEAS/develop
3.1.1 release
2 parents 515a912 + ad8f058 commit a01bcb8

File tree

15 files changed

+39
-94
lines changed

15 files changed

+39
-94
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ test/**
66
metadig-engine.iml
77
secret
88
demo/**
9+
.vscode/*

Docker/metadig-worker/Dockerfile

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ RUN mkdir -p /var/data/repos
1313
# Set the working directory
1414
WORKDIR /var/lib/metadig
1515

16-
ARG ENGINE_TAG=3.0.1
16+
ARG ENGINE_TAG=3.2.0-SNAPSHOT
1717
ENV ENGINE_TAG=${ENGINE_TAG}
1818

1919
# The most recently built jar file is copied from the maven build directory to this dir by maven, so that
@@ -26,7 +26,7 @@ COPY target/metadig-engine-$ENGINE_TAG.jar metadig-engine-$ENGINE_TAG.jar
2626
COPY Docker/metadig-worker/solr solr/
2727

2828
RUN apt-get update
29-
RUN apt-get install -y g++ git r-base r-base-dev r-base-core r-doc-html libc6-dev libssl-dev libxml2 libxml2-dev libcurl4-openssl-dev file libmagic-dev python3 python3-pip python3-dev openjdk-17-jdk
29+
RUN apt-get install -y g++ git r-base r-base-dev r-base-core r-doc-html libc6-dev libssl-dev libxml2 libxml2-dev libcurl4-openssl-dev file libjansson4 libmagic-mgc libmagic1 libyara8 yara python3 python3-pip python3-dev openjdk-17-jdk
3030

3131
# Set Java environment variables
3232
ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
@@ -36,15 +36,22 @@ ENV PATH="$PATH:$JAVA_HOME/bin"
3636
RUN echo 'options(repos = c(CRAN = "http://cran.rstudio.com"))' >> /usr/lib/R/etc/Rprofile.site
3737
RUN Rscript -e "install.packages(c('remotes', 'stringr', 'jsonlite'))"
3838
RUN Rscript -e "remotes::install_github('NCEAS/metadig-r', ref = 'v.0.2.0')"
39-
RUN Rscript -e "remotes::install_github('NCEAS/metadig-rake', ref = 'develop')"
4039

4140
# DataONE indexer prints copious error msgs if these files don't exist
4241
RUN mkdir -p /etc/dataone/index && touch /etc/dataone/index/d1client.properties && touch /etc/dataone/node.properties && touch /etc/dataone/index/cn-index-processor.properties
4342

4443
COPY Docker/metadig-worker/requirements.txt /opt/local/metadig/
45-
RUN pip install cython meson meson-python ninja
44+
4645
RUN pip install -r /opt/local/metadig/requirements.txt
4746

47+
# get the list of yara rules
48+
ADD https://github.com/YARAHQ/yara-forge/releases/download/20250615/yara-forge-rules-core.zip .
49+
50+
RUN unzip yara-forge-rules-core
51+
52+
RUN python3 -c "import yara; r = yara.compile(filepath='/var/lib/metadig/packages/core/yara-rules-core.yar'); r.save('/var/lib/metadig/yara-compiled.yc')"
53+
RUN chmod o+rx /var/lib/metadig/yara-compiled.yc
54+
4855
# Run the Worker process
4956
# Note: docker --buile-arg only allows one argument (one token only, multiple tokens inside quotes doesn't work, so have
5057
# to specify java options directly on command line.
Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,13 @@
11
numpy
2-
metadig @ git+https://github.com/NCEAS/metadig-py.git
32
requests
43
pandas
54
python-magic
6-
tabulate
5+
tabulate
6+
chardet
7+
cython
8+
meson
9+
meson-python
10+
ninja
11+
yara-python
12+
xarray[io]
13+
metadig @ git+https://github.com/NCEAS/metadig-py.git

helm/metadig-controller/Chart.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,4 @@ version: 1.0.7
2121
# incremented each time you make changes to the application. Versions are not expected to
2222
# follow Semantic Versioning. They should reflect the version the application is using.
2323
# It is recommended to use it with quotes.
24-
appVersion: "v.3.1.0"
24+
appVersion: "v.3.1.1"

helm/metadig-controller/config.dev/metadig.properties

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,12 @@ quartz.monitor.run.limit = 10
3939
jep.path = /usr/local/lib/python3.10/dist-packages/jep
4040
# Path to the downloadsList.csv to retrieve web resources
4141
downloadsList = /opt/local/metadig/data/downloadsList.csv
42-
# HashStore properties - Manual for dev.nceas
42+
# HashStore properties
4343
store.store_type = HashStore
44-
store.store_path = /var/data/repos/dev/metacat/hashstore
44+
# Path for knb dev.nceas
45+
# store.store_path = /var/data/repos/dev/metacat/hashstore
46+
# Path for test adc
47+
store.store_path = /var/data/repos/arctic/metacat/hashstore
4548
store.store_depth = 3
4649
store.store_width = 2
4750
store.store_algorithm = SHA-256

helm/metadig-controller/config/metadig.properties

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,3 +61,9 @@ quartz.monitor.processing.time = 12
6161
quartz.monitor.run.limit = 10
6262
# jep library
6363
jep.path = /usr/local/lib/python3.10/dist-packages/jep
64+
# arctic hashstore
65+
store.store_path = /var/data/repos/arctic/metacat/hashstore
66+
store.store_depth = 3
67+
store.store_width = 2
68+
store.store_algorithm = SHA-256
69+
store.store_metadata_namespace = https://ns.dataone.org/service/types/v2.0#SystemMetadata

helm/metadig-scheduler/Chart.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,4 @@ version: 1.0.7
2121
# incremented each time you make changes to the application. Versions are not expected to
2222
# follow Semantic Versioning. They should reflect the version the application is using.
2323
# It is recommended to use it with quotes.
24-
appVersion: "v.3.1.0"
24+
appVersion: "v.3.1.1"

helm/metadig-scheduler/config/taskList.csv

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ quality,quality-CA_OPC,metadig,25 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.
2828
quality,quality-sctld,metadig,30 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:SCTLD;2010-10-07T00:00:00.00Z;1;1000;"
2929
quality,quality-drp,metadig,35 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:DRP;2010-10-07T00:00:00.00Z;1;1000;"
3030
quality,quality-si,metadig,45 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:SI;2010-10-07T00:00:00.00Z;1;1000;"
31+
#quality,quality-data-suite,metadig,55 0/5 * * * ?,"^eml.*|^http.*eml.*;data-suite-0.1.0;urn:node:ARCTIC;1900-11-01T00:00:00.00Z;1;1000"
3132
#
3233
# Portal scoring tasks
3334
score,portal-KNB-FAIR,metadig,1 0/1 * * * ?,"*portals*;FAIR-suite-0.4.0;urn:node:KNB;2020-08-10T00:00:00.00Z;1;100;portal"

helm/metadig-scheduler/pv.yaml

Lines changed: 0 additions & 26 deletions
This file was deleted.

helm/metadig-scheduler/pvc.yaml

Lines changed: 0 additions & 14 deletions
This file was deleted.

0 commit comments

Comments
 (0)