Merge pull request #503 from NCEAS/develop

jeanetteclark · web-flow · commit 96fee8c90935 · 2025-07-16T15:30:27.000-07:00
Patch release for data quality suite
diff --git a/Docker/metadig-worker/requirements.txt b/Docker/metadig-worker/requirements.txt
@@ -10,4 +10,5 @@ meson-python
 ninja
 yara-python
 xarray[io]
+cftime
 metadig @ git+https://github.com/NCEAS/metadig-py.git
diff --git a/helm/metadig-controller/Chart.yaml b/helm/metadig-controller/Chart.yaml
@@ -21,4 +21,4 @@ version: 1.0.7
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v.3.1.1"
+appVersion: "v.3.1.2"
diff --git a/helm/metadig-controller/config/metadig.properties b/helm/metadig-controller/config/metadig.properties
@@ -62,6 +62,7 @@ quartz.monitor.run.limit = 10
 # jep library
 jep.path = /usr/local/lib/python3.10/dist-packages/jep
 # arctic hashstore
+store.store_type = HashStore
 store.store_path = /var/data/repos/arctic/metacat/hashstore
 store.store_depth = 3
 store.store_width = 2
diff --git a/helm/metadig-controller/values.yaml b/helm/metadig-controller/values.yaml
@@ -124,17 +124,13 @@ ingress:
     - api.test.dataone.org
     secretName: ingress-nginx-tls-cert
 
-resources: {}
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-  # limits:
-  #   cpu: 100m
-  #   memory: 128Mi
-  # requests:
-  #   cpu: 100m
-  #   memory: 128Mi
+resources:
+  requests:
+    cpu: 500m
+    memory: 1Gi
+  limits:
+    cpu: 4.0
+    memory: 4Gi
 
 autoscaling:
   enabled: false
diff --git a/helm/metadig-postgres/values.yaml b/helm/metadig-postgres/values.yaml
@@ -58,17 +58,13 @@ service:
   type: ClusterIP
   port: 5432
 
-resources: {}
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-  # limits:
-  #   cpu: 100m
-  #   memory: 128Mi
-  # requests:
-  #   cpu: 100m
-  #   memory: 128Mi
+resources:
+  requests:
+    cpu: 1
+    memory: 8Gi
+  limits:
+    cpu: 4
+    memory: 64Gi
 
 autoscaling:
   enabled: false
diff --git a/helm/metadig-rabbitmq/install-metadig-rabbitmq.sh b/helm/metadig-rabbitmq/install-metadig-rabbitmq.sh
@@ -3,28 +3,8 @@
 # https://github.com/bitnami/charts/tree/master/bitnami/rabbitmq#installing-the-chart
 
 # helm repo add bitnami https://charts.bitnami.com/bitnami
-
 helm upgrade metadig-rabbitmq bitnami/rabbitmq \
---version=12.8.2 \
---namespace metadig \
---set image.registry=docker.io \
---set image.repository=bitnami/rabbitmq \
---set auth.username=metadig \
---set auth.password=quality \
---set replicaCount=3 \
---set podSecurityContext.enabled=false \
---set podSecurityContext.fsGroup=1001 \
---set podManagementPolicy=Parallel \
---set service.type=ClusterIP \
---set service.externalTrafficPolicy=Cluster \
---set ingress.enabled=false \
---set ingress.tls=false \
---set ingress.ingressClassName="nginx" \
---set persistence.enabled=true \
---set persistence.storageClass=csi-rbd-sc \
---set persistence.size=10Gi \
---set volumePermissions.enabled=false \
---set volumePermissions.containerSecurityContext.runAsUser=1001 \
---set serviceAccount.create=false \
---set serviceAccount.name="metadig" \
---set tls.enabled=false
+  --version=12.8.2 \
+  --namespace metadig \
+  -f rabbitmq-values.yaml
+
diff --git a/helm/metadig-rabbitmq/rabbitmq-values.yaml b/helm/metadig-rabbitmq/rabbitmq-values.yaml
@@ -0,0 +1,49 @@
+image:
+  registry: docker.io
+  repository: bitnami/rabbitmq
+
+auth:
+  username: metadig
+  password: quality
+
+replicaCount: 6
+
+podSecurityContext:
+  enabled: false
+  fsGroup: 1001
+
+podManagementPolicy: Parallel
+
+service:
+  type: ClusterIP
+  externalTrafficPolicy: Cluster
+
+resources:
+  requests:
+    cpu: 1
+    memory: 2Gi
+  limits:
+    cpu: 2
+    memory: 4Gi
+
+ingress:
+  enabled: false
+  tls: false
+  ingressClassName: nginx
+
+persistence:
+  enabled: true
+  storageClass: csi-rbd-sc
+  size: 10Gi
+
+volumePermissions:
+  enabled: false
+  containerSecurityContext:
+    runAsUser: 1001
+
+serviceAccount:
+  create: false
+  name: metadig
+
+tls:
+  enabled: false
diff --git a/helm/metadig-scheduler/Chart.yaml b/helm/metadig-scheduler/Chart.yaml
@@ -21,4 +21,4 @@ version: 1.0.7
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v.3.1.1"
+appVersion: "v.3.1.2"
diff --git a/helm/metadig-scheduler/config/taskList.csv b/helm/metadig-scheduler/config/taskList.csv
@@ -27,8 +27,8 @@ quality,quality-cerp_sfwmd,metadig,20 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suit
 quality,quality-CA_OPC,metadig,25 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:CA_OPC;2010-10-07T00:00:00.00Z;1;1000;"
 quality,quality-sctld,metadig,30 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:SCTLD;2010-10-07T00:00:00.00Z;1;1000;"
 quality,quality-drp,metadig,35 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:DRP;2010-10-07T00:00:00.00Z;1;1000;"
-quality,quality-si,metadig,45 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:SI;2010-10-07T00:00:00.00Z;1;1000;"
-#quality,quality-data-suite,metadig,55 0/5 * * * ?,"^eml.*|^http.*eml.*;data-suite-0.1.0;urn:node:ARCTIC;1900-11-01T00:00:00.00Z;1;1000"
+#quality,quality-si,metadig,45 0/1 * * * ?,"^eml.*|^http.*eml.*;FAIR-suite-0.4.0;urn:node:SI;2010-10-07T00:00:00.00Z;1;1000;"
+quality,quality-data-suite,metadig,55 0/5 * * * ?,"^eml.*|^http.*eml.*;data-suite-0.1.0;urn:node:ARCTIC;1900-11-01T00:00:00.00Z;1;1000"
 #
 # Portal scoring tasks
 score,portal-KNB-FAIR,metadig,1 0/1 * * * ?,"*portals*;FAIR-suite-0.4.0;urn:node:KNB;2020-08-10T00:00:00.00Z;1;100;portal"
diff --git a/helm/metadig-scheduler/values.yaml b/helm/metadig-scheduler/values.yaml
@@ -51,17 +51,13 @@ service:
 ingress:
   enabled: false
 
-resources: {}
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-  # limits:
-  #   cpu: 100m
-  #   memory: 128Mi
-  # requests:
-  #   cpu: 100m
-  #   memory: 128Mi
+resources:
+  requests:
+    cpu: 500m
+    memory: 1Gi
+  limits:
+    cpu: 4.0
+    memory: 4Gi
 
 autoscaling:
   enabled: false
diff --git a/helm/metadig-scorer/Chart.yaml b/helm/metadig-scorer/Chart.yaml
@@ -21,4 +21,4 @@ version: 1.0.7
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v.3.1.1"
+appVersion: "v.3.1.2"
diff --git a/helm/metadig-scorer/values.yaml b/helm/metadig-scorer/values.yaml
@@ -51,17 +51,13 @@ service:
 ingress:
   enabled: false
 
-resources: {}
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-  # limits:
-  #   cpu: 100m
-  #   memory: 128Mi
-  # requests:
-  #   cpu: 100m
-  #   memory: 128Mi
+resources:
+  requests:
+    cpu: 500m
+    memory: 1Gi
+  limits:
+    cpu: 4.0
+    memory: 4Gi
 
 autoscaling:
   enabled: false
diff --git a/helm/metadig-solr/install-metadig-solr.sh b/helm/metadig-solr/install-metadig-solr.sh
@@ -5,41 +5,6 @@
 #helm repo add bitnami https://charts.bitnami.com/bitnami
 
 helm install metadig-solr bitnami/solr \
---version=3.0.3 \
---namespace metadig \
---set auth.enabled=false \
---set coreNames="quality" \
---set collection="quality" \
---set cloudEnabled=false \
---set image.tag=8.11.1-debian-10-r153 \
---set collectionShards=1 \
---set collectionReplicas=1 \
---set podSecurityContext.enabled=true \
---set podSecurityContext.fsGroup=1001 \
---set containerSecurityContext.enabled=true \
---set containerSecurityContext.runAsUser=1001 \
---set replicaCount=1 \
---set service.type=ClusterIP \
---set service.externalTrafficPolicy=Cluster \
---set ingress.enabled=false \
---set ingress.tls=false \
---set ingress.ingressClassName="nginx" \
---set ingress.hostname="solr.local" \
---set persistence.enabled=true \
---set persistence.mountPath=/bitnami/solr \
---set persistence.storageClass=csi-rbd-sc \
---set persistence.size=20Gi \
---set persistence.mountPath=/bitnami/solr \
---set volumePermissions.enabled=false \
---set volumePermissions.containerSecurityContext.runAsUser=1001 \
---set serviceAccount.create=false \
---set serviceAccount.name="metadig" \
---set tls.enabled=false \
---set tls.autoGenerated=false \
---set tls.certificatesSecretName="ingress-nginx-tls-cert-jks" \
---set tls.passwordsSecretName="metadig-solr/jks-password-secret" \
---set tls.keystorePassword="metadig" \
---set tls.truststorePassword="metadig" \
---set zookeeper.enabled=false \
---set zookeeper.persistence.enabled=false \
---set zookeeper.persistence.storageClass=""
+  --version=3.0.3 \
+  --namespace metadig \
+  -f solr-values.yaml
diff --git a/helm/metadig-solr/solr-values.yaml b/helm/metadig-solr/solr-values.yaml
@@ -0,0 +1,69 @@
+auth:
+  enabled: false
+
+coreNames: "quality"
+collection: "quality"
+cloudEnabled: false
+
+image:
+  tag: 8.11.1-debian-10-r153
+
+collectionShards: 1
+collectionReplicas: 1
+
+podSecurityContext:
+  enabled: true
+  fsGroup: 1001
+
+containerSecurityContext:
+  enabled: true
+  runAsUser: 1001
+
+replicaCount: 1
+
+service:
+  type: ClusterIP
+  externalTrafficPolicy: Cluster
+
+ingress:
+  enabled: false
+  tls: false
+  ingressClassName: "nginx"
+  hostname: "solr.local"
+
+persistence:
+  enabled: true
+  mountPath: /bitnami/solr
+  storageClass: csi-rbd-sc
+  size: 20Gi
+
+volumePermissions:
+  enabled: false
+  containerSecurityContext:
+    runAsUser: 1001
+
+serviceAccount:
+  create: false
+  name: "metadig"
+
+tls:
+  enabled: false
+  autoGenerated: false
+  certificatesSecretName: "ingress-nginx-tls-cert-jks"
+  passwordsSecretName: "metadig-solr/jks-password-secret"
+  keystorePassword: "metadig"
+  truststorePassword: "metadig"
+
+resources:
+  requests:
+    cpu: 1.0
+    memory: 2Gi
+  limits:
+    cpu: 1.5
+    memory: 3Gi
+
+zookeeper:
+  enabled: false
+  persistence:
+    enabled: false
+    storageClass: ""
diff --git a/helm/metadig-worker/Chart.yaml b/helm/metadig-worker/Chart.yaml
@@ -21,4 +21,4 @@ version: 1.0.7
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v.3.1.1"
+appVersion: "v.3.1.2"
diff --git a/helm/metadig-worker/templates/deployment.yaml b/helm/metadig-worker/templates/deployment.yaml
@@ -43,6 +43,8 @@ spec:
               secretKeyRef:
                 name: dataone-token
                 key: DataONEauthToken
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:
           - name: metadig-pv
             mountPath: {{ .Values.persistence.metadig.mountPath }}
diff --git a/helm/metadig-worker/values.yaml b/helm/metadig-worker/values.yaml
diff --git a/pom.xml b/pom.xml
diff --git a/src/main/java/edu/ucsb/nceas/mdqengine/DataONE.java b/src/main/java/edu/ucsb/nceas/mdqengine/DataONE.java
diff --git a/src/main/java/edu/ucsb/nceas/mdqengine/MDQEngine.java b/src/main/java/edu/ucsb/nceas/mdqengine/MDQEngine.java
diff --git a/src/main/java/edu/ucsb/nceas/mdqengine/Worker.java b/src/main/java/edu/ucsb/nceas/mdqengine/Worker.java
diff --git a/src/main/java/edu/ucsb/nceas/mdqengine/scheduler/RequestReportJob.java b/src/main/java/edu/ucsb/nceas/mdqengine/scheduler/RequestReportJob.java
