diff --git a/assemble/conf/accumulo-env.sh b/assemble/conf/accumulo-env.sh index 52f99ab7642..07cf2cf5ddd 100644 --- a/assemble/conf/accumulo-env.sh +++ b/assemble/conf/accumulo-env.sh @@ -64,7 +64,7 @@ fi ZK_JARS=$(find "$ZOOKEEPER_HOME/lib/" -maxdepth 1 -name '*.jar' -not -name '*slf4j*' -not -name '*log4j*' | paste -sd: -) # lib is set by calling script that sources this env file #shellcheck disable=SC2154 -CLASSPATH="${CLASSPATH}:${lib}/*:${HADOOP_CONF_DIR}:${ZOOKEEPER_HOME}/*:${ZK_JARS}:${HADOOP_HOME}/share/hadoop/client/*" +CLASSPATH="${CLASSPATH}:${lib}/*:${HADOOP_CONF_DIR}:${ZOOKEEPER_HOME}/*:${ZK_JARS}:${HADOOP_HOME}/share/hadoop/client/*:${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/common/lib/*" export CLASSPATH ################################################################## diff --git a/charts/accumulo/templates/accumulo-compactor-deployment.yaml b/charts/accumulo/templates/accumulo-compactor-deployment.yaml index 58ee0bc7d01..55859c33f56 100644 --- a/charts/accumulo/templates/accumulo-compactor-deployment.yaml +++ b/charts/accumulo/templates/accumulo-compactor-deployment.yaml @@ -80,6 +80,8 @@ spec: value: "/opt/accumulo" - name: ACCUMULO_SERVICE_INSTANCE value: "compactor" + - name: ALLUXIO_CLIENT_HOME + value: "/opt/alluxio/client" volumeMounts: - name: accumulo-config mountPath: /opt/accumulo/conf/accumulo.properties @@ -92,6 +94,11 @@ spec: subPath: log4j2-service.properties - name: logs mountPath: /opt/accumulo/logs + - name: alluxio-client + mountPath: /opt/alluxio/client + - name: hadoop-config + mountPath: /opt/hadoop/etc/hadoop/core-site.xml + subPath: core-site.xml resources: {{- toYaml .Values.accumulo.resources.compactor | nindent 10 }} volumes: @@ -101,4 +108,10 @@ spec: defaultMode: 0755 - name: logs emptyDir: {} + - name: alluxio-client + hostPath: + path: /srv/alluxio/client + - name: hadoop-config + configMap: + name: {{ include "accumulo.fullname" . }}-core-site {{- end }} diff --git a/charts/accumulo/templates/accumulo-gc-deployment.yaml b/charts/accumulo/templates/accumulo-gc-deployment.yaml index db9bafdccba..88851c960fc 100644 --- a/charts/accumulo/templates/accumulo-gc-deployment.yaml +++ b/charts/accumulo/templates/accumulo-gc-deployment.yaml @@ -72,6 +72,8 @@ spec: value: "/opt/accumulo" - name: ACCUMULO_SERVICE_INSTANCE value: "gc" + - name: ALLUXIO_CLIENT_HOME + value: "/opt/alluxio/client" volumeMounts: - name: accumulo-config mountPath: /opt/accumulo/conf/accumulo.properties @@ -84,6 +86,11 @@ spec: subPath: log4j2-service.properties - name: logs mountPath: /opt/accumulo/logs + - name: alluxio-client + mountPath: /opt/alluxio/client + - name: hadoop-config + mountPath: /opt/hadoop/etc/hadoop/core-site.xml + subPath: core-site.xml resources: {{- toYaml .Values.accumulo.resources.gc | nindent 10 }} volumes: @@ -93,4 +100,10 @@ spec: defaultMode: 0755 - name: logs emptyDir: {} + - name: alluxio-client + hostPath: + path: /srv/alluxio/client + - name: hadoop-config + configMap: + name: {{ include "accumulo.fullname" . }}-core-site {{- end }} diff --git a/charts/accumulo/templates/accumulo-manager-deployment.yaml b/charts/accumulo/templates/accumulo-manager-deployment.yaml index 6a0b91f3d53..33522b921a7 100644 --- a/charts/accumulo/templates/accumulo-manager-deployment.yaml +++ b/charts/accumulo/templates/accumulo-manager-deployment.yaml @@ -108,6 +108,8 @@ spec: - name: accumulo-config mountPath: /opt/accumulo/conf/log4j2-service.properties subPath: log4j2-service.properties + - name: alluxio-client + mountPath: /opt/alluxio/client containers: - name: manager image: {{ include "accumulo.image" . }} @@ -128,6 +130,8 @@ spec: value: "/opt/accumulo" - name: ACCUMULO_SERVICE_INSTANCE value: "manager" + - name: ALLUXIO_CLIENT_HOME + value: "/opt/alluxio/client" volumeMounts: - name: accumulo-config mountPath: /opt/accumulo/conf/accumulo.properties @@ -140,6 +144,11 @@ spec: subPath: log4j2-service.properties - name: logs mountPath: /opt/accumulo/logs + - name: alluxio-client + mountPath: /opt/alluxio/client + - name: hadoop-config + mountPath: /opt/hadoop/etc/hadoop/core-site.xml + subPath: core-site.xml resources: {{- toYaml .Values.accumulo.resources.manager | nindent 10 }} livenessProbe: @@ -161,4 +170,10 @@ spec: defaultMode: 0755 - name: logs emptyDir: {} + - name: alluxio-client + hostPath: + path: /srv/alluxio/client + - name: hadoop-config + configMap: + name: {{ include "accumulo.fullname" . }}-core-site {{- end }} diff --git a/charts/accumulo/templates/accumulo-monitor-deployment.yaml b/charts/accumulo/templates/accumulo-monitor-deployment.yaml index f4a34220f4f..f615d0e956b 100644 --- a/charts/accumulo/templates/accumulo-monitor-deployment.yaml +++ b/charts/accumulo/templates/accumulo-monitor-deployment.yaml @@ -76,6 +76,10 @@ spec: value: "/opt/accumulo" - name: ACCUMULO_SERVICE_INSTANCE value: "monitor" + - name: ALLUXIO_CLIENT_HOME + value: "/opt/alluxio/client" + - name: ALLUXIO_CLIENT_HOME + value: "/opt/alluxio/client" volumeMounts: - name: accumulo-config mountPath: /opt/accumulo/conf/accumulo.properties @@ -88,6 +92,11 @@ spec: subPath: log4j2-service.properties - name: logs mountPath: /opt/accumulo/logs + - name: alluxio-client + mountPath: /opt/alluxio/client + - name: hadoop-config + mountPath: /opt/hadoop/etc/hadoop/core-site.xml + subPath: core-site.xml resources: {{- toYaml .Values.accumulo.resources.monitor | nindent 10 }} livenessProbe: @@ -111,4 +120,10 @@ spec: defaultMode: 0755 - name: logs emptyDir: {} + - name: alluxio-client + hostPath: + path: /srv/alluxio/client + - name: hadoop-config + configMap: + name: {{ include "accumulo.fullname" . }}-core-site {{- end }} diff --git a/charts/accumulo/templates/accumulo-tserver-deployment.yaml b/charts/accumulo/templates/accumulo-tserver-deployment.yaml index fbd106698fa..22971c80451 100644 --- a/charts/accumulo/templates/accumulo-tserver-deployment.yaml +++ b/charts/accumulo/templates/accumulo-tserver-deployment.yaml @@ -85,6 +85,8 @@ spec: value: "/opt/accumulo" - name: ACCUMULO_SERVICE_INSTANCE value: "tserver" + - name: ALLUXIO_CLIENT_HOME + value: "/opt/alluxio/client" volumeMounts: - name: accumulo-config mountPath: /opt/accumulo/conf/accumulo.properties @@ -97,6 +99,11 @@ spec: subPath: log4j2-service.properties - name: logs mountPath: /opt/accumulo/logs + - name: alluxio-client + mountPath: /opt/alluxio/client + - name: hadoop-config + mountPath: /opt/hadoop/etc/hadoop/core-site.xml + subPath: core-site.xml resources: {{- toYaml .Values.accumulo.resources.tserver | nindent 10 }} livenessProbe: @@ -118,4 +125,10 @@ spec: defaultMode: 0755 - name: logs emptyDir: {} + - name: alluxio-client + hostPath: + path: /srv/alluxio/client + - name: hadoop-config + configMap: + name: {{ include "accumulo.fullname" . }}-core-site {{- end }} diff --git a/charts/accumulo/templates/alluxio-worker-daemonset.yaml b/charts/accumulo/templates/alluxio-worker-daemonset.yaml index 73b4009f5fd..33c83ac2a45 100644 --- a/charts/accumulo/templates/alluxio-worker-daemonset.yaml +++ b/charts/accumulo/templates/alluxio-worker-daemonset.yaml @@ -95,6 +95,8 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: WORKER_HOSTNAME + value: "$(POD_NAME).{{ include "accumulo.fullname" . }}-alluxio-worker.$(POD_NAMESPACE).svc.cluster.local" - name: ALLUXIO_MASTER_HOSTNAME value: {{include "accumulo.fullname" .}}-alluxio-master {{- if eq .Values.storage.provider "s3" }} diff --git a/charts/accumulo/templates/alluxio-worker-service.yaml b/charts/accumulo/templates/alluxio-worker-service.yaml new file mode 100644 index 00000000000..24af51abf6f --- /dev/null +++ b/charts/accumulo/templates/alluxio-worker-service.yaml @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "accumulo.fullname" . }}-alluxio-worker + labels: + {{- include "accumulo.labels" . | nindent 4 }} + app.kubernetes.io/component: alluxio-worker +spec: + clusterIP: None # headless + ports: + - name: rpc + port: 29999 + targetPort: rpc + protocol: TCP + - name: web + port: 30000 + targetPort: web + protocol: TCP + selector: + {{- include "accumulo.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: alluxio-worker diff --git a/charts/accumulo/templates/configmap.yaml b/charts/accumulo/templates/configmap.yaml index b012b185881..59bfc738dce 100644 --- a/charts/accumulo/templates/configmap.yaml +++ b/charts/accumulo/templates/configmap.yaml @@ -1,20 +1,22 @@ # - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # https://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, - # software distributed under the License is distributed on an - # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - # KIND, either express or implied. See the License for the - # specific language governing permissions and limitations - # under the License. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + --- apiVersion: v1 kind: ConfigMap @@ -30,7 +32,7 @@ data: accumulo.properties: | # Apache Accumulo Configuration for Kubernetes ## Instance configuration - instance.volumes="{{ .Values.accumulo.instance.volumes }}" + instance.volumes={{ .Values.accumulo.instance.volumes }} instance.zookeeper.host={{ include "accumulo.zookeeperHosts" . }} instance.secret={{ .Values.accumulo.instance.secret }} @@ -65,34 +67,52 @@ data: ## Alluxio-specific configuration general.vfs.context.class.name=org.apache.accumulo.core.spi.fs.VolumeChooserEnvironment general.vfs.cache.dir=/tmp/accumulo-vfs-cache + general.vfs.classpaths=file:///opt/alluxio/client/.*\.jar accumulo-env.sh: | #!/usr/bin/env bash ## Accumulo environment for Kubernetes deployment ## Required environment variables + export ACCUMULO_LOG_DIR="${ACCUMULO_LOG_DIR:-/opt/accumulo/logs}" export HADOOP_HOME="${HADOOP_HOME:-/opt/hadoop}" export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-/opt/hadoop/etc/hadoop}" export ZOOKEEPER_HOME="${ZOOKEEPER_HOME:-/opt/zookeeper}" + export ALLUXIO_CLIENT_HOME="${ALLUXIO_CLIENT_HOME:-/opt/alluxio/client}" ## Build classpath if [[ -n $CLASSPATH ]]; then - CLASSPATH="${CLASSPATH}:${ACCUMULO_CONF_DIR}" + CLASSPATH="${CLASSPATH}:${ACCUMULO_CONF_DIR}" else - CLASSPATH="${ACCUMULO_CONF_DIR}" + CLASSPATH="${ACCUMULO_CONF_DIR}" fi + # ✅ Add Hadoop configuration FIRST (core-site.xml must be visible before Hadoop libs) + CLASSPATH="${CLASSPATH}:${HADOOP_CONF_DIR}:${HADOOP_HOME}/etc/hadoop" + # Add Accumulo libraries CLASSPATH="${CLASSPATH}:${ACCUMULO_HOME}/lib/*" # Add Hadoop libraries - CLASSPATH="${CLASSPATH}:${HADOOP_CONF_DIR}:${HADOOP_HOME}/share/hadoop/client/*" + CLASSPATH="${CLASSPATH}:${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/client/*" # Add ZooKeeper libraries ZK_JARS=$(find "${ZOOKEEPER_HOME}" -maxdepth 1 -name '*.jar' -not -name '*slf4j*' -not -name '*log4j*' | paste -sd: -) CLASSPATH="${CLASSPATH}:${ZOOKEEPER_HOME}/*:${ZK_JARS}" + if [ -d "/opt/alluxio/client" ]; then + ALLUXIO_JARS=$(find /opt/alluxio/client -maxdepth 1 -name '*.jar' | paste -sd: -) + export ALLUXIO_JARS + export HADOOP_USER_CLASSPATH_FIRST=true + export ACCUMULO_CLASSPATH="${ACCUMULO_CLASSPATH}:${ALLUXIO_JARS}" + export CLASSPATH="${CLASSPATH}:${ALLUXIO_JARS}" + fi + + # Tell Accumulo startup scripts to use this classpath as-is + export ACCUMULO_USE_CUSTOM_CLASSPATH=true + + export CLASSPATH ## JVM options for all processes @@ -198,12 +218,21 @@ data: alluxio-site.properties: | # Alluxio configuration for Accumulo storage (Master) ## Master configuration - alluxio.master.hostname=${env.POD_IP} + #alluxio.master.hostname=${env.POD_IP} + alluxio.master.hostname={{ include "accumulo.fullname" . }}-alluxio-master + alluxio.job.master.enabled=true + #alluxio.master.hostname=accumulo-alluxio-master alluxio.master.port=19998 alluxio.master.web.port=19999 alluxio.master.journal.type=UFS alluxio.master.journal.folder=/opt/alluxio/journal + # Internal (what the master itself uses to talk to its own components) + alluxio.job.master.hostname={{ include "accumulo.fullname" . }}-alluxio-master + alluxio.job.master.bind.host=0.0.0.0 + alluxio.job.master.rpc.port=20001 + alluxio.job.master.port=20001 + ## Under storage system configuration {{- include "accumulo.storageConfig" . | nindent 4 }} @@ -211,6 +240,8 @@ data: # S3 configuration alluxio.underfs.s3.endpoint={{ .Values.storage.s3.endpoint }} alluxio.underfs.s3.region={{ .Values.storage.s3.region }} + alluxio.underfs.s3.signatureVersion: v4a + alluxio.underfs.s3.path.style.access: true s3a.access.key={{ .Values.storage.s3.accessKey }} s3a.secret.key={{ .Values.storage.s3.secretKey }} {{- else if eq .Values.storage.provider "minio" }} @@ -218,6 +249,8 @@ data: alluxio.underfs.s3.endpoint={{ .Values.storage.minio.endpoint }} alluxio.underfs.s3.disable.dns.buckets=true alluxio.underfs.s3.inherit.acl=false + alluxio.underfs.s3.signatureVersion: v4a + alluxio.underfs.s3.path.style.access: true s3a.access.key={{ .Values.storage.minio.accessKey }} s3a.secret.key={{ .Values.storage.minio.secretKey }} {{- else if eq .Values.storage.provider "gcs" }} @@ -253,11 +286,17 @@ data: alluxio-site.properties: | # Alluxio configuration for Accumulo storage (Worker) ## Worker configuration - #alluxio.worker.hostname=${env.POD_IP} + alluxio.master.hostname={{ include "accumulo.fullname" . }}-alluxio-master + alluxio.master.rpc.port=19998 + alluxio.master.web.port=19999 + alluxio.worker.hostname={{ include "accumulo.fullname" . }}-alluxio-worker alluxio.worker.port=29999 alluxio.worker.web.port=30000 alluxio.worker.data.port=29999 alluxio.worker.rpc.port=29999 + alluxio.worker.rpc.bind.host=0.0.0.0 + alluxio.worker.data.bind.host=0.0.0.0 + alluxio.worker.web.bind.host=0.0.0.0 ## Memory and storage configuration alluxio.worker.memory.size={{ .Values.alluxio.properties.alluxio.worker.memory.size }} @@ -310,3 +349,24 @@ data: ## Security configuration alluxio.security.authentication.type=NOSASL alluxio.security.authorization.permission.enabled=false +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "accumulo.fullname" . }}-core-site + labels: + {{- include "accumulo.labels" . | nindent 4 }} +data: + core-site.xml: | + + + + + fs.alluxio.impl + alluxio.hadoop.FileSystem + + + fs.AbstractFileSystem.alluxio.impl + alluxio.hadoop.AlluxioFileSystem + + diff --git a/charts/accumulo/templates/minio-pvc.yaml b/charts/accumulo/templates/minio-pvc.yaml new file mode 100644 index 00000000000..88e18a9295d --- /dev/null +++ b/charts/accumulo/templates/minio-pvc.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: accumulo-minio-minio-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + storageClassName: standard # or whatever your cluster uses \ No newline at end of file diff --git a/charts/accumulo/templates/serviceaccount.yaml b/charts/accumulo/templates/serviceaccount.yaml index bea6da64668..de428b620fa 100644 --- a/charts/accumulo/templates/serviceaccount.yaml +++ b/charts/accumulo/templates/serviceaccount.yaml @@ -15,6 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# + --- {{- if .Values.auth.serviceAccount.create -}} apiVersion: v1 diff --git a/charts/accumulo/values.yaml b/charts/accumulo/values.yaml index c66399187b3..8ab8fad14a1 100644 --- a/charts/accumulo/values.yaml +++ b/charts/accumulo/values.yaml @@ -33,6 +33,16 @@ global: # Accumulo configuration accumulo: + extraEnv: + - name: ALLUXIO_CLIENT_HOME + value: /opt/alluxio/client + extraVolumeMounts: + - name: alluxio-client + mountPath: /opt/alluxio/client + extraVolumes: + - name: alluxio-client + hostPath: + path: /srv/alluxio/client # or a PVC with Alluxio client jars # Accumulo instance configuration instance: # Instance name @@ -40,7 +50,8 @@ accumulo: # Instance secret (change before deployment!) secret: "DEFAULT_CHANGE_ME" # Instance volumes - will use Alluxio - volumes: "alluxio://alluxio-master:19998/accumulo" + volumes: "alluxio://accumulo-alluxio-master:19998/accumulo" + #volumes: "alluxio://alluxio-master:19998/accumulo" # Accumulo image configuration image: @@ -230,7 +241,7 @@ storage: # MinIO configuration (for local development) minio: - endpoint: "http://minio:9000" + endpoint: "http://accumulo-minio-minio:9000" bucket: "accumulo-data" accessKey: "minioadmin" secretKey: "minioadmin" diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000000..6d0a1757672 --- /dev/null +++ b/docker/README.md @@ -0,0 +1 @@ +minikube image load accumulo/accumulo:4.0.0-SNAPSHOT diff --git a/docker/accumulo/Dockerfile b/docker/accumulo/Dockerfile index d024ac8ae5b..6d472880930 100644 --- a/docker/accumulo/Dockerfile +++ b/docker/accumulo/Dockerfile @@ -27,9 +27,11 @@ ENV ACCUMULO_VERSION=4.0.0-SNAPSHOT ENV HADOOP_VERSION=3.3.6 ENV ZOOKEEPER_VERSION=3.8.4 ENV ACCUMULO_HOME=/opt/accumulo -ENV HADOOP_HOME=/opt/hadoop +ENV HADOOP_HOME=/opt/hadoop/hadoop-${HADOOP_VERSION} ENV ZOOKEEPER_HOME=/opt/zookeeper ENV JAVA_HOME=/opt/java/openjdk +ENV ALLUXIO_CLIENT_HOME=/opt/alluxio/client + # Install required packages RUN apt-get update && \ @@ -50,23 +52,68 @@ RUN groupadd -r accumulo && \ RUN mkdir -p $ACCUMULO_HOME $HADOOP_HOME $ZOOKEEPER_HOME && \ chown -R accumulo:accumulo $ACCUMULO_HOME $HADOOP_HOME $ZOOKEEPER_HOME +# Download and install Hadoop (client libraries only) + # Download and install Hadoop (client libraries only) RUN wget -q https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz && \ - tar -xzf hadoop-${HADOOP_VERSION}.tar.gz -C /opt && \ - mv /opt/hadoop-${HADOOP_VERSION} $HADOOP_HOME && \ + tar -xzf hadoop-${HADOOP_VERSION}.tar.gz -C /opt/hadoop && \ rm hadoop-${HADOOP_VERSION}.tar.gz && \ - chown -R accumulo:accumulo $HADOOP_HOME + chown -R accumulo:accumulo /opt/hadoop + # Download and install ZooKeeper (client libraries only) -RUN wget -q https://archive.apache.org/dist/zookeeper/zookeeper-${ZOOKEEPER_VERSION}/apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz && \ - tar -xzf apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz -C /opt && \ - mv /opt/apache-zookeeper-${ZOOKEEPER_VERSION}-bin $ZOOKEEPER_HOME && \ - rm apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz && \ - chown -R accumulo:accumulo $ZOOKEEPER_HOME +RUN set -eux; \ + cd /opt; \ + wget -q https://archive.apache.org/dist/zookeeper/zookeeper-${ZOOKEEPER_VERSION}/apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz; \ + tar -xzf apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz; \ + mkdir -p ${ZOOKEEPER_HOME}; \ + cp -r apache-zookeeper-${ZOOKEEPER_VERSION}-bin/* ${ZOOKEEPER_HOME}/; \ + rm -rf apache-zookeeper-${ZOOKEEPER_VERSION}-bin apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz; \ + test -d ${ZOOKEEPER_HOME}/lib || (echo "ERROR: ZooKeeper lib directory still missing" && ls -R ${ZOOKEEPER_HOME} && exit 1); \ + chown -R accumulo:accumulo ${ZOOKEEPER_HOME}; \ + echo "✅ ZooKeeper installed at ${ZOOKEEPER_HOME} with lib populated" + +# Download and install Alluxio client JARs +ENV ALLUXIO_VERSION=2.9.4 +# Download and install Alluxio client jars +RUN set -eux; \ + mkdir -p /opt/alluxio/client; \ + wget -q https://downloads.alluxio.io/downloads/files/${ALLUXIO_VERSION}/alluxio-${ALLUXIO_VERSION}-bin.tar.gz; \ + tar -xzf alluxio-${ALLUXIO_VERSION}-bin.tar.gz -C /opt; \ + if [ -d /opt/alluxio-${ALLUXIO_VERSION}/assembly/client ]; then \ + cp /opt/alluxio-${ALLUXIO_VERSION}/assembly/client/*.jar /opt/alluxio/client/; \ + elif [ -d /opt/alluxio-${ALLUXIO_VERSION}/client ]; then \ + cp /opt/alluxio-${ALLUXIO_VERSION}/client/*.jar /opt/alluxio/client/; \ + else \ + echo "ERROR: Alluxio client jars not found" && ls -R /opt/alluxio-${ALLUXIO_VERSION} && exit 1; \ + fi; \ + rm -rf /opt/alluxio-${ALLUXIO_VERSION} alluxio-${ALLUXIO_VERSION}-bin.tar.gz; \ + chown -R accumulo:accumulo /opt/alluxio; \ + echo "✅ Alluxio client installed at /opt/alluxio/client" + +# Add missing XML parser dependencies for Alluxio client +RUN curl -L -o /opt/alluxio/client/woodstox-core-6.4.0.jar https://repo1.maven.org/maven2/com/fasterxml/woodstox/woodstox-core/6.4.0/woodstox-core-6.4.0.jar && \ + curl -L -o /opt/alluxio/client/stax2-api-4.2.1.jar https://repo1.maven.org/maven2/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar && \ + chown -R accumulo:accumulo /opt/alluxio +RUN wget -q https://repo1.maven.org/maven2/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar \ + -O /opt/alluxio/client/commons-collections-3.2.2.jar + # Copy Accumulo distribution (built from source) COPY --chown=accumulo:accumulo dist/ $ACCUMULO_HOME/ +# Ensure Accumulo lib directory exists +RUN mkdir -p $ACCUMULO_HOME/lib && chown -R accumulo:accumulo $ACCUMULO_HOME + +# After you place Alluxio client + its deps +RUN cp /opt/alluxio/client/*.jar $ACCUMULO_HOME/lib/ + + +# Add ZooKeeper and Hadoop jars to classpath +ENV ACCUMULO_CLASSPATH="$ACCUMULO_HOME/lib/*:$ZOOKEEPER_HOME/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs/*" + + + # Create necessary directories RUN mkdir -p $ACCUMULO_HOME/logs $ACCUMULO_HOME/walogs $ACCUMULO_HOME/conf && \ chown -R accumulo:accumulo $ACCUMULO_HOME @@ -77,10 +124,14 @@ RUN echo 'export JAVA_HOME=/opt/java/openjdk' >> /etc/environment && \ echo 'export ZOOKEEPER_HOME=/opt/zookeeper' >> /etc/environment && \ echo 'export ACCUMULO_HOME=/opt/accumulo' >> /etc/environment +RUN ln -s $ZOOKEEPER_HOME/lib/*.jar $ACCUMULO_HOME/lib/ || true + # Copy entrypoint script COPY docker-entrypoint.sh /usr/local/bin/ RUN chmod +x /usr/local/bin/docker-entrypoint.sh + + # Switch to accumulo user USER accumulo WORKDIR $ACCUMULO_HOME @@ -95,6 +146,7 @@ ENV ACCUMULO_CONF_DIR=$ACCUMULO_HOME/conf HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ CMD $ACCUMULO_HOME/bin/accumulo info || exit 1 + # Default command ENTRYPOINT ["docker-entrypoint.sh"] CMD ["help"] \ No newline at end of file diff --git a/docker/accumulo/docker-entrypoint.sh b/docker/accumulo/docker-entrypoint.sh index b57c0b2e12c..16820f3a9f2 100755 --- a/docker/accumulo/docker-entrypoint.sh +++ b/docker/accumulo/docker-entrypoint.sh @@ -20,8 +20,14 @@ set -euo pipefail +# Set environment variables for Accumulo, Hadoop, and ZooKeeper +export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk} +export HADOOP_HOME=${HADOOP_HOME:-/opt/hadoop} +export ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/opt/zookeeper} +export ACCUMULO_HOME=${ACCUMULO_HOME:-/opt/accumulo} + # Default configuration directory -ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR:-"$ACCUMULO_HOME"/conf} +export ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR:-"$ACCUMULO_HOME"/conf} # Function to wait for a service to be available wait_for_service() { diff --git a/pom.xml b/pom.xml index 804aeecf49f..607b7488a08 100644 --- a/pom.xml +++ b/pom.xml @@ -881,7 +881,7 @@ under the License. ${rootlocation}/src/build/eclipse-codestyle.xml **/thrift/*.java - **/charts/** + **/charts/** LF true diff --git a/scripts/reload-image.sh b/scripts/reload-image.sh new file mode 100755 index 00000000000..20978c474ca --- /dev/null +++ b/scripts/reload-image.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +minikube ssh "docker rmi accumulo/accumulo:4.0.0-SNAPSHOT || true" +minikube image load accumulo/accumulo:4.0.0-SNAPSHOT \ No newline at end of file