Merge branch 'main' into feat/updates

# Conflicts:
#	CHANGELOG.md
#	pom.xml

Author: lfrancke

CHANGELOG.md

@@ -23,4 +23,3 @@ All notable changes to this project will be documented in this file.
 
 [#28]: https://github.com/stackabletech/hdfs-utils/pull/28
 [#29]: https://github.com/stackabletech/hdfs-utils/pull/29
-

Dockerfile

@@ -1,4 +1,8 @@
-FROM docker.stackable.tech/stackable/hadoop:3.3.6-stackable0.0.0-dev
+FROM docker.stackable.tech/stackable/hadoop:3.4.0-stackable0.0.0-dev
 
-COPY --chown=stackable:stackable ./hdfs-utils-*.jar /stackable/hadoop/share/hadoop/tools/lib/
-COPY --chown=stackable:stackable ./bom.json /stackable/hadoop/share/hadoop/tools/lib/hdfs-utils.cdx.json
+# Remove existing hdfs-utils jars, so we can ship our custom one
+RUN rm -f /stackable/hadoop/share/hadoop/common/lib/hdfs-utils-*.jar
+RUN rm -f /stackable/hadoop/share/hadoop/tools/lib/hdfs-utils-*.jar
+
+COPY --chown=stackable:stackable ./hdfs-utils-*.jar /stackable/hadoop/share/hadoop/common/lib/
+COPY --chown=stackable:stackable ./bom.json /stackable/hadoop/share/hadoop/common/lib/hdfs-utils.cdx.json

pom.xml

@@ -35,6 +35,10 @@
     <maven.compiler.release>${java.version}</maven.compiler.release>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 
+    <!-- Tip:
+     Use "mvn versions:display-dependency-updates" and
+     "mvn versions:display-plugin-updates"
+     to check for updates -->
     <cleanthat.version>2.22</cleanthat.version>
     <error-prone.version>2.35.1</error-prone.version>
     <google-java-format.version>1.24.0</google-java-format.version>

test/stack/20-hdfs.yaml

@@ -25,7 +25,7 @@ metadata:
   name: simple-hdfs
 spec:
   image:
-    productVersion: 3.3.6
+    productVersion: 3.4.0
     custom: hdfs # Will be overwritten by Tilt
     pullPolicy: IfNotPresent
   clusterConfig:

test/stack/30-test-hdfs-permissions.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: test-hdfs-permissions
-          image: docker.stackable.tech/stackable/hadoop:3.3.6-stackable0.0.0-dev
+          image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable0.0.0-dev
           env:
             - name: HADOOP_CONF_DIR
               value: /stackable/conf/hdfs

test/stack/31-benchmark-shell.yaml

@@ -0,0 +1,71 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: benchmark-shell
+spec:
+  template:
+    spec:
+      containers:
+        - name: benchmark-shell
+          image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable0.0.0-dev
+          env:
+            - name: HADOOP_CONF_DIR
+              value: /stackable/conf/hdfs
+            - name: KRB5_CONFIG
+              value: /stackable/kerberos/krb5.conf
+            - name: HADOOP_OPTS
+              value: -Djava.security.krb5.conf=/stackable/kerberos/krb5.conf
+          command:
+            - /bin/bash
+            - -c
+            - |
+              set -ex
+              klist -k /stackable/kerberos/keytab
+
+              log_in () { kdestroy; kinit -kt /stackable/kerberos/keytab $1/benchmark-shell.default.svc.cluster.local; }
+
+              log_in admin
+
+              bin/hdfs dfs -mkdir -p /bench
+              bin/hdfs dfs -ls /bench
+
+              # for i in $(seq 0 10); do echo "Creating $i" && bin/hdfs dfs -put -f /etc/hosts /bench/$i; done
+
+              # Watch out for the exact command you are using! (e.g. don't use "du -h /""). Checl the NameNode logs to
+              # make sure you actually produce enough OPA calls.
+              # time bin/hdfs dfs -du -h /bench
+
+              # So that you can run the benchmark manually
+              sleep infinity
+
+              exit 0
+          volumeMounts:
+            - name: hdfs-config
+              mountPath: /stackable/conf/hdfs
+            - name: kerberos
+              mountPath: /stackable/kerberos
+      volumes:
+        - name: hdfs-config
+          configMap:
+            name: simple-hdfs
+        - name: kerberos
+          ephemeral:
+            volumeClaimTemplate:
+              metadata:
+                annotations:
+                  secrets.stackable.tech/class: kerberos-default
+                  secrets.stackable.tech/scope: service=benchmark-shell
+                  secrets.stackable.tech/kerberos.service.names: admin,alice,bob
+              spec:
+                storageClassName: secrets.stackable.tech
+                accessModes:
+                  - ReadWriteOnce
+                resources:
+                  requests:
+                    storage: "1"
+      securityContext:
+        fsGroup: 1000
+        runAsGroup: 1000
+        runAsUser: 1000
+      restartPolicy: OnFailure

test/topology-provider/stack/03-hdfs.yaml

@@ -25,7 +25,7 @@ metadata:
   name: simple-hdfs
 spec:
   image:
-    productVersion: 3.3.6
+    productVersion: 3.4.0
     custom: hdfs # updated by tilt
     pullPolicy: IfNotPresent
   clusterConfig: