feat: SSH access for gitsync

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ default_language_version:
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # 6.0.0
+    rev: v6.0.0 # 6.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -16,26 +16,26 @@ repos:
 
   - repo: https://github.com/EmbarkStudios/cargo-deny
     # Failing to compile cargo-deny with version >0.18.3, needs rust 1.88.0
-    rev: baa02b0a0c54e0578aae6bb7c7181ad00dc290af # 0.18.3
+    rev: 0.18.6 # 0.18.3
     hooks:
       - id: cargo-deny
         args: ["--all-features", "check", "advisories", "bans", "licenses", "sources"]
 
   - repo: https://github.com/adrienverge/yamllint
-    rev: 79a6b2b1392eaf49cdd32ac4f14be1a809bbd8f7 # 1.37.1
+    rev: v1.37.1 # 1.37.1
     hooks:
       - id: yamllint
         args: ["--strict"]
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: 192ad822316c3a22fb3d3cc8aa6eafa0b8488360 # 0.45.0
+    rev: v0.46.0 # 0.45.0
     hooks:
       - id: markdownlint
         types: [text]
         files: \.md(\.j2)*$
 
   - repo: https://github.com/koalaman/shellcheck-precommit
-    rev: 99470f5e12208ff0fb17ab81c3c494f7620a1d8d # 0.11.0
+    rev: v0.11.0 # 0.11.0
     hooks:
       - id: shellcheck
         args: ["--severity=info"]
@@ -44,20 +44,20 @@ repos:
   # If you do not, you will need to delete the cached ruff binary shown in the
   # error message
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 3b4bc031619cde2e0a9f3c4441ac7cc8227245a4 # 0.14.1
+    rev: v0.14.7 # 0.14.1
     hooks:
       # Run the linter.
       - id: ruff-check
       # Run the formatter.
       - id: ruff-format
 
   - repo: https://github.com/rhysd/actionlint
-    rev: e7d448ef7507c20fc4c88a95d0c448b848cd6127 # 1.7.8
+    rev: v1.7.9 # 1.7.8
     hooks:
       - id: actionlint
 
   - repo: https://github.com/hadolint/hadolint
-    rev: 57e1618d78fd469a92c1e584e8c9313024656623 # 2.14.0
+    rev: v2.14.0 # 2.14.0
     hooks:
       - id: hadolint
 

Cargo.toml

@@ -11,7 +11,7 @@ repository = "https://github.com/stackabletech/airflow-operator"
 
 [workspace.dependencies]
 product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" }
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned"], tag = "stackable-operator-0.100.1" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned"], tag = "stackable-operator-0.100.3" }
 
 anyhow = "1.0"
 base64 = "0.22"
@@ -32,6 +32,6 @@ strum = { version = "0.27", features = ["derive"] }
 tokio = { version = "1.40", features = ["full"] }
 tracing = "0.1"
 
-# [patch."https://github.com/stackabletech/operator-rs.git"]
-# stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" }
-# stackable-operator = { path = "../operator-rs/crates/stackable-operator" }
+[patch."https://github.com/stackabletech/operator-rs.git"]
+stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "feat/gitsync-ssh" }
+#stackable-operator = { path = "../operator-rs/crates/stackable-operator" }

deploy/helm/airflow-operator/crds/crds.yaml

@@ -680,6 +680,7 @@ spec:
                               The referenced Secret must include two fields: `user` and `password`.
                               The `password` field can either be an actual password (not recommended) or a GitHub token,
                               as described in the git-sync [documentation].
+                              This cannot be provided if `ssh_secret` is also provided.
 
                               [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual
                             nullable: true
@@ -711,9 +712,19 @@ spec:
                               [example]: https://docs.stackable.tech/home/nightly/airflow/usage-guide/mounting-dags#_example
                             type: object
                           repo:
-                            description: 'The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator`.'
+                            description: 'The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator` or `ssh://[email protected]:stackable-airflow/dags.git`.'
                             format: uri
                             type: string
+                          sshSecret:
+                            description: |-
+                              The name of the Secret used for SSH access to the repository.
+
+                              The referenced Secret must include two fields: `key` and `knownHosts`.
+                              This cannot be provided if `credentials_secret` is also provided.
+
+                              [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual
+                            nullable: true
+                            type: string
                           wait:
                             default: 20s
                             description: |-
@@ -1602,6 +1613,20 @@ spec:
                       type: object
                       x-kubernetes-preserve-unknown-fields: true
                   type: object
+                objectOverrides:
+                  default: []
+                  description: |-
+                    A list of generic Kubernetes objects, which are merged into the objects that the operator
+                    creates.
+
+                    List entries are arbitrary YAML objects, which need to be valid Kubernetes objects.
+
+                    Read the [Object overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#object-overrides)
+                    for more information.
+                  items:
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
+                  type: array
                 schedulers:
                   description: |-
                     The `schedulers` is responsible for triggering jobs and persisting their metadata to the backend database.

docs/modules/airflow/examples/example-airflow-gitsync.yaml → docs/modules/airflow/examples/example-airflow-gitsync-https.yaml

@@ -24,3 +24,12 @@ spec:
           --git-config: http.sslCAInfo:/tmp/ca-cert/ca.crt # <11>
   webservers:
     ...
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: git-credentials # <8>
+type: Opaque
+data:
+  user: c3Rh...
+  password: Z2l0a...

docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: airflow.stackable.tech/v1alpha1
+kind: AirflowCluster
+metadata:
+  name: airflow
+spec:
+  clusterConfig:
+    dagsGitSync:
+      - repo: ssh://[email protected]/stackable-airflow/dags.git # <1>
+        sshSecret: git-sync-ssh # <2>
+...
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: git-sync-ssh # <2>
+type: Opaque
+data:
+  key: LS0tL...
+  knownHosts: Z2l0a...