diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5461297f..f88ad40f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # 6.0.0 + rev: v6.0.0 # 6.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -16,26 +16,26 @@ repos: - repo: https://github.com/EmbarkStudios/cargo-deny # Failing to compile cargo-deny with version >0.18.3, needs rust 1.88.0 - rev: baa02b0a0c54e0578aae6bb7c7181ad00dc290af # 0.18.3 + rev: 0.18.6 # 0.18.3 hooks: - id: cargo-deny args: ["--all-features", "check", "advisories", "bans", "licenses", "sources"] - repo: https://github.com/adrienverge/yamllint - rev: 79a6b2b1392eaf49cdd32ac4f14be1a809bbd8f7 # 1.37.1 + rev: v1.37.1 # 1.37.1 hooks: - id: yamllint args: ["--strict"] - repo: https://github.com/igorshubovych/markdownlint-cli - rev: 192ad822316c3a22fb3d3cc8aa6eafa0b8488360 # 0.45.0 + rev: v0.46.0 # 0.45.0 hooks: - id: markdownlint types: [text] files: \.md(\.j2)*$ - repo: https://github.com/koalaman/shellcheck-precommit - rev: 99470f5e12208ff0fb17ab81c3c494f7620a1d8d # 0.11.0 + rev: v0.11.0 # 0.11.0 hooks: - id: shellcheck args: ["--severity=info"] @@ -44,7 +44,7 @@ repos: # If you do not, you will need to delete the cached ruff binary shown in the # error message - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 3b4bc031619cde2e0a9f3c4441ac7cc8227245a4 # 0.14.1 + rev: v0.14.7 # 0.14.1 hooks: # Run the linter. - id: ruff-check @@ -52,12 +52,12 @@ repos: - id: ruff-format - repo: https://github.com/rhysd/actionlint - rev: e7d448ef7507c20fc4c88a95d0c448b848cd6127 # 1.7.8 + rev: v1.7.9 # 1.7.8 hooks: - id: actionlint - repo: https://github.com/hadolint/hadolint - rev: 57e1618d78fd469a92c1e584e8c9313024656623 # 2.14.0 + rev: v2.14.0 # 2.14.0 hooks: - id: hadolint diff --git a/Cargo.lock b/Cargo.lock index 12987668..e6318618 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1338,7 +1338,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "darling", "regex", @@ -2481,8 +2481,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.100.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +version = "0.100.3" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "chrono", "clap", @@ -2520,7 +2520,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "darling", "proc-macro2", @@ -2531,7 +2531,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "chrono", "k8s-openapi", @@ -2548,7 +2548,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "axum", "clap", @@ -2572,7 +2572,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.8.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "schemars", "serde", @@ -2585,7 +2585,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.8.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "convert_case", "darling", diff --git a/Cargo.nix b/Cargo.nix index e04eb219..e06f37de 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4187,9 +4187,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; libName = "k8s_version"; authors = [ @@ -8089,13 +8089,13 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.100.1"; + version = "0.100.3"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; libName = "stackable_operator"; authors = [ @@ -8262,9 +8262,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -8297,9 +8297,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; libName = "stackable_shared"; authors = [ @@ -8379,9 +8379,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; libName = "stackable_telemetry"; authors = [ @@ -8489,9 +8489,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; libName = "stackable_versioned"; authors = [ @@ -8533,9 +8533,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; procMacro = true; libName = "stackable_versioned_macros"; diff --git a/Cargo.toml b/Cargo.toml index 8945301c..be57cf89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/stackabletech/airflow-operator" [workspace.dependencies] product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned"], tag = "stackable-operator-0.100.1" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned"], tag = "stackable-operator-0.100.3" } anyhow = "1.0" base64 = "0.22" @@ -32,6 +32,6 @@ strum = { version = "0.27", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" -# [patch."https://github.com/stackabletech/operator-rs.git"] -# stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } -# stackable-operator = { path = "../operator-rs/crates/stackable-operator" } +[patch."https://github.com/stackabletech/operator-rs.git"] +stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "feat/gitsync-ssh" } +#stackable-operator = { path = "../operator-rs/crates/stackable-operator" } diff --git a/crate-hashes.json b/crate-hashes.json index 1c58f755..576d5189 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,10 +1,10 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#k8s-version@0.1.3": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-operator-derive@0.3.1": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-operator@0.100.1": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-shared@0.0.3": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-telemetry@0.6.1": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-versioned-macros@0.8.3": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-versioned@0.8.3": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#k8s-version@0.1.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator-derive@0.3.1": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator@0.100.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-shared@0.0.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-telemetry@0.6.1": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned-macros@0.8.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned@0.8.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index b8e45ff8..e728bcd2 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -680,6 +680,7 @@ spec: The referenced Secret must include two fields: `user` and `password`. The `password` field can either be an actual password (not recommended) or a GitHub token, as described in the git-sync [documentation]. + This cannot be provided if `ssh_secret` is also provided. [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual nullable: true @@ -711,9 +712,19 @@ spec: [example]: https://docs.stackable.tech/home/nightly/airflow/usage-guide/mounting-dags#_example type: object repo: - description: 'The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator`.' + description: 'The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator` or `ssh://git@github.com:stackable-airflow/dags.git`.' format: uri type: string + sshSecret: + description: |- + The name of the Secret used for SSH access to the repository. + + The referenced Secret must include two fields: `key` and `knownHosts`. + This cannot be provided if `credentials_secret` is also provided. + + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + nullable: true + type: string wait: default: 20s description: |- @@ -1602,6 +1613,20 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: object + objectOverrides: + default: [] + description: |- + A list of generic Kubernetes objects, which are merged into the objects that the operator + creates. + + List entries are arbitrary YAML objects, which need to be valid Kubernetes objects. + + Read the [Object overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#object-overrides) + for more information. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array schedulers: description: |- The `schedulers` is responsible for triggering jobs and persisting their metadata to the backend database. diff --git a/docs/modules/airflow/examples/example-airflow-gitsync.yaml b/docs/modules/airflow/examples/example-airflow-gitsync-https.yaml similarity index 87% rename from docs/modules/airflow/examples/example-airflow-gitsync.yaml rename to docs/modules/airflow/examples/example-airflow-gitsync-https.yaml index 6a4b5e46..49cabcfe 100644 --- a/docs/modules/airflow/examples/example-airflow-gitsync.yaml +++ b/docs/modules/airflow/examples/example-airflow-gitsync-https.yaml @@ -24,3 +24,12 @@ spec: --git-config: http.sslCAInfo:/tmp/ca-cert/ca.crt # <11> webservers: ... +--- +apiVersion: v1 +kind: Secret +metadata: + name: git-credentials # <8> +type: Opaque +data: + user: c3Rh... + password: Z2l0a... diff --git a/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml b/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml new file mode 100644 index 00000000..52c0a0f9 --- /dev/null +++ b/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: airflow.stackable.tech/v1alpha1 +kind: AirflowCluster +metadata: + name: airflow +spec: + clusterConfig: + dagsGitSync: + - repo: ssh://git@github.com/stackable-airflow/dags.git # <1> + sshSecret: git-sync-ssh # <2> +... + +--- +apiVersion: v1 +kind: Secret +metadata: + name: git-sync-ssh # <2> +type: Opaque +data: + key: LS0tL... + knownHosts: Z2l0a... diff --git a/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc b/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc index 4c1f3e1c..8b7aa8ff 100644 --- a/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc +++ b/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc @@ -1,6 +1,6 @@ = Mounting DAGs :description: Mount DAGs in Airflow via ConfigMap for single DAGs or use git-sync for multiple DAGs. git-sync pulls from a Git repo and handles updates automatically. -:git-sync: https://github.com/kubernetes/git-sync/tree/v4.2.1 +:git-sync: https://github.com/kubernetes/git-sync/tree/v4.2.4 DAGs can be mounted by using a ConfigMap or `git-sync`. This is best illustrated with an example of each, shown in the sections below. @@ -24,21 +24,31 @@ include::example$example-airflow-dags-configmap.yaml[] <7> The resource has to be defined using `subPath`: this is to prevent the versioning of ConfigMap elements which may cause a conflict with how Airflow propagates DAGs between its components. <8> If the mount path described above is anything other than the standard location (the default is `$AIRFLOW_HOME/dags`), then the location should be defined using the relevant environment variable. -WARNING: If a DAG mounted via ConfigMap consists of modularized files then using the standard location is mandatory as Python uses this as a "root" directory when looking for referenced files. +[WARNING] +-- +If a DAG mounted via ConfigMap consists of modularized files, Python uses this as a "root" directory when looking for referenced files. +If this is the case, then either the standard DAGs location should be used, or `PYTHONPATH` should be overriden to point to the new location (it is also necessary to include the logging configuration in the path) as shown below: +[source,yaml] +---- + envOverrides: &envOverrides + AIRFLOW__CORE__DAGS_FOLDER: "/dags" + PYTHONPATH: "/stackable/app/log_config:/dags" +---- +-- The advantage of this approach is that DAGs are provided "in-line". However, handling multiple DAGs this way becomes cumbersome, as each must be mapped individually. -For multiple DAGs, it is easier to expose them via a mounted volume, as shown below. +For multiple DAGs, it is easier to expose them via `gitsync`, as shown below. == Via `git-sync` {git-sync}[git-sync] is a command that pulls a git repository into a local directory and is supplied as a sidecar container for use within Kubernetes. The Stackable Airflow images already ship with git-sync included, and the operator takes care of calling the tool and mounting volumes, so that only the repository and synchronization details are required: -.git-sync usage example +.git-sync usage example: https [source,yaml] ---- -include::example$example-airflow-gitsync.yaml[] +include::example$example-airflow-gitsync-https.yaml[] ---- <1> A Secret used for accessing database and admin user details (included here to illustrate where different credential secrets are defined) @@ -60,10 +70,17 @@ include::example$example-airflow-gitsync.yaml[] <11> Git-sync settings can be provided inline, although some of these (`--dest`, `--root`) are specified internally in the operator and are ignored if provided by the user. Git-config settings can also be specified, although a warning is logged if `safe.directory` is specified as this is defined internally, and should not be defined by the user. +.git-sync usage example: ssh +[source,yaml] +---- +include::example$example-airflow-gitsync-ssh.yaml[] +---- + +<1> The name of the Secret used to access the repository if it is not public. + This should include two fields: `key` and `knownHosts`, both of which can contain multiple entries. +<2> The secret referenced above. -IMPORTANT: The example shows a _list_ of git-sync definitions, with a single element. -This is to avoid breaking-changes in future releases. -Currently, only one such git-sync definition is considered and processed. +IMPORTANT: Gitsync will not allow both `credentialsSecret` and `sshSecret` to be declared and the operator will throw an exception if this is attempted. NOTE: git-sync can be used with DAGs that make use of Python modules, as Python is configured to use the git-sync target folder as the "root" location when looking for referenced files. See the xref:usage-guide/applying-custom-resources.adoc[] example for more details. diff --git a/nix/sources.json b/nix/sources.json index f79c2cf0..dcfe59fa 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -29,10 +29,10 @@ "homepage": "", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a7fc11be66bdfb5cdde611ee5ce381c183da8386", - "sha256": "0h3gvjbrlkvxhbxpy01n603ixv0pjy19n9kf73rdkchdvqcn70j2", + "rev": "5c46f3bd98147c8d82366df95bbef2cab3a967ea", + "sha256": "1q085irshssrpnryw9kixd0smc2c0rhqs20niklybaywpv2zlywx", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/a7fc11be66bdfb5cdde611ee5ce381c183da8386.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/5c46f3bd98147c8d82366df95bbef2cab3a967ea.tar.gz", "url_template": "https://github.com///archive/.tar.gz" } } diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index e9e17d34..5f161213 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -436,6 +436,7 @@ pub async fn reconcile_airflow( AIRFLOW_CONTROLLER_NAME, &airflow.object_ref(&()), ClusterResourceApplyStrategy::from(&airflow.spec.cluster_operation), + airflow.spec.object_overrides.clone(), ) .context(CreateClusterResourcesSnafu)?; @@ -1452,6 +1453,8 @@ fn add_git_sync_resources( } pb.add_volumes(git_sync_resources.git_content_volumes.to_owned()) .context(AddVolumeSnafu)?; + pb.add_volumes(git_sync_resources.git_ssh_volumes.to_owned()) + .context(AddVolumeSnafu)?; cb.add_volume_mounts(git_sync_resources.git_content_volume_mounts.to_owned()) .context(AddVolumeMountSnafu)?; diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index aa1a234f..a7529513 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -20,6 +20,7 @@ use stackable_operator::{ merge::Merge, }, crd::git_sync, + deep_merger::ObjectOverrides, k8s_openapi::{ api::core::v1::{Volume, VolumeMount}, apimachinery::pkg::api::resource::Quantity, @@ -198,6 +199,9 @@ pub mod versioned { // no doc string - See ProductImage struct pub image: ProductImage, + #[serde(default)] + pub object_overrides: ObjectOverrides, + /// Configuration that applies to all roles and role groups. /// This includes settings for authentication, git sync, service exposition and volumes, among other things. pub cluster_config: v1alpha1::AirflowClusterConfig, diff --git a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 index 7d5d098f..3897ad62 100644 --- a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 @@ -3,23 +3,20 @@ kind: TestStep metadata: name: install-airflow timeout: 480 +{% if test_scenario['values']['access'] == 'ssh' %} --- apiVersion: v1 kind: Secret metadata: - name: test-airflow-credentials + name: git-sync-ssh type: Opaque -stringData: - adminUser.username: airflow - adminUser.firstname: Airflow - adminUser.lastname: Admin - adminUser.email: airflow@airflow.com - adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow -{% if test_scenario['values']['executor'] == 'celery' %} - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 +data: + # This is a combination of a read-only deploy key and known hosts (github.com) for the repo (stackable-airflow/dags). + # Contact github users @razvan or @adwk67 for details. + key: LS0tLS1CRUdJTiBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0KYjNCbGJuTnphQzFyWlhrdGRqRUFBQUFBQkc1dmJtVUFBQUFFYm05dVpRQUFBQUFBQUFBQkFBQUFNd0FBQUF0emMyZ3RaVwpReU5UVXhPUUFBQUNDTUpoLzdaUDFUWDBHem5sZG0zV3p3TW9Ed0xTVzh4dm9SUmpIZGVYTVViQUFBQUtDc3QzQ1NyTGR3CmtnQUFBQXR6YzJndFpXUXlOVFV4T1FBQUFDQ01KaC83WlAxVFgwR3pubGRtM1d6d01vRHdMU1c4eHZvUlJqSGRlWE1VYkEKQUFBRUQwSUtCOG1wMWlScXFVSlVjTDh5Y0tKWkRwZkRnYWRXUHVhcUxYQkkvTVY0d21IL3RrL1ZOZlFiT2VWMmJkYlBBeQpnUEF0SmJ6RytoRkdNZDE1Y3hSc0FBQUFGbVJsY0d4dmVTMXJaWGt0Wm05eUxXZHBkSE41Ym1NQkFnTUVCUVlICi0tLS0tRU5EIE9QRU5TU0ggUFJJVkFURSBLRVktLS0tLQo= + knownHosts: Z2l0aHViLmNvbSBzc2gtcnNhIEFBQUFCM056YUMxeWMyRUFBQUFEQVFBQkFBQUJnUUNqN25kTnhRb3dnY1FuanNoY0xycVBFaWlwaG50K1ZUVHZEUDZtSEJMOWoxYU5Va1k0VWUxZ3Z3bkdMVmxPaEdlWXJuWmFNZ1JLNitQS0NVWGFEYkM3cXRiVzhnSWtoTDdhR0NzT3IvQzU2U0pNeS9CQ1pmeGQxbld6QU94U0RQZ1ZzbWVyT0JZZk5xbHRWOS9oV0NxQnl3SU5JUis1ZElnNkpUSjcycGNFcEVqY1lnWGtFMllFRlhWMUpIbnNLZ2JMV05saFNjcWIyVW15UmtReXl0Ukx0TCszOFRHeGt4Q2ZsbU8rNVo4Q1NTTlk3R2lkak1JWjdRNHpNakEybjFuR3JsVERrendEQ3N3K3dxRlBHUUExNzljbmZHV09XUlZydWoxNno2WHl2eHZqSndiejB3UVo3NVhLNXRLU2I3Rk55ZUlFczRUVDRqaytTNGRoUGVBVUM1eStiRFlpcllnTTRHQzd1RW56dG5aeWFWV1E3QjM4MUFLNFFkcnd0NTFacUV4S2JRcFRVTm4rRWpxb1R3dnFOajRrcXg1UVVDSTBUaFMvWWtPeEpDWG1QVVdaYmhqcENnNTZpKzJhQjZDbUsySkdobjU3SzVtajBNTmRCWEE0L1dud0g2WG9QV0p6SzVOeXUyekIzbkFacCtTNWhwUXMrcDF2TjEvd3Nqaz0KZ2l0aHViLmNvbSBlY2RzYS1zaGEyLW5pc3RwMjU2IEFBQUFFMlZqWkhOaExYTm9ZVEl0Ym1semRIQXlOVFlBQUFBSWJtbHpkSEF5TlRZQUFBQkJCRW1LU0VOalFFZXpPbXhrWk15N29wS2d3RkI5bmt0NVlScllNak51RzVOODd1UmdnNkNMcmJvNXdBZFQveTZ2MG1LVjBVMncwV1oyWUIvKytUcG9ja2c9CmdpdGh1Yi5jb20gc3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSU9NcXFua1Z6cm0wU2RHNlVPb3FLTHNhYmdINUM5b2tXaTBkaDJsOUdLSmwK {% endif %} +{% if test_scenario['values']['access'] == 'https' %} --- apiVersion: v1 kind: Secret @@ -32,6 +29,24 @@ data: # This token doesn't expire. user: c3RhY2thYmxlLWFpcmZsb3c= password: Z2l0aHViX3BhdF8xMUJLUURCRVkwSk1EWlNVQk1RYTdoX0c2OGlhbWtpRkpFV1RMTTF0ajFwbHFTVFNyZ3p3dHZneXI5b2tubGRXaGpVRDZITFRFV0JJcm9yT0dXCg== +{% endif %} +--- +apiVersion: v1 +kind: Secret +metadata: + name: test-airflow-credentials +type: Opaque +stringData: + adminUser.username: airflow + adminUser.firstname: Airflow + adminUser.lastname: Admin + adminUser.email: airflow@airflow.com + adminUser.password: airflow + connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow +{% if test_scenario['values']['executor'] == 'celery' %} + connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow + connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 +{% endif %} --- apiVersion: v1 kind: ConfigMap @@ -60,13 +75,19 @@ spec: {% endif %} credentialsSecret: test-airflow-credentials dagsGitSync: +{% if test_scenario['values']['access'] == 'ssh' %} + - repo: ssh://git@github.com/stackable-airflow/dags.git + sshSecret: git-sync-ssh +{% endif %} +{% if test_scenario['values']['access'] == 'https' %} - repo: https://github.com/stackable-airflow/dags + credentialsSecret: git-credentials +{% endif %} {% if test_scenario['values']['executor'] == 'celery' %} # Just setting some values to increase the test coverage (defaults should work just fine) branch: main wait: 5s {% endif %} - credentialsSecret: git-credentials gitSyncConf: # supply some config to check that safe.directory is correctly set --git-config: http.sslVerify:false diff --git a/tests/templates/kuttl/mount-dags-gitsync/31-assert.yaml.j2 b/tests/templates/kuttl/mount-dags-gitsync/31-assert.yaml.j2 index 8c7e90a8..cbe76bf8 100644 --- a/tests/templates/kuttl/mount-dags-gitsync/31-assert.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-gitsync/31-assert.yaml.j2 @@ -11,8 +11,10 @@ commands: # will expect 2 (two containers, base and gitsync) - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | grep "AIRFLOW_TEST_VAR" | wc -l | grep 2 # will expect 1 (one container, gitsync) +{% if test_scenario['values']['access'] == 'https' %} - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | grep "GITSYNC_USERNAME" | wc -l | grep 1 - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | grep "GITSYNC_PASSWORD" | wc -l | grep 1 +{% endif %} {% else %} # check that the statefulset contains mounts and envs # will expect 6 (2 from from the volume declaration + mounts to 3 containers, base and 2 gitsyncs, plus configmap restarter) @@ -20,6 +22,8 @@ commands: # will expect 3 (two containers, base and gitsync-1, and one initContainer gitsync-0) - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "AIRFLOW_TEST_VAR" | wc -l | grep 3 # will expect 2 (one container, gitsync-1, and one initContainer gitsync-0) +{% if test_scenario['values']['access'] == 'https' %} - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "GITSYNC_USERNAME" | wc -l | grep 2 - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "GITSYNC_PASSWORD" | wc -l | grep 2 {% endif %} +{% endif %} diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 39c632b3..367ddeae 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -33,6 +33,10 @@ dimensions: values: - celery - kubernetes + - name: access + values: + - https + - ssh tests: - name: smoke dimensions: @@ -49,6 +53,7 @@ tests: - airflow-latest - openshift - executor + - access - name: ldap dimensions: - airflow-latest