From 369cf1ae2fb1adc6c4b34191aed621a6d83f3a9a Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 10 Oct 2025 17:44:00 -0400 Subject: [PATCH 1/3] [operator_build] Update the go.mod replace to use local repos When building from a PR that has multiple dependencies in the same repo, the incorrect versions are built. This happens because the build tasks extract a sha from an operators list that is built from zuul.items, which contains a list of the dependencies on a PR (from Depends-On). This list can contain multiple PRs from the same repo, however, the operator list only takes the first item for each repo. The information is used to update the openstack-operator go.mod file with a "replaces" directive that tell it where to find the operators to deploy. The line being added was referencing a particular commit on the PR source branch previously. This is incorrect when using zuul, as zuul uses speculative merging and prepares a version of the repositories that have all the dependencies merged together. The "replaces" is updated to reference the local version of the operator i.e. the one prepared by zuul. --- roles/operator_build/tasks/build.yml | 19 ++++++++++++++----- roles/operator_build/tasks/main.yml | 20 ++++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/roles/operator_build/tasks/build.yml b/roles/operator_build/tasks/build.yml index 6aefbe2c1e..8408a0dde1 100644 --- a/roles/operator_build/tasks/build.yml +++ b/roles/operator_build/tasks/build.yml @@ -38,13 +38,15 @@ ansible.builtin.set_fact: operator_api_path: "github.com/{{ cifmw_operator_build_org }}/{{ operator.name }}/{{ operator_base_module_name }}" + # This is telling the openstack operator to look for dependency on a particular branch on github. + # This should be the zuul src path. - name: "{{ operator.name }} - Update the go.mod file in meta operator for provided PR_SHA" # noqa: name[template] ansible.builtin.shell: | - go mod edit -replace {{ operator_api_path }}=github.com/{{ operator.pr_owner }}/{{ operator_base_module_name }}@{{ operator.pr_sha }} + go mod edit -replace {{ operator_api_path }}={{ operator.src }}/{{ operator_base_module_name }} go mod tidy if [ -d ./apis ]; then pushd ./apis/ - go mod edit -replace {{ operator_api_path }}=github.com/{{ operator.pr_owner }}/{{ operator_base_module_name }}@{{ operator.pr_sha }} + go mod edit -replace {{ operator_api_path }}={{ operator.src }}/{{ operator_base_module_name }} go mod tidy popd fi @@ -63,17 +65,24 @@ chdir: "{{ operator.src }}" register: git_head_out + # The PR sha does not need to be pulled from the zuul.projects, since this __should__ match HEAD - name: "{{ operator.name }} - Set pr_sha to be used as image tag" # noqa: name[template] ansible.builtin.set_fact: - pr_sha: "{{ operator.pr_sha | default(git_head_out.stdout | trim) }}" + #pr_sha: "{{ operator.pr_sha | default(git_head_out.stdout | trim) }}" + pr_sha: "{{ git_head_out.stdout | trim }}" + +- name: Skip if operator.pr_sha matches HEAD + when: pr_sha != operator.pr_sha | default('') + ansible.builtin.debug: + msg: "Something" - name: "{{ operator.name }} - Update the go.mod file using latest commit if no PR is provided" # noqa: name[template] ansible.builtin.shell: | - go mod edit -replace {{ operator_api_path }}={{ operator_api_path }}@{{ pr_sha }} + go mod edit -replace {{ operator_api_path }}={{ operator.src }}/{{ operator_base_module_name }} go mod tidy if [ -d ./apis ]; then pushd ./apis/ - go mod edit -replace {{ operator_api_path }}={{ operator_api_path }}@{{ pr_sha }} + go mod edit -replace {{ operator_api_path }}={{ operator.src }}/{{ operator_base_module_name }} go mod tidy popd fi diff --git a/roles/operator_build/tasks/main.yml b/roles/operator_build/tasks/main.yml index a0aec86600..0fc5e85f47 100644 --- a/roles/operator_build/tasks/main.yml +++ b/roles/operator_build/tasks/main.yml @@ -32,6 +32,16 @@ cifmw_operator_build_output: "{{ cifmw_operator_build_output }}" cifmw_operator_build_meta_name: "{{ cifmw_operator_build_meta_name }}" + # This builds a list of operators from the zuul.items list. + # zuul.items is a list of all the changes that are included via Depends-On. + # This can have duplicates if you have multiple changes from the same repo. + # e.g. when there's a CI PR trying to test some other unmerged change. + # In the case that there are duplicates, only one version is taken, which is incorrect. + # zuul.projects contains a list of the prepared repos that the job needs (listed in required-projects) + # zuul.projects contains reporitories that have the multiple changes from each repo merged. + # The operator we want to build is from this source. + # The zuul.items list can be used to build a list of operators that need to be build i.e. all the operators that have been changed and are depended on by the PR being tested. + # However, the repository/version to build from should be obtained from zuul.projects[$operator_name], not zuul.items.search($operator_name) | first. - name: Populate operators list with zuul info when: - zuul is defined @@ -42,6 +52,16 @@ loop_control: loop_var: zuul_item + # The task above gets a list of the projects that have changed and will need to be rebuilt. + # Since we're not using the github URLs in go.mod replace lines, we don't need to get any of the URLs + # The PR shas are also no longer used, since we set the image sha from the git HEAD. + + # Before running this, the zuul_info_operators var needs to be updated with the information from zuul.projects + # The information that it needs from zuul.projects is the commit hash? + # operators_list is used in build.yml, where is it looped through and passed in one-at-a-time as the operator variable + # The values needed are the URL of the source branch of the PR and the commit sha + # The usage of this information has been updated, so that the go.mod redirects point to the local repos and + # the commit sha is looked up in the local repo - name: Merge lists of operators ansible.builtin.set_fact: operators_list: "{{ [cifmw_operator_build_operators, zuul_info_operators | default([])] \ From 48e85fc74a3d8c0f6167575b988cf2f22fd21d0d Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Mon, 13 Oct 2025 09:29:17 -0400 Subject: [PATCH 2/3] Update build to copy deps into the container for building --- roles/operator_build/tasks/build.yml | 32 +++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/roles/operator_build/tasks/build.yml b/roles/operator_build/tasks/build.yml index 8408a0dde1..0a35c6cdcf 100644 --- a/roles/operator_build/tasks/build.yml +++ b/roles/operator_build/tasks/build.yml @@ -38,15 +38,29 @@ ansible.builtin.set_fact: operator_api_path: "github.com/{{ cifmw_operator_build_org }}/{{ operator.name }}/{{ operator_base_module_name }}" + + # Copy the dep into openstack-operator/deps + # This copies the prepared zuul repo into the meta operator dir, which gets copied into the build container +- name: "{{ operator.name }} - Copy dependancy source into meta operator" + ansible.builtin.command: | + git clone {{ operator.src }} deps/{{ operator.name }} + args: + chdir: "{{ cifmw_operator_build_meta_src }}" + when: + - operator.name != cifmw_operator_build_meta_name + # This is telling the openstack operator to look for dependency on a particular branch on github. # This should be the zuul src path. + # If we switch to using local sources, they need to be copied into the container for the build. + # This can be done by copying the modified operator code into the openstack-operator dir + # The replacement here will then point to ./deps/{{ operator.name }} - name: "{{ operator.name }} - Update the go.mod file in meta operator for provided PR_SHA" # noqa: name[template] ansible.builtin.shell: | - go mod edit -replace {{ operator_api_path }}={{ operator.src }}/{{ operator_base_module_name }} + go mod edit -replace {{ operator_api_path }}=./deps/{{ operator.name }}/{{ operator_base_module_name }} go mod tidy if [ -d ./apis ]; then pushd ./apis/ - go mod edit -replace {{ operator_api_path }}={{ operator.src }}/{{ operator_base_module_name }} + go mod edit -replace {{ operator_api_path }}=./deps/{{ operator.name }}/operator_base_module_name go mod tidy popd fi @@ -68,7 +82,7 @@ # The PR sha does not need to be pulled from the zuul.projects, since this __should__ match HEAD - name: "{{ operator.name }} - Set pr_sha to be used as image tag" # noqa: name[template] ansible.builtin.set_fact: - #pr_sha: "{{ operator.pr_sha | default(git_head_out.stdout | trim) }}" + # pr_sha: "{{ operator.pr_sha | default(git_head_out.stdout | trim) }}" pr_sha: "{{ git_head_out.stdout | trim }}" - name: Skip if operator.pr_sha matches HEAD @@ -76,13 +90,21 @@ ansible.builtin.debug: msg: "Something" +- name: Show the {{ item.name }} - {{ item.value }} + ansible.builtin.debug: + var: item + with_items: + - {name: operator_api_path, value: "{{ operator_api_path | default('') }}" } + - {name: operator_base_module_name, value: "{{ operator_base_module_name | default('') }}" } + - {name: operator.name, value: "{{ operator.name | default('')}}" } + - name: "{{ operator.name }} - Update the go.mod file using latest commit if no PR is provided" # noqa: name[template] ansible.builtin.shell: | - go mod edit -replace {{ operator_api_path }}={{ operator.src }}/{{ operator_base_module_name }} + go mod edit -replace {{ operator_api_path }}=./deps/{{ operator.name }}/{{ operator_base_module_name }} go mod tidy if [ -d ./apis ]; then pushd ./apis/ - go mod edit -replace {{ operator_api_path }}={{ operator.src }}/{{ operator_base_module_name }} + go mod edit -replace {{ operator_api_path }}=./deps/{{ operator.name }}/{{ operator_base_module_name }} go mod tidy popd fi From 1a213a18fb464a8a107c72feb9213fc204a6b545 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Mon, 13 Oct 2025 16:42:07 -0400 Subject: [PATCH 3/3] Convert shell to git module --- roles/operator_build/tasks/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/operator_build/tasks/build.yml b/roles/operator_build/tasks/build.yml index 0a35c6cdcf..0a0ac873b0 100644 --- a/roles/operator_build/tasks/build.yml +++ b/roles/operator_build/tasks/build.yml @@ -41,12 +41,12 @@ # Copy the dep into openstack-operator/deps # This copies the prepared zuul repo into the meta operator dir, which gets copied into the build container -- name: "{{ operator.name }} - Copy dependancy source into meta operator" - ansible.builtin.command: | - git clone {{ operator.src }} deps/{{ operator.name }} - args: - chdir: "{{ cifmw_operator_build_meta_src }}" +- name: "{{ operator.name }} - Copy dependancy source into meta operator" # noqa: name[template] + ansible.builtin.git: + repo: "{{ operator.src }}" + dest: "{{ cifmw_operator_build_meta_src }}/deps/{{ operator.name }}" when: + - cifmw_operator_build_meta_build - operator.name != cifmw_operator_build_meta_name # This is telling the openstack operator to look for dependency on a particular branch on github.