From 3620ed06e4b900e1ade3759dcaf3e051cefb2b4f Mon Sep 17 00:00:00 2001 From: Mark Phillips Date: Sat, 14 Jun 2025 13:26:12 +1000 Subject: [PATCH 1/9] Fixed lint issues --- gitopsconfig.yaml | 12 ------------ src/configuration/gitops_config.py | 10 +++++----- src/configuration/gitops_config_operator.py | 8 +------- src/configuration/gitops_connector.py | 5 +++-- src/configuration/gitops_connector_manager.py | 1 + src/gitops_event_handler.py | 10 ++++------ 6 files changed, 14 insertions(+), 32 deletions(-) delete mode 100644 gitopsconfig.yaml diff --git a/gitopsconfig.yaml b/gitopsconfig.yaml deleted file mode 100644 index 27f553e..0000000 --- a/gitopsconfig.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: example.com/v1 -kind: GitOpsConfig -metadata: - name: ezievent-identityservice-gitops-stage-dev -spec: - gitRepositoryType: "AZDO" - ciCdOrchestratorType: "AZDO" - gitOpsOperatorType: "ARGOCD" - gitOpsAppURL: "https://dev.azure.com/ezievent/EziEvent/_git/ezievent-identityservice-gitops" - azdoGitOpsRepoName: "ezievent-identityservice-gitops" - azdoPrRepoName: "ezievent-identityservice-gitops" - azdoOrgUrl: "https://dev.azure.com/ezievent/EziEvent" diff --git a/src/configuration/gitops_config.py b/src/configuration/gitops_config.py index 57b5023..de045d9 100644 --- a/src/configuration/gitops_config.py +++ b/src/configuration/gitops_config.py @@ -1,10 +1,10 @@ class GitOpsConfig: def __init__(self, - name, - git_repository_type, - cicd_orchestrator_type, - gitops_operator_type, - gitops_app_url, + name, + git_repository_type, + cicd_orchestrator_type, + gitops_operator_type, + gitops_app_url, azdo_gitops_repo_name=None, azdo_pr_repo_name=None, azdo_org_url=None, diff --git a/src/configuration/gitops_config_operator.py b/src/configuration/gitops_config_operator.py index 69ee2f8..99bf989 100644 --- a/src/configuration/gitops_config_operator.py +++ b/src/configuration/gitops_config_operator.py @@ -1,9 +1,7 @@ import kopf import logging from kubernetes import config as k8s_config -from threading import Thread from configuration.gitops_config import GitOpsConfig -from typing import Callable, Optional, List from configuration.gitops_connector_manager import GitOpsConnectorManager class GitOpsConfigOperator: @@ -61,10 +59,6 @@ def parse_config(self, spec, name): def get_configuration(self, name): """Get the configuration object by name.""" return self.configurations.get(name) - - # def get_gitops_connector(self, name): - # """Get the gitops_connector object by name.""" - # return self.connector_manager.connectors.get(name) def stop_all(self): - self.connector_manager.stop_all() \ No newline at end of file + self.connector_manager.stop_all() diff --git a/src/configuration/gitops_connector.py b/src/configuration/gitops_connector.py index cd2acb3..1411842 100644 --- a/src/configuration/gitops_connector.py +++ b/src/configuration/gitops_connector.py @@ -17,6 +17,7 @@ PR_CLEANUP_INTERVAL = 1 * 30 DISABLE_POLLING_PR_TASK = False + # Instance is shared across threads. class GitopsConnector: @@ -29,7 +30,7 @@ def __init__(self, gitops_config: GitOpsConfig): self.status_thread = None self.status_thread_running = False - + self.cleanup_task = Timeloop() self.cleanup_task_running = False @@ -47,7 +48,7 @@ def pr_polling_thread_worker(): def is_supported_message(self, payload): return self._gitops_operator.is_supported_message(payload) - + def start_background_work(self): self._start_status_thread() self._start_cleanup_task() diff --git a/src/configuration/gitops_connector_manager.py b/src/configuration/gitops_connector_manager.py index c616e04..7abf441 100644 --- a/src/configuration/gitops_connector_manager.py +++ b/src/configuration/gitops_connector_manager.py @@ -2,6 +2,7 @@ from configuration.gitops_connector import GitopsConnector from configuration.gitops_config import GitOpsConfig + class GitOpsConnectorManager: """Manages configurations and the lifecycle of GitOpsConnector instances.""" def __init__(self): diff --git a/src/gitops_event_handler.py b/src/gitops_event_handler.py index 366198e..0b6aa7c 100644 --- a/src/gitops_event_handler.py +++ b/src/gitops_event_handler.py @@ -5,7 +5,6 @@ import logging import kopf from kubernetes import client, config -from kubernetes.client.rest import ApiException import atexit import time import utils @@ -42,11 +41,11 @@ else: logging.debug('Detected no ENV configuration data. Running in multiple instance configuration mode via gitopsconfig resources.') try: - cluster_domain=utils.getenv('CLUSTER_DOMAIN') + cluster_domain = utils.getenv('CLUSTER_DOMAIN') logging.debug(f"cluster domain: '{cluster_domain}'") config.load_incluster_config() # In-cluster Kubernetes config api_instance = client.CustomObjectsApi() - instances = api_instance.list_cluster_custom_object(cluster_domain, "v1", "gitopsconfigs") + instances = api_instance.list_cluster_custom_object(cluster_domain, "v1", "gitopsconfigs") for instance in instances.get("items"): config_name = instance.get("metadata").get("name") config_namespace = instance.get("metadata").get("namespace") @@ -76,8 +75,6 @@ def run_kopf_operator(): kopf_thread = Thread(target=run_kopf_operator) kopf_thread.start() - - @application.route("/gitopsphase", methods=['POST']) def gitopsphase(): # Use per process timer to stash the time we got the request @@ -99,7 +96,7 @@ def gitopsphase(): logging.debug(f'GitOps phase: {payload}') gitops_connector = connector_manager.get_supported_gitops_connector(payload) - if gitops_connector != None: + if gitops_connector is not None: gitops_connector.process_gitops_phase(payload, req_time) return f'GitOps phase: {payload}', 200 @@ -108,6 +105,7 @@ def gitopsphase(): def interrupt(): connector_manager.stop_all() + atexit.register(interrupt) From 87823da995b16de2090cc4d5d81542de1415185b Mon Sep 17 00:00:00 2001 From: Mark Phillips Date: Sat, 14 Jun 2025 13:34:33 +1000 Subject: [PATCH 2/9] Added license headers --- src/configuration/gitops_config.py | 3 +++ src/configuration/gitops_config_operator.py | 3 +++ src/configuration/gitops_connector_manager.py | 3 +++ src/gitops_event_handler.py | 1 + src/orchestrators/cicd_orchestrator_factory.py | 1 - 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/configuration/gitops_config.py b/src/configuration/gitops_config.py index de045d9..366d23a 100644 --- a/src/configuration/gitops_config.py +++ b/src/configuration/gitops_config.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + class GitOpsConfig: def __init__(self, name, diff --git a/src/configuration/gitops_config_operator.py b/src/configuration/gitops_config_operator.py index 99bf989..ad42035 100644 --- a/src/configuration/gitops_config_operator.py +++ b/src/configuration/gitops_config_operator.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + import kopf import logging from kubernetes import config as k8s_config diff --git a/src/configuration/gitops_connector_manager.py b/src/configuration/gitops_connector_manager.py index 7abf441..71f519e 100644 --- a/src/configuration/gitops_connector_manager.py +++ b/src/configuration/gitops_connector_manager.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + import logging from configuration.gitops_connector import GitopsConnector from configuration.gitops_config import GitOpsConfig diff --git a/src/gitops_event_handler.py b/src/gitops_event_handler.py index 0b6aa7c..bba4c1f 100644 --- a/src/gitops_event_handler.py +++ b/src/gitops_event_handler.py @@ -75,6 +75,7 @@ def run_kopf_operator(): kopf_thread = Thread(target=run_kopf_operator) kopf_thread.start() + @application.route("/gitopsphase", methods=['POST']) def gitopsphase(): # Use per process timer to stash the time we got the request diff --git a/src/orchestrators/cicd_orchestrator_factory.py b/src/orchestrators/cicd_orchestrator_factory.py index d6fadd1..d5daa63 100644 --- a/src/orchestrators/cicd_orchestrator_factory.py +++ b/src/orchestrators/cicd_orchestrator_factory.py @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import utils from orchestrators.cicd_orchestrator import CicdOrchestratorInterface from repositories.git_repository import GitRepositoryInterface from orchestrators.azdo_cicd_orchestrator import AzdoCicdOrchestrator From 3d247dfbd3489cc9430802e943daf7cf5513a981 Mon Sep 17 00:00:00 2001 From: Mark Phillips Date: Sat, 14 Jun 2025 13:44:33 +1000 Subject: [PATCH 3/9] Fixed more lint errors --- src/clients/azdo_client.py | 4 ++-- src/clients/github_client.py | 2 +- src/configuration/gitops_config.py | 4 ++-- src/configuration/gitops_connector.py | 1 - src/operators/argo_gitops_operator.py | 5 +++-- src/operators/flux_gitops_operator.py | 6 +++--- src/operators/gitops_operator.py | 4 ++-- src/operators/gitops_operator_factory.py | 3 +-- src/orchestrators/azdo_cicd_orchestrator.py | 4 ++-- src/orchestrators/cicd_orchestrator_factory.py | 2 +- src/orchestrators/github_cicd_orchestrator.py | 3 +-- src/repositories/azdo_git_repository.py | 6 ++---- src/repositories/git_repository_factory.py | 5 ++--- src/repositories/github_git_repository.py | 4 ++-- 14 files changed, 24 insertions(+), 29 deletions(-) diff --git a/src/clients/azdo_client.py b/src/clients/azdo_client.py index 99246c2..abd0fd7 100644 --- a/src/clients/azdo_client.py +++ b/src/clients/azdo_client.py @@ -6,11 +6,11 @@ import logging from configuration.gitops_config import GitOpsConfig + class AzdoClient: def __init__(self, gitops_config: GitOpsConfig): - # https://dev.azure.com/csedevops/GitOps - self.org_url = gitops_config.azdo_org_url # utils.getenv("AZDO_ORG_URL") + self.org_url = gitops_config.azdo_org_url # token is supposed to be stored in a secret without any transformations token = base64.b64encode(f':{utils.getenv("PAT")}'.encode("ascii")).decode("ascii") diff --git a/src/clients/github_client.py b/src/clients/github_client.py index abda2b2..e07bd85 100644 --- a/src/clients/github_client.py +++ b/src/clients/github_client.py @@ -8,7 +8,7 @@ class GitHubClient: def __init__(self, gitops_config: GitOpsConfig): - self.org_url = gitops_config.github_org_url # utils.getenv("GITHUB_ORG_URL") # https://api.github.com/repos/kaizentm + self.org_url = gitops_config.github_org_url # token is supposed to be stored in a secret without any transformations self.token = utils.getenv("PAT") self.headers = {'Authorization': f'token {self.token}'} diff --git a/src/configuration/gitops_config.py b/src/configuration/gitops_config.py index 366d23a..9bd72c1 100644 --- a/src/configuration/gitops_config.py +++ b/src/configuration/gitops_config.py @@ -8,8 +8,8 @@ def __init__(self, cicd_orchestrator_type, gitops_operator_type, gitops_app_url, - azdo_gitops_repo_name=None, - azdo_pr_repo_name=None, + azdo_gitops_repo_name=None, + azdo_pr_repo_name=None, azdo_org_url=None, github_gitops_repo_name=None, github_gitops_manifests_repo_name=None, diff --git a/src/configuration/gitops_connector.py b/src/configuration/gitops_connector.py index 1411842..f3650a4 100644 --- a/src/configuration/gitops_connector.py +++ b/src/configuration/gitops_connector.py @@ -143,4 +143,3 @@ def drain_commit_status_queue(self): except Exception as e: logging.error(f'Unexpected exception in the message queue draining thread: {e}') - diff --git a/src/operators/argo_gitops_operator.py b/src/operators/argo_gitops_operator.py index b8c9489..7185643 100644 --- a/src/operators/argo_gitops_operator.py +++ b/src/operators/argo_gitops_operator.py @@ -7,6 +7,7 @@ from operators.git_commit_status import GitCommitStatus from configuration.gitops_config import GitOpsConfig + class ArgoGitopsOperator(GitopsOperatorInterface): def __init__(self, gitops_config: GitOpsConfig): @@ -48,7 +49,7 @@ def is_finished(self, phase_data): logging.debug(f'is_finished called. phase_data: {json.dumps(phase_data, indent=2)}') phase_status, _, health_status = self._get_statuses(phase_data) logging.debug(f'is_finished: phase_status: {phase_status}, health_status: {health_status}') - + is_finished = \ phase_status != 'Inconclusive' \ and phase_status != 'Running' \ @@ -80,7 +81,7 @@ def is_supported_operator(self, phase_data) -> bool: def is_supported_message(self, phase_data) -> bool: if ((not self.is_supported_operator(phase_data)) or - phase_data['commitid'] == "" or + phase_data['commitid'] == "" or phase_data['resources'] == None): return False return True diff --git a/src/operators/flux_gitops_operator.py b/src/operators/flux_gitops_operator.py index 8b2886b..9efdb06 100644 --- a/src/operators/flux_gitops_operator.py +++ b/src/operators/flux_gitops_operator.py @@ -132,11 +132,11 @@ def get_commit_id(self, phase_data) -> str: revision = phase_data['metadata']['source.toolkit.fluxcd.io/revision'] return parse_commit_id(revision) - + def is_supported_operator(self, phase_data) -> bool: return (self.gitops_config.name == 'singleInstance' or - (self.gitops_config.name != 'singleInstance' and - 'gitops_connector_config_name' in phase_data['metadata'] and + (self.gitops_config.name != 'singleInstance' and + 'gitops_connector_config_name' in phase_data['metadata'] and phase_data['metadata']['gitops_connector_config_name'] == self.gitops_config.name)) def is_supported_message(self, phase_data) -> bool: diff --git a/src/operators/gitops_operator.py b/src/operators/gitops_operator.py index d243ea8..a767f6c 100644 --- a/src/operators/gitops_operator.py +++ b/src/operators/gitops_operator.py @@ -1,15 +1,15 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import utils from abc import ABC, abstractmethod from configuration.gitops_config import GitOpsConfig + class GitopsOperatorInterface(ABC): def __init__(self, gitops_config: GitOpsConfig): self.gitops_config = gitops_config - self.callback_url = gitops_config.gitops_app_url # utils.getenv("GITOPS_APP_URL") + self.callback_url = gitops_config.gitops_app_url @abstractmethod def extract_commit_statuses(self, phase_data): diff --git a/src/operators/gitops_operator_factory.py b/src/operators/gitops_operator_factory.py index a40f389..10fa3df 100644 --- a/src/operators/gitops_operator_factory.py +++ b/src/operators/gitops_operator_factory.py @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import utils from operators.argo_gitops_operator import ArgoGitopsOperator from operators.flux_gitops_operator import FluxGitopsOperator from operators.gitops_operator import GitopsOperatorInterface @@ -16,7 +15,7 @@ class GitopsOperatorFactory: @staticmethod def new_gitops_operator(gitops_config: GitOpsConfig) -> GitopsOperatorInterface: - gitops_operator_type = gitops_config.gitops_operator_type # utils.getenv("GITOPS_OPERATOR_TYPE", FLUX_TYPE) + gitops_operator_type = gitops_config.gitops_operator_type if gitops_operator_type == FLUX_TYPE: return FluxGitopsOperator(gitops_config) diff --git a/src/orchestrators/azdo_cicd_orchestrator.py b/src/orchestrators/azdo_cicd_orchestrator.py index 7f5fcd6..07e532e 100644 --- a/src/orchestrators/azdo_cicd_orchestrator.py +++ b/src/orchestrators/azdo_cicd_orchestrator.py @@ -77,7 +77,7 @@ def _update_pr_task(self, is_successful, pr_num, is_alive=True): def _get_pr_task_data(self, pr_num, is_alive=True): logging.debug(f'_get_pr_task_data called. pr_num: {pr_num}, is_alive: {is_alive}') return self.git_repository.get_pr_metadata(pr_num) - + # Given a PR task, check if it's parent plan has already completed. # Note: Completed does not necessarily mean it succeeded. def _plan_already_completed(self, pr_task): @@ -120,7 +120,7 @@ def _build_job_already_completed(self, pr_task, plan_info): logging.debug(f'Check if job {job_id} already completed: state = {job_state}') job_state_completed = job_state == 'completed' return job_state_completed - + def notify_abandoned_pr_tasks(self): logging.debug('notify_abandoned_pr_tasks called') update_count = 0 diff --git a/src/orchestrators/cicd_orchestrator_factory.py b/src/orchestrators/cicd_orchestrator_factory.py index d5daa63..d0e8ca2 100644 --- a/src/orchestrators/cicd_orchestrator_factory.py +++ b/src/orchestrators/cicd_orchestrator_factory.py @@ -16,7 +16,7 @@ class CicdOrchestratorFactory: @staticmethod def new_cicd_orchestrator(git_repository: GitRepositoryInterface, gitops_config: GitOpsConfig) -> CicdOrchestratorInterface: - cicd_orchestrator_type = gitops_config.cicd_orchestrator_type # utils.getenv("CICD_ORCHESTRATOR_TYPE", AZDO_TYPE) + cicd_orchestrator_type = gitops_config.cicd_orchestrator_type if cicd_orchestrator_type == AZDO_TYPE: return AzdoCicdOrchestrator(git_repository, gitops_config) diff --git a/src/orchestrators/github_cicd_orchestrator.py b/src/orchestrators/github_cicd_orchestrator.py index ed3d364..a9ae90c 100644 --- a/src/orchestrators/github_cicd_orchestrator.py +++ b/src/orchestrators/github_cicd_orchestrator.py @@ -2,7 +2,6 @@ # Licensed under the MIT License. import logging -import utils import requests from orchestrators.cicd_orchestrator import CicdOrchestratorInterface from repositories.git_repository import GitRepositoryInterface @@ -14,7 +13,7 @@ class GitHubCicdOrchestrator(CicdOrchestratorInterface): def __init__(self, git_repository: GitRepositoryInterface, gitops_config: GitOpsConfig): super().__init__(git_repository) - self.gitops_repo_name = gitops_config.github_gitops_repo_name # utils.getenv("GITHUB_GITOPS_REPO_NAME") # cloud-native-ops + self.gitops_repo_name = gitops_config.github_gitops_repo_name self.github_client = GitHubClient(gitops_config) self.headers = self.github_client.get_rest_api_headers() self.rest_api_url = self.github_client.get_rest_api_url() diff --git a/src/repositories/azdo_git_repository.py b/src/repositories/azdo_git_repository.py index ee7ab24..d1b7279 100644 --- a/src/repositories/azdo_git_repository.py +++ b/src/repositories/azdo_git_repository.py @@ -1,11 +1,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import os import logging import json import requests -import utils from clients.azdo_client import AzdoClient from repositories.git_repository import GitRepositoryInterface from configuration.gitops_config import GitOpsConfig @@ -16,8 +14,8 @@ class AzdoGitRepository(GitRepositoryInterface): def __init__(self, gitops_config: GitOpsConfig): - self.gitops_repo_name = gitops_config.azdo_gitops_repo_name # utils.getenv("AZDO_GITOPS_REPO_NAME") - self.pr_repo_name = gitops_config.azdo_pr_repo_name # os.getenv("AZDO_PR_REPO_NAME", self.gitops_repo_name) + self.gitops_repo_name = gitops_config.azdo_gitops_repo_name + self.pr_repo_name = gitops_config.azdo_pr_repo_name self.azdo_client = AzdoClient(gitops_config) self.repository_api = f'{self.azdo_client.get_rest_api_url()}/_apis/git/repositories/{self.gitops_repo_name}' self.pr_repository_api = f'{self.azdo_client.get_rest_api_url()}/_apis/git/repositories/{self.pr_repo_name}' diff --git a/src/repositories/git_repository_factory.py b/src/repositories/git_repository_factory.py index 023391d..43ebe0f 100644 --- a/src/repositories/git_repository_factory.py +++ b/src/repositories/git_repository_factory.py @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -import os from repositories.git_repository import GitRepositoryInterface from repositories.azdo_git_repository import AzdoGitRepository from repositories.github_git_repository import GitHubGitRepository @@ -15,8 +14,8 @@ class GitRepositoryFactory: @staticmethod - def new_git_repository(gitops_config:GitOpsConfig) -> GitRepositoryInterface: - git_repository_type = gitops_config.git_repository_type # os.getenv("GIT_REPOSITORY_TYPE", AZDO_TYPE) + def new_git_repository(gitops_config: GitOpsConfig) -> GitRepositoryInterface: + git_repository_type = gitops_config.git_repository_type if git_repository_type == AZDO_TYPE: return AzdoGitRepository(gitops_config) diff --git a/src/repositories/github_git_repository.py b/src/repositories/github_git_repository.py index d2a06c7..adca2bd 100644 --- a/src/repositories/github_git_repository.py +++ b/src/repositories/github_git_repository.py @@ -2,18 +2,18 @@ # Licensed under the MIT License. import requests -import utils import logging from clients.github_client import GitHubClient from repositories.git_repository import GitRepositoryInterface from configuration.gitops_config import GitOpsConfig + class GitHubGitRepository(GitRepositoryInterface): MAX_DESCR_LENGTH = 140 def __init__(self, gitops_config: GitOpsConfig): - self.gitops_repo_name = gitops_config.github_gitops_manifests_repo_name # utils.getenv("GITHUB_GITOPS_MANIFEST_REPO_NAME") # gitops-manifests + self.gitops_repo_name = gitops_config.github_gitops_manifests_repo_name self.github_client = GitHubClient(gitops_config) self.headers = self.github_client.get_rest_api_headers() self.rest_api_url = self.github_client.get_rest_api_url() From 70a7f36e21da87de81e33c7f37b11da755e67cbe Mon Sep 17 00:00:00 2001 From: Mark Phillips Date: Sat, 14 Jun 2025 13:58:11 +1000 Subject: [PATCH 4/9] Cleaner condition logic --- src/configuration/gitops_config_operator.py | 1 + src/operators/argo_gitops_operator.py | 9 ++------- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/configuration/gitops_config_operator.py b/src/configuration/gitops_config_operator.py index ad42035..fee30cf 100644 --- a/src/configuration/gitops_config_operator.py +++ b/src/configuration/gitops_config_operator.py @@ -7,6 +7,7 @@ from configuration.gitops_config import GitOpsConfig from configuration.gitops_connector_manager import GitOpsConnectorManager + class GitOpsConfigOperator: def __init__(self, connector_manager: GitOpsConnectorManager): self.configurations = {} # Store configuration objects indexed by resource name diff --git a/src/operators/argo_gitops_operator.py b/src/operators/argo_gitops_operator.py index 7185643..a279a91 100644 --- a/src/operators/argo_gitops_operator.py +++ b/src/operators/argo_gitops_operator.py @@ -76,15 +76,10 @@ def _new_git_commit_status(self, commit_id, status_name, state, message: str): genre='ArgoCD') def is_supported_operator(self, phase_data) -> bool: - return (self.gitops_config.name == 'singleInstance' or - self.gitops_config.name != 'singleInstance' and phase_data.get('gitops_connector_config_name') == self.gitops_config.name) + return self.gitops_config.name == 'singleInstance' or phase_data.get('gitops_connector_config_name') == self.gitops_config.name def is_supported_message(self, phase_data) -> bool: - if ((not self.is_supported_operator(phase_data)) or - phase_data['commitid'] == "" or - phase_data['resources'] == None): - return False - return True + return self.is_supported_operator(phase_data) and phase_data.get('commitid') != "" and phase_data.get('resources') is not None def _get_deployment_status_summary(self, resources): total = len(resources) From e0b4338b25d62689b5139e3233820f4174718065 Mon Sep 17 00:00:00 2001 From: Mark Phillips Date: Tue, 24 Jun 2025 10:56:25 +1000 Subject: [PATCH 5/9] Push to repository owner --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8c3e541..41d2551 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -53,7 +53,7 @@ jobs: with: push: true context: ./src - tags: ghcr.io/azure/gitops-connector:${{ env.IMAGE_TAG }}, ghcr.io/azure/gitops-connector:latest + tags: ghcr.io/${{ github.repository_owner }}/gitops-connector:${{ env.IMAGE_TAG }}, ghcr.io/${{ github.repository_owner }}/gitops-connector:latest - name: Upload Image Tags uses: actions/upload-artifact@v2.2.2 with: From 247ce00e8f1ec8b3e21108cd00ad806276b45943 Mon Sep 17 00:00:00 2001 From: Mark Phillips Date: Tue, 24 Jun 2025 10:59:54 +1000 Subject: [PATCH 6/9] Bump upload-artifacts deprecated version --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 41d2551..5dc9ca9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -55,12 +55,12 @@ jobs: context: ./src tags: ghcr.io/${{ github.repository_owner }}/gitops-connector:${{ env.IMAGE_TAG }}, ghcr.io/${{ github.repository_owner }}/gitops-connector:latest - name: Upload Image Tags - uses: actions/upload-artifact@v2.2.2 + uses: actions/upload-artifact@v4 with: name: image_tags path: ${{ github.workspace }}/IMAGE_TAG - name: Upload Manifests Templates - uses: actions/upload-artifact@v2.2.2 + uses: actions/upload-artifact@v4 with: name: manifests path: ${{ github.workspace }}/manifests From a00e9e7df4cc25c318b674a409768b38740b2885 Mon Sep 17 00:00:00 2001 From: Mark Phillips Date: Tue, 24 Jun 2025 11:11:10 +1000 Subject: [PATCH 7/9] Image tag with run_number --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5dc9ca9..64284d7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -44,8 +44,8 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Generate Image Tag run: | - # IMAGE_TAG=${{ secrets.MAJOR_VERSION }}.${{ secrets.MINOR_VERSION }}.${{ secrets.HF_VERSION }}-${{ github.run_number }} - IMAGE_TAG=${{ secrets.MAJOR_VERSION }}.${{ secrets.MINOR_VERSION }}.${{ secrets.HF_VERSION }} + # IMAGE_TAG=${{ secrets.MAJOR_VERSION }}.${{ secrets.MINOR_VERSION }}.${{ secrets.HF_VERSION }} + IMAGE_TAG=${{ secrets.MAJOR_VERSION }}.${{ secrets.MINOR_VERSION }}.${{ secrets.HF_VERSION }}-${{ github.run_number }} echo "IMAGE_TAG=$IMAGE_TAG" >> $GITHUB_ENV echo $IMAGE_TAG > $GITHUB_WORKSPACE/IMAGE_TAG - name: Build and Push to Docker Hub From ee7a6264beaab43b5603cb57b60a8be746b519e8 Mon Sep 17 00:00:00 2001 From: Mark Phillips Date: Tue, 24 Jun 2025 15:01:28 +1000 Subject: [PATCH 8/9] Support custom crd domain --- manifests/helm/templates/crds/gitops-config.yaml | 4 ++-- manifests/helm/templates/service-account.yaml | 6 +++++- manifests/helm/values.yaml | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/manifests/helm/templates/crds/gitops-config.yaml b/manifests/helm/templates/crds/gitops-config.yaml index a13adb7..5150614 100644 --- a/manifests/helm/templates/crds/gitops-config.yaml +++ b/manifests/helm/templates/crds/gitops-config.yaml @@ -2,9 +2,9 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: gitopsconfigs.example.com # Replace 'example.com' with your domain + name: gitopsconfigs.{{ .Values.multipleInstances.clusterDomain }} spec: - group: example.com # Replace 'example.com' with your domain + group: {{ .Values.multipleInstances.clusterDomain }} names: kind: GitOpsConfig plural: gitopsconfigs diff --git a/manifests/helm/templates/service-account.yaml b/manifests/helm/templates/service-account.yaml index fa192dc..0567bc0 100644 --- a/manifests/helm/templates/service-account.yaml +++ b/manifests/helm/templates/service-account.yaml @@ -11,12 +11,13 @@ metadata: automountServiceAccountToken: {{ .Values.serviceAccount.automount }} {{- end }} --- +{{- if not .Values.singleInstance }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gitops-connector-config-reader rules: -- apiGroups: ["example.com"] # "" indicates the core API group +- apiGroups: [{{ .Values.multipleInstances.clusterDomain | quote }}] # "" indicates the core API group resources: ["gitopsconfigs"] verbs: ["get", "watch", "list", "patch"] - apiGroups: [""] # "" indicates the core API group @@ -28,7 +29,9 @@ rules: - apiGroups: [""] resources: ["namespaces"] verbs: ["watch", "list"] +{{- end }} --- +{{- if not .Values.singleInstance }} apiVersion: rbac.authorization.k8s.io/v1 # This role binding allows "jane" to read pods in the "default" namespace. # You need to already have a Role named "pod-reader" in that namespace. @@ -45,3 +48,4 @@ roleRef: kind: ClusterRole #this must be Role or ClusterRole name: gitops-connector-config-reader # this must match the name of the Role or ClusterRole you wish to bind to apiGroup: rbac.authorization.k8s.io +{{- end }} \ No newline at end of file diff --git a/manifests/helm/values.yaml b/manifests/helm/values.yaml index 077e6e3..620ae09 100644 --- a/manifests/helm/values.yaml +++ b/manifests/helm/values.yaml @@ -54,7 +54,7 @@ singleInstance: # specify values for a multiple instances configuration multipleInstances: - clusterDomain: apps-crc.testing + clusterDomain: example.com # -- Partially override resource names (adds suffix) # @section -- Common From 5d3c1f8b75a44874267f395f65973eb63de8a39c Mon Sep 17 00:00:00 2001 From: Mark Phillips Date: Tue, 24 Jun 2025 18:25:20 +1000 Subject: [PATCH 9/9] Filter abandoned pr to last hour since closed --- src/orchestrators/azdo_cicd_orchestrator.py | 4 ++-- src/repositories/azdo_git_repository.py | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/orchestrators/azdo_cicd_orchestrator.py b/src/orchestrators/azdo_cicd_orchestrator.py index 07e532e..713ad29 100644 --- a/src/orchestrators/azdo_cicd_orchestrator.py +++ b/src/orchestrators/azdo_cicd_orchestrator.py @@ -58,7 +58,7 @@ def _update_pr_task(self, is_successful, pr_num, is_alive=True): planurl = pr_task['planurl'] projectid = pr_task['projectid'] planid = pr_task['planid'] - url = f'{planurl}{projectid}/_apis/distributedtask/hubs/build/plans/{planid}/events?api-version=2.0-preview.1' + url = f'{planurl}{projectid}/_apis/distributedtask/hubs/build/plans/{planid}/events?api-version=7.1' data = { 'name': "TaskCompleted", 'taskId': pr_task['taskid'], @@ -84,7 +84,7 @@ def _plan_already_completed(self, pr_task): planurl = pr_task['planurl'] projectid = pr_task['projectid'] planid = pr_task['planid'] - url = f'{planurl}{projectid}/_apis/distributedtask/hubs/build/plans/{planid}' + url = f'{planurl}{projectid}/_apis/distributedtask/hubs/build/plans/{planid}?api-version=7.1' response = requests.get(url=url, headers=self.headers) # Throw appropriate exception if request failed diff --git a/src/repositories/azdo_git_repository.py b/src/repositories/azdo_git_repository.py index d1b7279..d214db7 100644 --- a/src/repositories/azdo_git_repository.py +++ b/src/repositories/azdo_git_repository.py @@ -66,10 +66,17 @@ def get_pr_metadata(self, pr_num): # Returns an array of PR dictionaries with an optional status filter # pr_status values: https://docs.microsoft.com/en-us/rest/api/azure/devops/git/pull%20requests/get%20pull%20requests?view=azure-devops-rest-6.0#pullrequeststatus def get_prs(self, pr_status): + from datetime import datetime, timedelta, timezone + pr_status_param = '' if pr_status: pr_status_param = f'searchCriteria.status={pr_status}&' - url = f'{self.pr_repository_api}/pullRequests?{pr_status_param}api-version=6.0' + if pr_status == "abandoned": + # Calculate minTime as 1 hour ago in UTC, formatted as ISO 8601 + min_time = (datetime.now(timezone.utc) - timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M:%SZ') + pr_status_param += f'searchCriteria.minTime={min_time}&searchCriteria.queryTimeRangeType=closed&' + + url = f'{self.pr_repository_api}/pullRequests?{pr_status_param}api-version=7.1' logging.debug(f'get_prs: url: {url}') response = requests.get(url=url, headers=self.headers)