.gitlab-ci.jsonnet

// Imports
local variables = import '.jsonnet-libs/extras/helm_chart_repo/variables.libsonnet';

// Shortcuts
local repositories = variables.helm.repositories;
local charts = variables.helm.charts;
local public_charts = variables.helm.public_charts;

local go_cache = { key: { files: ['go.mod', 'go.sum'] }, paths: ['/go/pkg/mod/'] };

// resolving deps with deps, ct lint will not resolve that properly;
local update_2nd_degree_chart_deps(chart) = ['yq e \'.dependencies[] | select (.repository == "file*").repository | sub("^file://","")\' stable/' + chart + '/Chart.yaml  | xargs -I % helm dependencies build --skip-refresh stable/' + chart + '/%'];

local helm_config_dependencies = [
  'helm repo add %s %s' % [std.strReplace(name, '_', '-'), repositories[name]]
  for name in std.objectFields(repositories)
];

local helm_fetch_dependencies = helm_config_dependencies +
                                [
                                  'helm repo update',
                                ];

local skip_when_dependency_upgrade = {
  rules: [{
    @'if': '$UPDATE_STACKGRAPH_VERSION',
    when: 'never',
  }, {
    @'if': '$UPDATE_AAD_CHART_VERSION',
    when: 'never',
  }, {
    @'if': '$UPDATE_STACKSTATE_DOCKER_VERSION',
    when: 'never',
  }, {
    @'if': '$UPDATE_STACKPACKS_DOCKER_VERSION',
    when: 'never',
  }, {
    @'if': '$RUN_UPDATECLI',
    when: 'never',
  }] + super.rules,
};

// Build a chart with all dependencies and then share it as an artifact with next jobs in the pipepline
// This step is the most expensive step so we want to execute it only once
local build_chart_job(chart) = {
  image: variables.images.stackstate_helm_test,
  before_script: helm_fetch_dependencies,
  script:
    update_2nd_degree_chart_deps(chart) + [
      'helm dependencies build stable/' + chart,
      // To avoid a race condition with index.yaml mondifaction in the push_test_charts_jobs job, I package all modifed charts now and then upload only modified charts to s3
      'mkdir -p stable/' + chart + '/build; helm package --destination stable/' + chart + '/build stable/' + chart,
    ],
  stage: 'build',
  rules: [
    {
      @'if': '$CI_PIPELINE_SOURCE == "merge_request_event"',
      changes: ['stable/' + chart + '/**/*'],
    },
  ],
  artifacts: {
    paths: [
      'stable/' + chart + '/charts/',
      'stable/' + chart + '/build/',
    ],
  },
};
local build_chart_jobs = {
  ['build_%s' % chart]: (build_chart_job(chart))
  for chart in (charts + public_charts)
};

// Validates modified charts: formats files, execute lint commands
local validate_chart_job(chart) = {
  image: variables.images.chart_testing,
  before_script: ['.gitlab/validate_before_script.sh'],
  script: [
    'yamale --schema /etc/ct/chart_schema.yaml stable/' + chart + '/Chart.yaml',
    'yamllint --config-file /etc/ct/lintconf.yaml stable/' + chart + '/Chart.yaml',
    'yamllint --config-file /etc/ct/lintconf.yaml stable/' + chart + '/values.yaml',
    'if [ -f stable/' + chart + '/ci/default-values.yaml ]; then yamllint --config-file /etc/ct/lintconf.yaml stable/' + chart + '/ci/default-values.yaml; fi',
    'if [ -f stable/' + chart + '/ci/default-values.yaml ]; then helm lint stable/' + chart + ' --values stable/' + chart + '/ci/default-values.yaml; fi',
    '.gitlab/validate_kubeconform.sh',
  ],
  stage: 'validate',
  rules: [
    {
      @'if': '$CI_PIPELINE_SOURCE == "merge_request_event"',
      changes: ['stable/' + chart + '/**/*'],
    },
  ],
};
local validate_chart_jobs = {
  ['validate_%s' % chart]: (validate_chart_job(chart))
  for chart in (charts + public_charts)
};

// Checks if a chart version has been updated, if not then return an error
local check_chart_version_job(chart) = {
  image: variables.images.chart_testing,
  before_script: ['.gitlab/validate_before_script.sh'],
  script: [
    '.gitlab/verify_versions_bumped.sh ' + chart,
  ],
  stage: 'validate',
  rules: [
    {
      @'if': '$CI_PIPELINE_SOURCE == "merge_request_event"',
      changes: ['stable/' + chart + '/**/*'],
    },
  ],
};
local check_chart_version_jobs = {
  ['check_%s_version' % chart]: (check_chart_version_job(chart))
  for chart in (charts + public_charts)
  if chart != 'stackstate' && chart != 'suse-observability'
};

// Validation jobs for suse-observability-sizing chart (not in charts/public_charts lists)
local check_sizing_chart_jobs = {
  // Checks if suse-observability-sizing version has been bumped.
  // Adding suse-observability-sizing to the .jsonnet-libs/extras/helm_chart_repo/variables.libsonnet charts list generates
  // build, validate, test, and push jobs - which is unnecessary overhead for a library chart that's only used as a dependency.
  'check_suse-observability-sizing_version': {
    image: variables.images.chart_testing,
    before_script: ['.gitlab/validate_before_script.sh'],
    script: [
      '.gitlab/verify_versions_bumped.sh suse-observability-sizing',
    ],
    stage: 'validate',
    rules: [
      {
        @'if': '$CI_PIPELINE_SOURCE == "merge_request_event"',
        changes: ['stable/suse-observability-sizing/**/*'],
      },
    ],
  },
  // Validates that all dependent charts have updated their dependency versions.
  // Runs after version check passes to ensure we're checking against the correct version.
  check_sizing_chart_dependencies: {
    image: variables.images.chart_testing,
    before_script: ['pip install pyyaml'],
    script: [
      'python3 scripts/bump-chart-version/bump_chart_version.py --check suse-observability-sizing',
    ],
    stage: 'validate',
    needs: ['check_suse-observability-sizing_version'],
    rules: [
      {
        @'if': '$CI_PIPELINE_SOURCE == "merge_request_event"',
        changes: ['stable/suse-observability-sizing/**/*'],
      },
    ],
  },
};

// Runs unit tests on all charts with "test" directory
local test_chart_job(chart) = {
  image: variables.images.stackstate_helm_test,
  tags: ['sts-k8s-xl-runner'],
  script: [
    'go mod download',
    'go test ./stable/' + chart + '/test/...',
  ],
  stage: 'test',
  rules: [
    {
      @'if': '$CI_PIPELINE_SOURCE == "merge_request_event"',
      changes: ['stable/' + chart + '/**/*'],
      exists: ['stable/' + chart + '/test/*.go'],
    },
  ],
  variables: {
    CGO_ENABLED: 0,
    GOPATH: '/go',
  },
  cache: go_cache,
};
local test_chart_jobs = {
  ['test_%s' % chart]: (test_chart_job(chart))
  for chart in (charts + public_charts)
};

local resource_usage = {
  resource_usage: {
    image: variables.images.stackstate_helm_test,
    tags: ['sts-k8s-xl-runner'],
    before_script: helm_fetch_dependencies,
    script: update_2nd_degree_chart_deps('suse-observability') + [
      'helm dependencies build stable/suse-observability',
      'helm dependencies build stable/suse-observability-values',
      'go mod download',
      'go test ./test/...',
    ],
    stage: 'test',
    rules: [
      {
        @'if': '$CI_PIPELINE_SOURCE == "merge_request_event"',
        changes: [
          'test/*.go',
          'test/**/*',
          'stable/suse-observability/**/*',
          'stable/suse-observability-values/**/*',
        ],
      },
    ],
    variables: {
      CGO_ENABLED: 0,
      GOPATH: '/go',
    },
    artifacts: {
      paths: [
        'test/resource_usage.txt',
      ],
    },
    cache: go_cache,
  },
};

local push_chart_job_if(chart, script, rules) = {
  script: script,
  image: variables.images.stackstate_devops,
  rules: rules,
  variables: {
    CHART: 'stable/' + chart,
  },
} + skip_when_dependency_upgrade;

local push_chart_job(chart, script, when, autoTriggerOnCommitMsg) =
  push_chart_job_if(
    chart,
    script,
    [
      {
        @'if': '$CI_COMMIT_BRANCH == "master" && $CI_COMMIT_AUTHOR == "stackstate-system-user <suse-observability-ops@stackstate.com>"  && $CI_COMMIT_MESSAGE =~ /\\[' + autoTriggerOnCommitMsg + ']/',
        changes: ['stable/' + chart + '/**/*'],
        when: 'on_success',
      },
      {
        @'if': '$CI_COMMIT_BRANCH == "master"',
        changes: ['stable/' + chart + '/**/*'],
        when: when,
      },
      {
        @'if': '$CI_COMMIT_TAG =~ /^' + chart + '\\/.*/',
        when: 'on_success',
      },
    ]
  );

local push_chart_script(chart, repository_url, repository_username, repository_password) =
  (if chart == 'stackstate' || chart == 'suse-observability' then update_2nd_degree_chart_deps(chart) else [])
  + [
    'helm dependencies update --skip-refresh ${CHART}',
    'helm cm-push --username ' + repository_username + ' --password ' + repository_password + ' ${CHART} ' + repository_url,
  ];

local push_stackstate_chart_releases =
  {
    push_stackstate_release_to_internal: push_chart_job_if(
      'stackstate',
      push_chart_script(
        'stackstate',
        '${CHARTMUSEUM_INTERNAL_URL}',
        '${CHARTMUSEUM_INTERNAL_USERNAME}',
        '${CHARTMUSEUM_INTERNAL_PASSWORD}',
      ),
      variables.rules.tag.all_release_rules,
    ) {
      before_script: helm_fetch_dependencies,
      stage: 'push-charts-to-internal',
    },
    push_stackstate_release_to_public: push_chart_job_if(
      'stackstate',
      push_chart_script(
        'stackstate',
        '${CHARTMUSEUM_URL}',
        '${CHARTMUSEUM_USERNAME}',
        '${CHARTMUSEUM_PASSWORD}',
      ),
      [variables.rules.tag.release_rule],
    ) {
      before_script: helm_fetch_dependencies,
      stage: 'push-charts-to-public',
    },
  };

local push_charts_to_internal_jobs = {
  ['push_%s_to_internal' % chart]: (push_chart_job(
                                      chart,
                                      push_chart_script(
                                        chart,
                                        '${CHARTMUSEUM_INTERNAL_URL}',
                                        '${CHARTMUSEUM_INTERNAL_USERNAME}',
                                        '${CHARTMUSEUM_INTERNAL_PASSWORD}',
                                      ),
                                      'on_success',
                                      if chart == 'suse-observability-agent' then 'publish-suse-observability-agent' else 'publish-' + chart
                                    ) + {
                                      stage: 'push-charts-to-internal',
                                    } + (
                                      if chart == 'stackstate' then
                                        { before_script: helm_fetch_dependencies + ['.gitlab/bump_sts_chart_master_version.sh stackstate-internal ' + chart] }
                                      else if chart == 'suse-observability' then
                                        { before_script: helm_fetch_dependencies + [
                                          '.gitlab/configure_git.sh',
                                          // tags don't have CI_COMMIT_BRANCH, so I fetches the current branch(s) for current HEAD (HEAD points to a detached commit)
                                          // but there may be multiple branches so I iterate all of them and push a commit to each branch
                                          'export BRANCHES=${CI_COMMIT_BRANCH:-$(git for-each-ref --format="%(objectname) %(refname:short)" refs/remotes/origin | awk -v branch="$(git rev-parse HEAD)" \'$1==branch && $2!="origin" {print $2}\' | sed -E "s/^origin\\/(.*)$/\\1/")}',
                                          // It extracts version from tag, e.g. suse-observability/1.3.2 => 1.3.2
                                          '.gitlab/set_sts_chart_master_version.sh stable/' + chart + " $(echo $CI_COMMIT_TAG | sed -E 's/^" + chart + "\\/(.*)$/\\1/')",
                                        ] }
                                        { script+: [
                                          '.gitlab/tag_sts_chart_pre_release.sh ' + chart,
                                          '.gitlab/bump_sts_chart_master_version_v2.sh stable/' + chart,
                                        ] }
                                      else { before_script: helm_fetch_dependencies }
                                    ))
  for chart in (charts + public_charts)
};

local push_charts_to_public_jobs = {
  ['push_%s_to_public' % chart]: (push_chart_job(
                                    chart,
                                    push_chart_script(
                                      chart,
                                      '${CHARTMUSEUM_URL}',
                                      '${CHARTMUSEUM_USERNAME}',
                                      '${CHARTMUSEUM_PASSWORD}',
                                    ),
                                    'manual',
                                    if chart == 'suse-observability-agent' then 'publish-suse-observability-agent' else 'publish-' + chart
                                  ) + {
                                    stage: 'push-charts-to-public',

                                    needs: ['push_%s_to_internal' % chart],
                                  } + (
                                    if chart == 'stackstate' || chart == 'suse-observability' then
                                      { before_script: helm_fetch_dependencies + ['.gitlab/bump_sts_chart_master_version.sh stackstate ' + chart] }
                                    else { before_script: helm_fetch_dependencies }
                                  ))
  for chart in public_charts
  if chart != 'stackstate' && chart != 'suse-observability'
};

local push_suse_observability_to_rancher_registry = {
  'push_suse-observability-agent_to_rancher': (push_chart_job(
                                                 'suse-observability-agent',
                                                 [
                                                   '.gitlab/publish-suse-agent-to-rancher.sh',
                                                 ],
                                                 'manual',
                                                 'publish-suse-observability-agent',
                                               ) + {
                                                 stage: 'push-charts-to-rancher',
                                                 needs: ['push_suse-observability-agent_to_internal'],
                                               }),
  'push_suse-observability-values_to_rancher': (push_chart_job(
                                                  'suse-observability-values',
                                                  [
                                                    '.gitlab/publish-suse-observability-values-to-rancher.sh',
                                                  ],
                                                  'manual',
                                                  'publish-suse-observability-values',
                                                ) + {
                                                  stage: 'push-charts-to-rancher',
                                                  needs: ['push_suse-observability-values_to_internal'],
                                                }),
};

local update_sg_version = {
  update_stackgraph_version: {
    image: variables.images.stackstate_helm_test,
    stage: 'update',
    before_script: helm_fetch_dependencies,
    rules: [
      {
        @'if': '$UPDATE_STACKGRAPH_VERSION',
        when: 'always',
      },
    ],
    script: [
      '.gitlab/update_sg_version.sh stable/hbase ""',
      '.gitlab/update_sg_version.sh stable/suse-observability "hbase."',
      '.gitlab/update_chart_version.sh stable/suse-observability hbase local:stable/hbase',
      '.gitlab/commit_changes_and_push.sh StackGraph $UPDATE_STACKGRAPH_VERSION',
    ],
  },
};

local update_aad_chart_version = {
  update_aad_chart_version: {
    image: variables.images.stackstate_helm_test,
    stage: 'update',
    before_script: helm_fetch_dependencies,
    rules: [
      {
        @'if': '$UPDATE_AAD_CHART_VERSION',
        when: 'always',
      },
    ],
    script: [
      '.gitlab/update_chart_version.sh stable/suse-observability anomaly-detection $UPDATE_AAD_CHART_VERSION',
      '.gitlab/commit_changes_and_push.sh anomaly-detection $UPDATE_AAD_CHART_VERSION',
    ],
  },
};

local updatecli_job = {
  update_helm_chart_docker_images: {
    image: variables.images.container_tools_dev,
    stage: 'update',
    variables: {
      UPDATE_CLI_EMAIL: '$STACKSTATE_SYSTEM_USER_EMAIL',
      UPDATE_CLI_USER: '$STACKSTATE_SYSTEM_USER_NAME',
    },
    before_script: [
      '.gitlab/configure_git.sh',
      'export GITLAB_TOKEN="$gitlab_api_scope_token"',
      'export UPDATE_CLI_PGP_KEY="$(cat $STACKSTATE_SYSTEM_USER_PGP_KEY)"',
      'export UPDATE_CLI_PGP_PASSPHRASE="$STACKSTATE_SYSTEM_USER_PGP_PASS_PHRASE"',
    ],
    rules: [
      {
        @'if': '$RUN_UPDATECLI',
        when: 'always',
      },
      {
        when: 'never',
      },
    ],
    script: [
      'updatecli apply -c updatecli/updatecli.d/update-docker-images/ -v updatecli/values.d/values.yaml',
    ],
  },
  finalize_helm_chart_docker_images: {
    image: variables.images.container_tools_dev,
    stage: 'update',
    variables: {
      UPDATE_CLI_EMAIL: '$STACKSTATE_SYSTEM_USER_EMAIL',
      UPDATE_CLI_USER: '$STACKSTATE_SYSTEM_USER_NAME',
    },
    before_script: [
      '.gitlab/configure_git.sh',
      'export GITLAB_TOKEN="$gitlab_api_scope_token"',
      'export UPDATE_CLI_PGP_KEY="$(cat $STACKSTATE_SYSTEM_USER_PGP_KEY)"',
      'export UPDATE_CLI_PGP_PASSPHRASE="$STACKSTATE_SYSTEM_USER_PGP_PASS_PHRASE"',
    ],
    rules: [
      {
        @'if': '$RUN_UPDATECLI',
        when: 'always',
      },
      {
        when: 'never',
      },
    ],
    needs: ['update_helm_chart_docker_images'],
    script: [
      'updatecli apply -c updatecli/updatecli.d/finalize-docker-images/ -v updatecli/values.d/values.yaml',
    ],
  },
  open_updatecli_docker_images_mr: {
    image: variables.images.container_tools_dev,
    stage: 'update',
    before_script: [
      '.gitlab/configure_git.sh',
      'export GITLAB_TOKEN="$gitlab_api_scope_token"',
    ],
    rules: [
      {
        @'if': '$RUN_UPDATECLI',
        when: 'always',
      },
      {
        when: 'never',
      },
    ],
    needs: ['finalize_helm_chart_docker_images'],
    script: [
      '.gitlab/open_updatecli_mr.sh updatecli-master-docker-images master "[master] Bump helm chart docker images"',
    ],
  },
};

local update_docker_images = {
  local job(requiredEnvName, scripts) = {
    image: variables.images.stackstate_devops,
    stage: 'update',
    before_script: [
      '.gitlab/configure_git.sh',
      // tags don't have CI_COMMIT_BRANCH, so fetch the current branch(s) for current HEAD (HEAD points to a detached commit)
      // but there may be multiple branches so iterate all of them and push a commit to each branch
      'export BRANCHES=${CI_COMMIT_BRANCH:-$(git for-each-ref --format="%(objectname) %(refname:short)" refs/remotes/origin | awk -v branch="$(git rev-parse HEAD)" \'$1==branch && $2!="origin" {print $2}\' | sed -E "s/^origin\\/(.*)$/\\1/")}',
    ],
    rules: [
      {
        @'if': '$' + requiredEnvName,
        when: 'always',
      },
      {
        when: 'never',
      },
    ],
    script: scripts,
  },

  update_stackstate_version_to_latest: job('UPDATE_STACKSTATE_DOCKER_VERSION', ['.gitlab/suse-observability/update_stackstate_version_to_latest.sh']),
  update_stackpacks_version_to_latest: job('UPDATE_STACKPACKS_DOCKER_VERSION', ['.gitlab/suse-observability/update_stackpacks_version_to_latest.sh']),
};

local beest_triggers = {
  beest_agent_trigger: {
    image: variables.images.stackstate_devops,
    stage: 'update',
    script: [
      'AGENT_HELM_CHART_VERSION=$(yq e ".version" stable/suse-observability-agent/Chart.yaml)',
      'echo "Triggering beest pipeline with AGENT_HELM_CHART_VERSION=${AGENT_HELM_CHART_VERSION}"',
      'curl --verbose --fail-with-body --request POST --form "token=$STS_BEEST_TRIGGER_TOKEN" --form ref=main --form "variables[AGENT_HELM_CHART_VERSION]=${AGENT_HELM_CHART_VERSION}" --form "variables[TRIGGER_AGENT_X86_TESTS]=true" "https://gitlab.com/api/v4/projects/$BEEST_PROJECT_ID/trigger/pipeline"',
    ],
    rules: [
      {
        @'if': '$CI_COMMIT_BRANCH == "master"',
        changes: ['stable/suse-observability-agent/**/*'],
        when: 'manual',
      },
      {
        @'if': '$CI_COMMIT_BRANCH',
        changes: ['stable/suse-observability-agent/**/*'],
        when: 'manual',
      },
    ],
    allow_failure: true,
  },

  beest_stackstate_trigger: {
    image: variables.images.stackstate_devops,
    stage: 'update',
    script: [
      'STACKSTATE_HELM_CHART_VERSION=$(yq e ".version" stable/suse-observability/Chart.yaml)',
      'echo "Triggering beest pipeline with STACKSTATE_HELM_CHART_VERSION=${STACKSTATE_HELM_CHART_VERSION}"',
      'curl --verbose --fail-with-body --request POST --form "token=$STS_BEEST_TRIGGER_TOKEN" --form ref=main --form "variables[STACKSTATE_HELM_CHART_VERSION]=${STACKSTATE_HELM_CHART_VERSION}" --form "variables[TRIGGER_AGENT_X86_TESTS]=true" "https://gitlab.com/api/v4/projects/$BEEST_PROJECT_ID/trigger/pipeline"',
    ],
    rules: [
      {
        @'if': '$CI_COMMIT_BRANCH == "master"',
        changes: ['stable/suse-observability/**/*'],
        when: 'manual',
      },
      {
        @'if': '$CI_COMMIT_BRANCH',
        changes: ['stable/suse-observability/**/*'],
        when: 'manual',
      },
    ],
    allow_failure: true,
  },
};

// Main
{
  // Only run for merge requests, tags, the default (master) branch, or when RUN_UPDATECLI triggers
  workflow: {
    rules: [
      { @'if': '$CI_MERGE_REQUEST_IID' },
      { @'if': '$CI_COMMIT_TAG' },
      { @'if': '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' },
      { @'if': '$RUN_UPDATECLI' },
    ],
  },
  image: variables.images.chart_testing,
  stages: ['build', 'push-charts-to-test', 'validate', 'test', 'update', 'push-charts-to-internal', 'push-charts-to-public', 'push-charts-to-rancher'],

  variables: {
    HELM_VERSION: 'v3.1.2',
    PROMOTION_DRY_RUN: 'no',
  },
}
+ build_chart_jobs
+ validate_chart_jobs
+ check_chart_version_jobs
+ check_sizing_chart_jobs
+ test_chart_jobs
+ resource_usage

+ push_charts_to_internal_jobs
+ push_charts_to_public_jobs
+ push_stackstate_chart_releases
+ update_sg_version
+ update_aad_chart_version
+ updatecli_job
+ update_docker_images
+ push_suse_observability_to_rancher_registry
+ beest_triggers