Skip to content

Commit f24c465

Browse files
v1vpkoutsovasilis
v1v
authored and
pkoutsovasilis
committed
github-actions: create a serverless project daily
bk(serverless): use OIDC for running the ITs
1 parent 92e139c commit f24c465

File tree

2 files changed

+129
-0
lines changed

2 files changed

+129
-0
lines changed

‎.buildkite/integration.pipeline.yml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,31 @@ env:
44
DOCKER_REGISTRY: "docker.elastic.co"
55
VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
66

7+
# This section is used to define the plugins that will be used in the pipeline.
8+
# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
9+
common:
10+
- google_oidc_plugin: &google_oidc_plugin
11+
# See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/elastic-agent/01-gcp-oidc.tf
12+
# This plugin authenticates to Google Cloud using the OIDC token.
13+
elastic/oblt-google-auth#v1.3.0:
14+
lifetime: 10800 # seconds
15+
project-id: "elastic-observability-ci"
16+
project-number: "911195782929"
17+
# see https://github.com/avaly/gcp-secret-manager-buildkite-plugin/pull/10
18+
# see https://github.com/avaly/gcp-secret-manager-buildkite-plugin/pull/11
19+
# - gcp_serverless_secrets_plugin: &gcp_serverless_secrets_plugin
20+
#avaly/gcp-secret-manager#v1.2.0:
21+
- gcp_serverless_secrets_plugin: &gcp_serverless_secrets_plugin
22+
elastic/gcp-secret-manager#v1.3.0-elastic:
23+
env:
24+
# These secrets are created in .github/workflows/serverless-project.yml
25+
ELASTICSEARCH_HOST: ea-serverless-it-elasticsearch-hostname
26+
ELASTICSEARCH_PASSWORD: ea-serverless-it-elasticsearch-password
27+
ELASTICSEARCH_USERNAME: ea-serverless-it-elasticsearch-username
28+
KIBANA_HOST: ea-serverless-it-kibana-hostname
29+
KIBANA_USERNAME: ea-serverless-it-kibana-username
30+
KIBANA_PASSWORD: ea-serverless-it-kibana-password
31+
732
steps:
833
- group: "Integration tests: packaging"
934
key: "int-packaging"

‎.github/workflows/serverless-project.yml‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
---
2+
name: serverless-project
3+
4+
on:
5+
workflow_dispatch:
6+
schedule:
7+
# To run more often if needed, for now weekly on sunday at 4:00 UTC
8+
- cron: "0 4 * * 0"
9+
10+
permissions:
11+
contents: read
12+
13+
jobs:
14+
create-serverless:
15+
permissions:
16+
id-token: write
17+
runs-on: ubuntu-latest
18+
env:
19+
PREFIX: "ea-serverless-it"
20+
steps:
21+
####################################
22+
# 1. Create the serverless project
23+
####################################
24+
- name: Get token
25+
id: get_token
26+
uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
27+
with:
28+
app_id: ${{ secrets.OBS_AUTOMATION_APP_ID }}
29+
private_key: ${{ secrets.OBS_AUTOMATION_APP_PEM }}
30+
permissions: >-
31+
{
32+
"checks": "read",
33+
"contents": "write",
34+
"pull_requests": "write"
35+
}
36+
repositories: >-
37+
["observability-test-environments"]
38+
39+
- uses: elastic/oblt-actions/git/setup@v1
40+
with:
41+
github-token: ${{ steps.get_token.outputs.token }}
42+
43+
- name: Get day of the week
44+
id: get_day
45+
run: echo "day=$(date +'%a' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
46+
47+
- uses: elastic/oblt-actions/oblt-cli/cluster-create-custom@v1
48+
id: create_serverless
49+
with:
50+
template: 'serverless-ea-it'
51+
parameters: '{"Target":"production","ProjectType":"observability"}'
52+
cluster-name-prefix: "${{ env.PREFIX }}-${{ steps.get_day.outputs.day }}"
53+
github-token: ${{ steps.get_token.outputs.token }}
54+
gitops: true
55+
wait: '15'
56+
57+
# Authenticate to the elastic-observability to get the cluster credentials
58+
- uses: elastic/oblt-actions/google/auth@v1
59+
60+
- uses: elastic/oblt-actions/oblt-cli/cluster-credentials@v1
61+
with:
62+
cluster-name: ${{ steps.create_serverless.outputs.cluster-name }}
63+
github-token: ${{ steps.get_token.outputs.token }}
64+
65+
- name: Smoke test
66+
run: curl -X GET ${ELASTICSEARCH_HOST}/_cat/indices?v -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}
67+
68+
####################################
69+
# 2. Copy the serverless secrets
70+
####################################
71+
# Authenticate to the elastic-observability-ci to rotate the cluster credentials
72+
- uses: elastic/oblt-actions/google/auth@v1
73+
with:
74+
project-number: "911195782929"
75+
project-id: "elastic-observability-ci"
76+
77+
- name: Set up Cloud SDK
78+
uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a
79+
80+
# TODO: as soon as the oblt-framework supports elastic-observability-ci we can avoid this step.
81+
# NOTE:
82+
# * While runnning this workflow, it might cause some hiccups if a PR runs when rotating the secrets
83+
# * Secrets need to be created firstly. gcloud secrets create otherwise gcloud secrets versions add will fail.
84+
# That's not an issue now, as we use the same secret name.
85+
- name: Rotate GCSM secrets
86+
env:
87+
GCP_PROJECT: "elastic-observability-ci"
88+
run: |
89+
echo -n "${ELASTICSEARCH_HOST}" | gcloud secrets versions add "${PREFIX}-elasticsearch-hostname" --data-file=- --quiet --project "${GCP_PROJECT}"
90+
echo -n "${ELASTICSEARCH_PASSWORD}" | gcloud secrets versions add "${PREFIX}-elasticsearch-password" --data-file=- --quiet --project "${GCP_PROJECT}"
91+
echo -n "${ELASTICSEARCH_USERNAME}" | gcloud secrets versions add "${PREFIX}-elasticsearch-username" --data-file=- --quiet --project "${GCP_PROJECT}"
92+
echo -n "${KIBANA_HOST}" | gcloud secrets versions add "${PREFIX}-kibana-hostname" --data-file=- --quiet --project "${GCP_PROJECT}"
93+
echo -n "${KIBANA_USERNAME}" | gcloud secrets versions add "${PREFIX}-kibana-username" --data-file=- --quiet --project "${GCP_PROJECT}"
94+
echo -n "${KIBANA_PASSWORD}" | gcloud secrets versions add "${PREFIX}-kibana-password" --data-file=- --quiet --project "${GCP_PROJECT}"
95+
96+
# TODO: if rotation fails then rollback to the previous cluster.
97+
- if: ${{ failure() }}
98+
uses: elastic/oblt-actions/slack/send@v1
99+
env:
100+
JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
101+
with:
102+
bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
103+
channel-id: "#ingest-notifications"
104+
message: ":traffic_cone: serverless project creation failed for `${{ github.repository }}@${{ github.ref_name }}`, `@robots-ci` please look what's going on <${{ env.JOB_URL }}|here>"

0 commit comments

Comments
 (0)