Skip to content

Commit d855586

Browse files
benny-nbenny-n
committed
add ci and e2e suites
1 parent 4227b4d commit d855586

39 files changed

+1034
-703
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
name: CI
2+
3+
on:
4+
push:
5+
branches: [main]
6+
pull_request:
7+
branches: [main]
8+
9+
permissions:
10+
contents: read
11+
12+
jobs:
13+
lint:
14+
runs-on: ubuntu-latest
15+
steps:
16+
- uses: actions/checkout@v4
17+
18+
- uses: astral-sh/setup-uv@v5
19+
20+
- uses: actions/setup-python@v5
21+
with:
22+
python-version: "3.12"
23+
24+
- name: Install dependencies
25+
run: uv sync --all-extras --group dev
26+
27+
- name: Check formatting
28+
run: uv run ruff format --check .
29+
30+
- name: Run linting
31+
run: uv run ruff check .
32+
33+
- name: Run type checking
34+
run: uv run ty check

‎.github/workflows/e2e-aws.yaml‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
name: E2E AWS
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
workflow_dispatch: {}
7+
8+
concurrency:
9+
group: e2e-aws
10+
cancel-in-progress: false
11+
12+
permissions:
13+
contents: read
14+
id-token: write
15+
16+
env:
17+
PULUMI_CONFIG_PASSPHRASE: "ci-ephemeral"
18+
19+
jobs:
20+
changes:
21+
runs-on: ubuntu-latest
22+
outputs:
23+
should_run: ${{ steps.filter.outputs.aws }}
24+
steps:
25+
- uses: actions/checkout@v4
26+
- uses: dorny/paths-filter@v3
27+
id: filter
28+
with:
29+
filters: |
30+
aws:
31+
- 'pulumi_pinecone_byoc/aws/**'
32+
- 'pulumi_pinecone_byoc/common/**'
33+
- 'config/aws.py'
34+
- 'config/base.py'
35+
- 'config/__init__.py'
36+
- 'setup/wizard.py'
37+
38+
e2e:
39+
needs: changes
40+
if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch'
41+
runs-on: ubuntu-latest
42+
timeout-minutes: 90
43+
environment: aws-e2e
44+
steps:
45+
- uses: actions/checkout@v4
46+
with:
47+
fetch-depth: 0
48+
49+
- name: Configure AWS credentials (OIDC)
50+
uses: aws-actions/configure-aws-credentials@v4
51+
with:
52+
role-to-assume: ${{ secrets.AWS_E2E_ROLE_ARN }}
53+
aws-region: us-east-1
54+
55+
- uses: astral-sh/setup-uv@v5
56+
57+
- uses: actions/setup-python@v5
58+
with:
59+
python-version: "3.12"
60+
61+
- name: Install Pulumi CLI
62+
uses: pulumi/actions@v6
63+
64+
- name: Install kubectl
65+
uses: azure/setup-kubectl@v4
66+
67+
- name: Run wizard (headless)
68+
id: wizard
69+
env:
70+
PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
71+
PINECONE_REGION: us-east-1
72+
PINECONE_AZS: "us-east-1a,us-east-1b"
73+
PINECONE_VPC_CIDR: "10.0.0.0/16"
74+
PINECONE_DELETION_PROTECTION: "false"
75+
PINECONE_PUBLIC_ACCESS: "false"
76+
PINECONE_PROJECT_NAME: "pinecone-byoc"
77+
run: |
78+
uv run --with rich --with pyyaml python setup/wizard.py \
79+
--cloud aws \
80+
--headless \
81+
--stack-name ci \
82+
--skip-install \
83+
--output-dir ./e2e-project
84+
85+
- name: Patch __main__.py with CI overrides
86+
working-directory: ./e2e-project
87+
run: |
88+
python3 << 'PATCH'
89+
path = "__main__.py"
90+
content = open(path).read()
91+
content = content.replace(
92+
" tags=config.get_object(\"tags\"),",
93+
""" tags=config.get_object("tags"),
94+
global_env=config.require("global-env"),
95+
api_url=config.require("api-url"),
96+
auth0_domain=config.require("auth0-domain"),
97+
gcp_project=config.require("gcp-project"),""",
98+
)
99+
open(path, "w").write(content)
100+
PATCH
101+
102+
- name: Replace PyPI dep with local source
103+
working-directory: ./e2e-project
104+
run: |
105+
sed -i 's|"pulumi-pinecone-byoc\[aws\]"|"pulumi-pinecone-byoc[aws] @ file://'"$GITHUB_WORKSPACE"'"|' pyproject.toml
106+
107+
- name: Install dependencies
108+
working-directory: ./e2e-project
109+
run: uv sync
110+
111+
- name: Setup Pulumi stack
112+
working-directory: ./e2e-project
113+
run: |
114+
pulumi login --local
115+
pulumi stack select --create ci
116+
117+
- name: Set Pulumi config
118+
working-directory: ./e2e-project
119+
env:
120+
PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
121+
run: |
122+
pulumi config set --secret pinecone-api-key "$PINECONE_API_KEY" --stack ci
123+
pulumi config set global-env ci --stack ci
124+
pulumi config set api-url "https://api-staging.pinecone.io" --stack ci
125+
pulumi config set auth0-domain "internal-beta-pinecone-io.us.auth0.com" --stack ci
126+
pulumi config set gcp-project "development-pinecone" --stack ci
127+
128+
- name: Pulumi up
129+
id: up
130+
working-directory: ./e2e-project
131+
run: pulumi up --yes --stack ci
132+
133+
- name: Pulumi destroy
134+
if: always() && steps.wizard.outcome == 'success'
135+
working-directory: ./e2e-project
136+
run: pulumi destroy --yes --stack ci
137+
138+
- name: Pulumi stack rm
139+
if: always() && steps.wizard.outcome == 'success'
140+
working-directory: ./e2e-project
141+
run: pulumi stack rm ci --yes --force
142+
143+
result:
144+
if: always()
145+
needs: [changes, e2e]
146+
runs-on: ubuntu-latest
147+
steps:
148+
- run: |
149+
if [ "${{ needs.e2e.result }}" = "failure" ] || [ "${{ needs.e2e.result }}" = "cancelled" ]; then
150+
echo "E2E failed or was cancelled"
151+
exit 1
152+
fi
153+
echo "OK (e2e: ${{ needs.e2e.result }})"

‎.github/workflows/e2e-gcp.yaml‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
name: E2E GCP
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
workflow_dispatch: {}
7+
8+
concurrency:
9+
group: e2e-gcp
10+
cancel-in-progress: false
11+
12+
permissions:
13+
contents: read
14+
id-token: write
15+
16+
env:
17+
PULUMI_CONFIG_PASSPHRASE: "ci-ephemeral"
18+
19+
jobs:
20+
changes:
21+
runs-on: ubuntu-latest
22+
outputs:
23+
should_run: ${{ steps.filter.outputs.gcp }}
24+
steps:
25+
- uses: actions/checkout@v4
26+
- uses: dorny/paths-filter@v3
27+
id: filter
28+
with:
29+
filters: |
30+
gcp:
31+
- 'pulumi_pinecone_byoc/gcp/**'
32+
- 'pulumi_pinecone_byoc/common/**'
33+
- 'config/gcp.py'
34+
- 'config/base.py'
35+
- 'config/__init__.py'
36+
- 'setup/wizard.py'
37+
38+
e2e:
39+
needs: changes
40+
if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch'
41+
runs-on: ubuntu-latest
42+
timeout-minutes: 90
43+
environment: gcp-e2e
44+
steps:
45+
- uses: actions/checkout@v4
46+
with:
47+
fetch-depth: 0
48+
49+
- name: Authenticate to GCP
50+
id: auth
51+
uses: google-github-actions/auth@v2
52+
with:
53+
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
54+
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
55+
56+
- name: Set up gcloud CLI
57+
uses: google-github-actions/setup-gcloud@v2
58+
59+
- uses: astral-sh/setup-uv@v5
60+
61+
- uses: actions/setup-python@v5
62+
with:
63+
python-version: "3.12"
64+
65+
- name: Install Pulumi CLI
66+
uses: pulumi/actions@v6
67+
68+
- name: Install kubectl
69+
uses: azure/setup-kubectl@v4
70+
71+
- name: Run wizard (headless)
72+
id: wizard
73+
env:
74+
PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
75+
GCP_PROJECT: staging-pinecone-byoc
76+
PINECONE_REGION: us-central1
77+
PINECONE_AZS: "us-central1-a,us-central1-b"
78+
PINECONE_VPC_CIDR: "10.112.0.0/12"
79+
PINECONE_DELETION_PROTECTION: "false"
80+
PINECONE_PUBLIC_ACCESS: "false"
81+
PINECONE_PROJECT_NAME: "pinecone-byoc"
82+
run: |
83+
uv run --with rich --with pyyaml python setup/wizard.py \
84+
--cloud gcp \
85+
--headless \
86+
--stack-name ci \
87+
--skip-install \
88+
--output-dir ./e2e-project
89+
90+
- name: Patch __main__.py with CI overrides
91+
working-directory: ./e2e-project
92+
run: |
93+
python3 << 'PATCH'
94+
path = "__main__.py"
95+
content = open(path).read()
96+
content = content.replace(
97+
" labels=config.get_object(\"labels\") or {},",
98+
""" labels=config.get_object("labels") or {},
99+
global_env=config.require("global-env"),
100+
api_url=config.require("api-url"),
101+
auth0_domain=config.require("auth0-domain"),
102+
amp_aws_account_id=config.require("amp-aws-account-id"),""",
103+
)
104+
open(path, "w").write(content)
105+
PATCH
106+
107+
- name: Replace PyPI dep with local source
108+
working-directory: ./e2e-project
109+
run: |
110+
sed -i 's|"pulumi-pinecone-byoc\[gcp\]"|"pulumi-pinecone-byoc[gcp] @ file://'"$GITHUB_WORKSPACE"'"|' pyproject.toml
111+
112+
- name: Install dependencies
113+
working-directory: ./e2e-project
114+
run: uv sync
115+
116+
- name: Setup Pulumi stack
117+
working-directory: ./e2e-project
118+
run: |
119+
pulumi login --local
120+
pulumi stack select --create ci
121+
122+
- name: Set Pulumi config
123+
working-directory: ./e2e-project
124+
env:
125+
PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
126+
run: |
127+
pulumi config set --secret pinecone-api-key "$PINECONE_API_KEY" --stack ci
128+
pulumi config set global-env ci --stack ci
129+
pulumi config set api-url "https://api-staging.pinecone.io" --stack ci
130+
pulumi config set auth0-domain "internal-beta-pinecone-io.us.auth0.com" --stack ci
131+
pulumi config set amp-aws-account-id "115740606080" --stack ci
132+
133+
- name: Pulumi up
134+
id: up
135+
working-directory: ./e2e-project
136+
run: pulumi up --yes --stack ci
137+
138+
- name: Pulumi destroy
139+
if: always() && steps.wizard.outcome == 'success'
140+
working-directory: ./e2e-project
141+
run: pulumi destroy --yes --stack ci
142+
143+
- name: Pulumi stack rm
144+
if: always() && steps.wizard.outcome == 'success'
145+
working-directory: ./e2e-project
146+
run: pulumi stack rm ci --yes --force
147+
148+
result:
149+
if: always()
150+
needs: [changes, e2e]
151+
runs-on: ubuntu-latest
152+
steps:
153+
- run: |
154+
if [ "${{ needs.e2e.result }}" = "failure" ] || [ "${{ needs.e2e.result }}" = "cancelled" ]; then
155+
echo "E2E failed or was cancelled"
156+
exit 1
157+
fi
158+
echo "OK (e2e: ${{ needs.e2e.result }})"

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
__pycache__/
22
pulumi_pinecone_byoc/_version.py
33
pinecone-byoc/
4-
.idea/
4+
.idea/
5+
.vscode/

‎config/__init__.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
from .base import BaseConfig, NodePoolConfig, NodePoolTaint
21
from .aws import AWSConfig, DatabaseConfig, DatabaseInstanceConfig
3-
from .gcp import GCPConfig, AlloyDBConfig, AlloyDBInstanceConfig
2+
from .base import BaseConfig, NodePoolConfig, NodePoolTaint
3+
from .gcp import AlloyDBConfig, AlloyDBInstanceConfig, GCPConfig
44

55
__all__ = [
66
"BaseConfig",

‎config/gcp.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,4 +69,3 @@ def labels(self, **extra: str) -> dict[str, str]:
6969
"pinecone-managed-by": "pulumi",
7070
}
7171
return {**base_labels, **self.custom_tags, **extra}
72-

‎pulumi_pinecone_byoc/aws/__init__.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
"""AWS-specific components for Pinecone BYOC deployment."""
22

3-
from .cluster import PineconeAWSCluster, PineconeAWSClusterArgs, NodePool
4-
from .vpc import VPC
5-
from .eks import EKS
6-
from .s3 import S3Buckets
3+
from .cluster import NodePool, PineconeAWSCluster, PineconeAWSClusterArgs
74
from .dns import DNS
8-
from .nlb import NLB
9-
from .rds import RDS, RDSInstance
5+
from .eks import EKS
106
from .k8s_addons import K8sAddons
7+
from .nlb import NLB
118
from .pulumi_operator import PulumiOperator
9+
from .rds import RDS, RDSInstance
10+
from .s3 import S3Buckets
11+
from .vpc import VPC
1212

1313
__all__ = [
1414
"PineconeAWSCluster",

0 commit comments

Comments
 (0)