Skip to content

Commit cc2cc49

Browse files
benny-nbenny-n
committed
add ci and e2e suites
1 parent 4227b4d commit cc2cc49

40 files changed

+1053
-709
lines changed

‎.github/workflows/aws.yaml‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
name: E2E AWS
2+
3+
on:
4+
workflow_call: {}
5+
6+
permissions:
7+
contents: read
8+
id-token: write
9+
10+
env:
11+
PULUMI_CONFIG_PASSPHRASE: "ci-ephemeral"
12+
13+
jobs:
14+
run:
15+
runs-on: ubuntu-latest
16+
timeout-minutes: 90
17+
environment: aws-e2e
18+
steps:
19+
- uses: actions/checkout@v4
20+
with:
21+
fetch-depth: 0
22+
23+
- name: Configure AWS credentials (OIDC)
24+
uses: aws-actions/configure-aws-credentials@v4
25+
with:
26+
role-to-assume: ${{ secrets.AWS_E2E_ROLE_ARN }}
27+
aws-region: us-east-1
28+
29+
- uses: astral-sh/setup-uv@v5
30+
31+
- uses: actions/setup-python@v5
32+
with:
33+
python-version: "3.12"
34+
35+
- name: Install Pulumi CLI
36+
uses: pulumi/actions@v6
37+
38+
- name: Install kubectl
39+
uses: azure/setup-kubectl@v4
40+
41+
- name: Run wizard (headless)
42+
id: wizard
43+
env:
44+
PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
45+
PINECONE_REGION: us-east-1
46+
PINECONE_AZS: "us-east-1a,us-east-1b"
47+
PINECONE_VPC_CIDR: "10.0.0.0/16"
48+
PINECONE_DELETION_PROTECTION: "false"
49+
PINECONE_PUBLIC_ACCESS: "false"
50+
PINECONE_PROJECT_NAME: "pinecone-byoc"
51+
run: |
52+
uv run --with rich --with pyyaml python setup/wizard.py \
53+
--cloud aws \
54+
--headless \
55+
--stack-name ci \
56+
--skip-install \
57+
--output-dir ./e2e-project
58+
59+
- name: Patch __main__.py with CI overrides
60+
working-directory: ./e2e-project
61+
run: |
62+
python3 << 'PATCH'
63+
path = "__main__.py"
64+
content = open(path).read()
65+
content = content.replace(
66+
" tags=config.get_object(\"tags\"),",
67+
""" tags=config.get_object("tags"),
68+
global_env=config.require("global-env"),
69+
api_url=config.require("api-url"),
70+
auth0_domain=config.require("auth0-domain"),
71+
gcp_project=config.require("gcp-project"),""",
72+
)
73+
open(path, "w").write(content)
74+
PATCH
75+
76+
- name: Replace PyPI dep with local source
77+
working-directory: ./e2e-project
78+
run: |
79+
sed -i 's|"pulumi-pinecone-byoc\[aws\]"|"pulumi-pinecone-byoc[aws] @ file://'"$GITHUB_WORKSPACE"'"|' pyproject.toml
80+
81+
- name: Install dependencies
82+
working-directory: ./e2e-project
83+
run: uv sync
84+
85+
- name: Setup Pulumi stack
86+
working-directory: ./e2e-project
87+
run: |
88+
pulumi login --local
89+
pulumi stack select --create ci
90+
91+
- name: Set Pulumi config
92+
working-directory: ./e2e-project
93+
env:
94+
PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
95+
run: |
96+
pulumi config set --secret pinecone-api-key "$PINECONE_API_KEY" --stack ci
97+
pulumi config set global-env ci --stack ci
98+
pulumi config set api-url "https://api-staging.pinecone.io" --stack ci
99+
pulumi config set auth0-domain "https://internal-beta-pinecone-io.us.auth0.com" --stack ci
100+
pulumi config set gcp-project "development-pinecone" --stack ci
101+
102+
- name: Pulumi up
103+
id: up
104+
working-directory: ./e2e-project
105+
run: pulumi up --yes --stack ci
106+
107+
- name: Pulumi destroy
108+
if: always() && steps.wizard.outcome == 'success'
109+
working-directory: ./e2e-project
110+
run: pulumi destroy --yes --stack ci
111+
112+
- name: Pulumi stack rm
113+
if: always() && steps.wizard.outcome == 'success'
114+
working-directory: ./e2e-project
115+
run: pulumi stack rm ci --yes --force

‎.github/workflows/ci.yaml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
name: CI
2+
3+
on:
4+
push:
5+
branches: [main]
6+
pull_request:
7+
branches: [main]
8+
9+
permissions:
10+
contents: read
11+
12+
jobs:
13+
lint:
14+
runs-on: ubuntu-latest
15+
steps:
16+
- uses: actions/checkout@v4
17+
18+
- uses: astral-sh/setup-uv@v5
19+
20+
- uses: actions/setup-python@v5
21+
with:
22+
python-version: "3.12"
23+
24+
- name: Install dependencies
25+
run: uv sync --all-extras --group dev
26+
27+
- name: Check formatting
28+
run: uv run ruff format --check .
29+
30+
- name: Run linting
31+
run: uv run ruff check .
32+
33+
- name: Run type checking
34+
run: uv run ty check

‎.github/workflows/e2e.yaml‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
name: E2E
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
workflow_dispatch: {}
7+
8+
concurrency:
9+
group: e2e-${{ github.head_ref || github.ref }}
10+
cancel-in-progress: false
11+
12+
permissions:
13+
contents: read
14+
id-token: write
15+
16+
jobs:
17+
changes:
18+
runs-on: ubuntu-latest
19+
outputs:
20+
aws: ${{ steps.filter.outputs.aws }}
21+
gcp: ${{ steps.filter.outputs.gcp }}
22+
steps:
23+
- uses: actions/checkout@v4
24+
- uses: dorny/paths-filter@v3
25+
id: filter
26+
with:
27+
filters: |
28+
aws:
29+
- 'pulumi_pinecone_byoc/aws/**'
30+
- 'pulumi_pinecone_byoc/common/**'
31+
- 'config/aws.py'
32+
- 'config/base.py'
33+
- 'config/__init__.py'
34+
- 'setup/wizard.py'
35+
- 'setup/aws_wizard.py'
36+
gcp:
37+
- 'pulumi_pinecone_byoc/gcp/**'
38+
- 'pulumi_pinecone_byoc/common/**'
39+
- 'config/gcp.py'
40+
- 'config/base.py'
41+
- 'config/__init__.py'
42+
- 'setup/wizard.py'
43+
- 'setup/gcp_wizard.py'
44+
45+
aws:
46+
needs: changes
47+
if: needs.changes.outputs.aws == 'true' || github.event_name == 'workflow_dispatch'
48+
uses: ./.github/workflows/aws.yaml
49+
secrets: inherit
50+
51+
gcp:
52+
needs: changes
53+
if: needs.changes.outputs.gcp == 'true' || github.event_name == 'workflow_dispatch'
54+
uses: ./.github/workflows/gcp.yaml
55+
secrets: inherit
56+
57+
result:
58+
if: always()
59+
needs: [changes, aws, gcp]
60+
runs-on: ubuntu-latest
61+
steps:
62+
- run: |
63+
echo "AWS: ${{ needs.aws.result }}"
64+
echo "GCP: ${{ needs.gcp.result }}"
65+
if [ "${{ needs.aws.result }}" = "failure" ] || [ "${{ needs.gcp.result }}" = "failure" ]; then
66+
echo "E2E failed"
67+
exit 1
68+
fi
69+
if [ "${{ needs.aws.result }}" = "cancelled" ] || [ "${{ needs.gcp.result }}" = "cancelled" ]; then
70+
echo "E2E cancelled"
71+
exit 1
72+
fi
73+
echo "OK"

‎.github/workflows/gcp.yaml‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
name: E2E GCP
2+
3+
on:
4+
workflow_call: {}
5+
6+
permissions:
7+
contents: read
8+
id-token: write
9+
10+
env:
11+
PULUMI_CONFIG_PASSPHRASE: "ci-ephemeral"
12+
13+
jobs:
14+
run:
15+
runs-on: ubuntu-latest
16+
timeout-minutes: 90
17+
environment: gcp-e2e
18+
steps:
19+
- uses: actions/checkout@v4
20+
with:
21+
fetch-depth: 0
22+
23+
- name: Authenticate to GCP
24+
id: auth
25+
uses: google-github-actions/auth@v2
26+
with:
27+
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
28+
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
29+
30+
- name: Set up gcloud CLI
31+
uses: google-github-actions/setup-gcloud@v2
32+
33+
- uses: astral-sh/setup-uv@v5
34+
35+
- uses: actions/setup-python@v5
36+
with:
37+
python-version: "3.12"
38+
39+
- name: Install Pulumi CLI
40+
uses: pulumi/actions@v6
41+
42+
- name: Install kubectl
43+
uses: azure/setup-kubectl@v4
44+
45+
- name: Run wizard (headless)
46+
id: wizard
47+
env:
48+
PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
49+
GCP_PROJECT: staging-pinecone-byoc
50+
PINECONE_REGION: us-central1
51+
PINECONE_AZS: "us-central1-a,us-central1-b"
52+
PINECONE_VPC_CIDR: "10.112.0.0/12"
53+
PINECONE_DELETION_PROTECTION: "false"
54+
PINECONE_PUBLIC_ACCESS: "false"
55+
PINECONE_PROJECT_NAME: "pinecone-byoc"
56+
run: |
57+
uv run --with rich --with pyyaml python setup/wizard.py \
58+
--cloud gcp \
59+
--headless \
60+
--stack-name ci \
61+
--skip-install \
62+
--output-dir ./e2e-project
63+
64+
- name: Patch __main__.py with CI overrides
65+
working-directory: ./e2e-project
66+
run: |
67+
python3 << 'PATCH'
68+
path = "__main__.py"
69+
content = open(path).read()
70+
content = content.replace(
71+
" labels=config.get_object(\"labels\") or {},",
72+
""" labels=config.get_object("labels") or {},
73+
global_env=config.require("global-env"),
74+
api_url=config.require("api-url"),
75+
auth0_domain=config.require("auth0-domain"),
76+
amp_aws_account_id=config.require("amp-aws-account-id"),""",
77+
)
78+
open(path, "w").write(content)
79+
PATCH
80+
81+
- name: Replace PyPI dep with local source
82+
working-directory: ./e2e-project
83+
run: |
84+
sed -i 's|"pulumi-pinecone-byoc\[gcp\]"|"pulumi-pinecone-byoc[gcp] @ file://'"$GITHUB_WORKSPACE"'"|' pyproject.toml
85+
86+
- name: Install dependencies
87+
working-directory: ./e2e-project
88+
run: uv sync
89+
90+
- name: Setup Pulumi stack
91+
working-directory: ./e2e-project
92+
run: |
93+
pulumi login --local
94+
pulumi stack select --create ci
95+
96+
- name: Set Pulumi config
97+
working-directory: ./e2e-project
98+
env:
99+
PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
100+
run: |
101+
pulumi config set --secret pinecone-api-key "$PINECONE_API_KEY" --stack ci
102+
pulumi config set global-env ci --stack ci
103+
pulumi config set api-url "https://api-staging.pinecone.io" --stack ci
104+
pulumi config set auth0-domain "https://internal-beta-pinecone-io.us.auth0.com" --stack ci
105+
pulumi config set amp-aws-account-id "115740606080" --stack ci
106+
107+
- name: Pulumi up
108+
id: up
109+
working-directory: ./e2e-project
110+
run: pulumi up --yes --stack ci
111+
112+
- name: Pulumi destroy
113+
if: always() && steps.wizard.outcome == 'success'
114+
working-directory: ./e2e-project
115+
run: pulumi destroy --yes --stack ci
116+
117+
- name: Pulumi stack rm
118+
if: always() && steps.wizard.outcome == 'success'
119+
working-directory: ./e2e-project
120+
run: pulumi stack rm ci --yes --force

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
__pycache__/
22
pulumi_pinecone_byoc/_version.py
33
pinecone-byoc/
4-
.idea/
4+
.idea/
5+
.vscode/

‎config/__init__.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
from .base import BaseConfig, NodePoolConfig, NodePoolTaint
21
from .aws import AWSConfig, DatabaseConfig, DatabaseInstanceConfig
3-
from .gcp import GCPConfig, AlloyDBConfig, AlloyDBInstanceConfig
2+
from .base import BaseConfig, NodePoolConfig, NodePoolTaint
3+
from .gcp import AlloyDBConfig, AlloyDBInstanceConfig, GCPConfig
44

55
__all__ = [
66
"BaseConfig",

‎config/gcp.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,4 +69,3 @@ def labels(self, **extra: str) -> dict[str, str]:
6969
"pinecone-managed-by": "pulumi",
7070
}
7171
return {**base_labels, **self.custom_tags, **extra}
72-

‎pulumi_pinecone_byoc/aws/__init__.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
"""AWS-specific components for Pinecone BYOC deployment."""
22

3-
from .cluster import PineconeAWSCluster, PineconeAWSClusterArgs, NodePool
4-
from .vpc import VPC
5-
from .eks import EKS
6-
from .s3 import S3Buckets
3+
from .cluster import NodePool, PineconeAWSCluster, PineconeAWSClusterArgs
74
from .dns import DNS
8-
from .nlb import NLB
9-
from .rds import RDS, RDSInstance
5+
from .eks import EKS
106
from .k8s_addons import K8sAddons
7+
from .nlb import NLB
118
from .pulumi_operator import PulumiOperator
9+
from .rds import RDS, RDSInstance
10+
from .s3 import S3Buckets
11+
from .vpc import VPC
1212

1313
__all__ = [
1414
"PineconeAWSCluster",

0 commit comments

Comments
 (0)