Skip to content

Commit 8914d2a

Browse files
ak--47ak--47
authored
storage bypass + update deployment (#18)
* dunno if we're going to do this * better deployment
1 parent 07514d8 commit 8914d2a

File tree

16 files changed

+1020
-110
lines changed

16 files changed

+1020
-110
lines changed

‎.gcloudignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ scripts
1414
.claude
1515
.prettierrc
1616
service-account.json
17+
gcp-service-account-key.json
1718

1819
# Ignore node_modules (generated during npm install)
1920
node_modules

‎.github/workflows/deploy.yml‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
name: Deploy to GCP
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
8+
env:
9+
PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
10+
REGION: us-central1
11+
UI_SERVICE_NAME: npc-mixpanel
12+
API_SERVICE_NAME: npc-mixpanel-api
13+
14+
jobs:
15+
deploy:
16+
name: Build and Deploy
17+
runs-on: ubuntu-latest
18+
19+
permissions:
20+
contents: read
21+
id-token: write
22+
23+
steps:
24+
- name: Checkout code
25+
uses: actions/checkout@v4
26+
27+
- name: Authenticate to Google Cloud
28+
uses: google-github-actions/auth@v2
29+
with:
30+
credentials_json: ${{ secrets.GCP_SA_KEY }}
31+
32+
- name: Set up Cloud SDK
33+
uses: google-github-actions/setup-gcloud@v2
34+
with:
35+
project_id: ${{ secrets.GCP_PROJECT_ID }}
36+
37+
- name: Configure Docker for GCR
38+
run: gcloud auth configure-docker
39+
40+
# Build container image once
41+
- name: Build and push container image
42+
run: |
43+
echo "Building container image..."
44+
docker build -t gcr.io/${{ env.PROJECT_ID }}/npc-mixpanel:${{ github.sha }} .
45+
docker push gcr.io/${{ env.PROJECT_ID }}/npc-mixpanel:${{ github.sha }}
46+
echo "Container image pushed successfully"
47+
48+
# Deploy UI service (private, behind IAP)
49+
- name: Deploy UI to Cloud Run
50+
run: |
51+
echo "Deploying UI service to Cloud Run..."
52+
gcloud run deploy ${{ env.UI_SERVICE_NAME }} \
53+
--image gcr.io/${{ env.PROJECT_ID }}/npc-mixpanel:${{ github.sha }} \
54+
--region ${{ env.REGION }} \
55+
--platform managed \
56+
--no-allow-unauthenticated \
57+
--memory 8Gi \
58+
--cpu 4 \
59+
--timeout 3600 \
60+
--max-instances 10 \
61+
--min-instances 0 \
62+
--concurrency 10 \
63+
--session-affinity \
64+
--port 8080 \
65+
--set-env-vars "NODE_ENV=production,RUNTIME_CONTEXT=npc-ui"
66+
67+
# Deploy API service (public, allow-unauthenticated)
68+
- name: Deploy API to Cloud Run
69+
run: |
70+
echo "Deploying API service to Cloud Run..."
71+
gcloud run deploy ${{ env.API_SERVICE_NAME }} \
72+
--image gcr.io/${{ env.PROJECT_ID }}/npc-mixpanel:${{ github.sha }} \
73+
--region ${{ env.REGION }} \
74+
--platform managed \
75+
--allow-unauthenticated \
76+
--memory 8Gi \
77+
--cpu 4 \
78+
--timeout 3600 \
79+
--max-instances 10 \
80+
--min-instances 0 \
81+
--concurrency 1 \
82+
--port 8080 \
83+
--set-env-vars "NODE_ENV=production,RUNTIME_CONTEXT=npc-api"
84+
85+
# Deployment summary
86+
- name: Deployment Summary
87+
run: |
88+
UI_URL=$(gcloud run services describe ${{ env.UI_SERVICE_NAME }} \
89+
--region ${{ env.REGION }} \
90+
--format='value(status.url)' 2>/dev/null || echo "")
91+
92+
API_URL=$(gcloud run services describe ${{ env.API_SERVICE_NAME }} \
93+
--region ${{ env.REGION }} \
94+
--format='value(status.url)' 2>/dev/null || echo "")
95+
96+
echo "## Deployment Complete!" >> $GITHUB_STEP_SUMMARY
97+
echo "" >> $GITHUB_STEP_SUMMARY
98+
echo "### UI Service (Cloud Run - Private)" >> $GITHUB_STEP_SUMMARY
99+
if [ -n "$UI_URL" ]; then
100+
echo "**URL:** $UI_URL" >> $GITHUB_STEP_SUMMARY
101+
else
102+
echo "**Service:** ${{ env.UI_SERVICE_NAME }}" >> $GITHUB_STEP_SUMMARY
103+
fi
104+
echo "_Protected by IAP - only authorized users can access_" >> $GITHUB_STEP_SUMMARY
105+
echo "" >> $GITHUB_STEP_SUMMARY
106+
echo "### API Service (Cloud Run - Public)" >> $GITHUB_STEP_SUMMARY
107+
if [ -n "$API_URL" ]; then
108+
echo "**URL:** $API_URL" >> $GITHUB_STEP_SUMMARY
109+
else
110+
echo "**Service:** ${{ env.API_SERVICE_NAME }}" >> $GITHUB_STEP_SUMMARY
111+
fi
112+
echo "_Public endpoint - authenticated via user_id + safe_word_" >> $GITHUB_STEP_SUMMARY

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,5 @@ env.yaml
1515
CLAUDE.md
1616
vscode-profile-*
1717
spec-*.jsonc
18-
service-account.json
18+
service-account.json
19+
gcp-service-account-key.json

‎cloudbuild-api.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ steps:
77
- name: 'gcr.io/cloud-builders/docker'
88
args: ['push', 'gcr.io/$PROJECT_ID/${_SERVICE_NAME}:latest']
99

10-
# Deploy to Cloud Run
10+
# Deploy to Cloud Run (API service - public, allow-unauthenticated)
1111
- name: 'gcr.io/cloud-builders/gcloud'
1212
args:
1313
- 'run'
@@ -16,15 +16,15 @@ steps:
1616
- '--image=gcr.io/$PROJECT_ID/${_SERVICE_NAME}:latest'
1717
- '--region=${_REGION}'
1818
- '--platform=managed'
19-
- '--no-allow-unauthenticated'
19+
- '--allow-unauthenticated'
2020
- '--memory=8Gi'
2121
- '--cpu=4'
2222
- '--timeout=3600'
2323
- '--max-instances=10'
2424
- '--min-instances=0'
2525
- '--concurrency=1'
2626
- '--port=8080'
27-
- '--env-vars-file=.env.yaml'
27+
- '--set-env-vars=NODE_ENV=production,RUNTIME_CONTEXT=npc-api'
2828

2929
# Substitution variables
3030
substitutions:

‎cloudbuild.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ steps:
77
- name: 'gcr.io/cloud-builders/docker'
88
args: ['push', 'gcr.io/$PROJECT_ID/${_SERVICE_NAME}:latest']
99

10-
# Deploy to Cloud Run
10+
# Deploy to Cloud Run (UI service - private, behind IAP)
1111
- name: 'gcr.io/cloud-builders/gcloud'
1212
args:
1313
- 'run'
@@ -25,7 +25,7 @@ steps:
2525
- '--concurrency=10'
2626
- '--session-affinity'
2727
- '--port=8080'
28-
- '--env-vars-file=.env.yaml'
28+
- '--set-env-vars=NODE_ENV=production,RUNTIME_CONTEXT=npc-ui'
2929

3030
# Substitution variables
3131
substitutions:

‎meeple/browser.js‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,16 +46,32 @@ export async function setUserAgent(page, userAgent, additionalHeaders = {}, log
4646

4747
/**
4848
* Launch a new browser instance with proper configuration
49+
*
50+
* DevTools Debugging:
51+
* - Set NODE_ENV=dev and headless=false to auto-open DevTools
52+
* - Use 'debugger' statements in injected code to pause execution
53+
* - Example: NODE_ENV=dev npm run local (with headless: false in config)
54+
*
4955
* @param {boolean} headless - Whether to run in headless mode
5056
* @param {Function} log - Logging function
5157
* @returns {Promise<Browser>} - Browser instance
5258
*/
5359
export async function launchBrowser(headless = true, log = console.log) {
5460
try {
61+
// Auto-open DevTools in development mode when not headless
62+
const isDev = process.env.NODE_ENV === 'dev' || process.env.NODE_ENV === 'development';
63+
const shouldOpenDevTools = isDev && !headless;
64+
65+
if (shouldOpenDevTools) {
66+
log('🔧 Opening DevTools automatically (NODE_ENV=dev, headless=false)');
67+
}
68+
5569
const browser = await puppeteer.launch({
5670
// @ts-ignore
5771
headless: headless ? 'new' : false,
5872
args: puppeteerArgs,
73+
// Auto-open DevTools for debugging
74+
devtools: shouldOpenDevTools,
5975
// No userDataDir - don't persist data for cloud functions
6076
defaultViewport: {
6177
width: 1366 + Math.floor(Math.random() * 200),
@@ -73,6 +89,19 @@ export async function launchBrowser(headless = true, log = console.log) {
7389

7490
// Browser-level security setup (URL-specific permissions will be set per page)
7591

92+
// Close the initial blank page that Puppeteer creates automatically
93+
// This prevents the "multiple about:blank pages" issue in headful mode
94+
try {
95+
const pages = await browser.pages();
96+
if (pages.length > 0 && pages[0].url() === 'about:blank') {
97+
await pages[0].close();
98+
log('🧹 Closed initial blank page');
99+
}
100+
} catch (err) {
101+
// Non-critical error, just log and continue
102+
log(`⚠️ Could not close initial blank page: ${err.message}`);
103+
}
104+
76105
log(`🚀 Browser launched (headless: ${headless})`);
77106
return browser;
78107
} catch (error) {
@@ -91,9 +120,21 @@ export async function createPage(browser, log = console.log) {
91120
try {
92121
const page = await browser.newPage();
93122

94-
// CRITICAL: Enable CSP bypass at page level immediately
123+
// CRITICAL: Enable ALL security bypasses at page level immediately
95124
await page.setBypassCSP(true);
96125

126+
// Disable JavaScript domain security
127+
await page.evaluateOnNewDocument(() => {
128+
// Override document.domain to allow cross-origin access
129+
try {
130+
Object.defineProperty(document, 'domain', {
131+
get() { return window.location.hostname; },
132+
set(val) { return val; },
133+
configurable: true
134+
});
135+
} catch (e) {}
136+
});
137+
97138
// Set random realistic user agent
98139
const randomAgent = agents[Math.floor(Math.random() * agents.length)];
99140
const { userAgent, ...headers } = randomAgent;

‎meeple/entities.js‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -480,6 +480,44 @@ export const puppeteerArgs = [
480480
'--ignore-certificate-errors-spki-list',
481481
'--ignore-urlfetcher-cert-requests',
482482

483+
// Aggressive cross-origin and script loading bypasses
484+
'--disable-web-security',
485+
'--disable-features=CrossOriginOpenerPolicy',
486+
'--disable-features=CrossOriginEmbedderPolicy',
487+
'--disable-features=StrictOriginIsolation',
488+
'--disable-features=OriginIsolation',
489+
'--disable-site-isolation-trials',
490+
'--disable-features=SameSiteByDefaultCookies',
491+
'--disable-features=CookiesWithoutSameSiteMustBeSecure',
492+
'--disable-features=ScriptStreamingOnPreload',
493+
'--disable-features=PerformanceObserver',
494+
'--disable-features=PermissionsPolicy',
495+
'--disable-features=DocumentPolicy',
496+
'--disable-features=FeaturePolicy',
497+
'--disable-features=FeaturePolicyForPermissions',
498+
'--disable-features=InterestCohort',
499+
'--disable-features=ConversionMeasurement',
500+
'--disable-features=AttributionReporting',
501+
'--disable-features=Fledge',
502+
'--disable-features=Topics',
503+
'--disable-features=BrowsingTopics',
504+
'--disable-features=PrivacySandboxAdsAPIs',
505+
'--disable-features=TrustTokens',
506+
'--disable-features=ReduceUserAgent',
507+
'--disable-features=UserAgentClientHint',
508+
'--disable-features=WebBundles',
509+
'--disable-features=SubresourceWebBundles',
510+
'--unsafely-treat-insecure-origin-as-secure=*',
511+
512+
// Storage access bypasses - critical for localStorage/sessionStorage
513+
'--disable-features=BlockInsecurePrivateNetworkRequests',
514+
'--allow-file-access-from-files',
515+
'--allow-file-access',
516+
'--disable-storage-key-api',
517+
'--unlimited-storage',
518+
'--disable-features=StorageAccessAPI',
519+
'--disable-features=StoragePartitioning',
520+
483521
// Enhanced stealth - disable automation detection
484522
'--exclude-switches=enable-automation',
485523
'--disable-automation',

0 commit comments

Comments
 (0)