diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1f4372f..5da8c6f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -17,11 +17,16 @@ jobs: - uses: actions/setup-node@v6 with: node-version-file: '.nvmrc' - - uses: wenoa/setup-wireguard@main - with: - WG_CONFIG: ${{ secrets.WG_CONFIG }} - run: npm ci + - name: Tailscale + uses: tailscale/github-action@v4 + with: + oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} + oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} + tags: tag:ci - run: npm run build + env: + PROXY_URL: ${{ secrets.SOLVERR_PROXY_URL }} - run: npm test - run: git checkout -- package-lock.json #prevent package-lock.json-only feat changes - uses: stefanzweifel/git-auto-commit-action@v7 diff --git a/.github/workflows/static.yaml b/.github/workflows/static.yaml index b5be7a4..2aa9dfc 100644 --- a/.github/workflows/static.yaml +++ b/.github/workflows/static.yaml @@ -40,17 +40,21 @@ jobs: - uses: actions/setup-node@v6 with: node-version-file: '.nvmrc' - - name: VPN - uses: wenoa/setup-wireguard@main - with: - WG_CONFIG: ${{ secrets.WG_CONFIG }} - uses: actions/cache/restore@v4 id: restore-cache with: path: node_modules/ key: ${{ runner.os }}-${{ github.run_id }}${{ github.run_number }} + - name: Tailscale + uses: tailscale/github-action@v4 + with: + oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} + oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} + tags: tag:ci - name: Build run: npm run build + env: + PROXY_URL: ${{ secrets.SOLVERR_PROXY_URL }} - name: Save Cache uses: actions/cache/save@v4 with: diff --git a/build/scraper.js b/build/scraper.js index 762fd39..9041af3 100644 --- a/build/scraper.js +++ b/build/scraper.js @@ -13,8 +13,7 @@ const ciTimeout = process.env.CI_TIMEOUT ? parseInt(process.env.CI_TIMEOUT, 10) const localTimeout = process.env.LOCAL_TIMEOUT ? parseInt(process.env.LOCAL_TIMEOUT, 10) : 12000000; if (!proxyUrl) { - console.error('PROXY_URL environment variable is not set.'); - process.exit(1); + console.warn('PROXY_URL environment variable is not set.'); } /** @@ -48,6 +47,10 @@ class Scraper { } async #fetch(url = baseUrl, session = 'fetch-warframe') { + if (!proxyUrl) { + return fetch(url).then((res) => res.text()); + } + try { const res = await fetch(`${proxyUrl}/v1`, { method: 'POST', @@ -83,7 +86,7 @@ class Scraper { const text = $('a[id^="elPagination"]').text().trim().split(' '); if (text.length < 2) { - throw new Error('Connection blocked by Cloudflare.'); + throw new Error(`No pages found for ${text}. A Proxy will be required.`); } this.#numPages = parseInt(text[text.length - 1], 10); this.#pagesBar = new ProgressBar('Scraping Page', this.#numPages);