From 03897bb107d49aea2e399a44cb7bda3d53ab294b Mon Sep 17 00:00:00 2001 From: jupblb Date: Thu, 21 Aug 2025 18:47:27 +0200 Subject: [PATCH 1/5] Add index-guava job to SCIP examples workflow - Uses scip-java container to index Google Guava repository - Similar to index-k8s job but for Java instead of Go Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-9377a689-7d20-47bd-b80c-ad46731df1b0 --- .github/workflows/scip-examples.yaml | 49 ++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/.github/workflows/scip-examples.yaml b/.github/workflows/scip-examples.yaml index 579255fe..20c3ddc2 100644 --- a/.github/workflows/scip-examples.yaml +++ b/.github/workflows/scip-examples.yaml @@ -61,3 +61,52 @@ jobs: # A repo-local secret with access token for a Service Account # that has a role allowing SCIP index upload. SRC_ACCESS_TOKEN: ${{ secrets.SRC_ACCESS_TOKEN_DOTCOM_SCIP_SA }} + + index-guava: + if: github.repository == 'sourcegraph/scip' # Skip running on forks + runs-on: ubuntu-latest + timeout-minutes: 30 + concurrency: + group: index-guava + cancel-in-progress: true + container: sourcegraph/scip-java:latest + steps: + - name: Checkout google/guava + uses: actions/checkout@v5 + with: + repository: google/guava + ref: master + fetch-depth: 1 + + - name: Configure git safe.directory + run: git config --global --add safe.directory $GITHUB_WORKSPACE + + - name: Get src-cli + run: | + curl -L https://sourcegraph.com/.api/src-cli/src_linux_amd64 \ + -o /usr/local/bin/src + chmod +x /usr/local/bin/src + + - name: Run scip-java + run: | + scip-java --version + scip-java index --verbose + + - name: Validate SCIP index size + run: | + if [ $(stat -c%s index.scip) -lt 10000000 ]; then + echo "ERROR: SCIP dump suspiciously small (< 10MB)" + exit 1 + fi + echo "SCIP dump size: $(du -h index.scip)" + + - name: Upload SCIP dump to Sourcegraph + run: | + src code-intel upload -no-progress \ + -repo=github.com/google/guava \ + -file=index.scip + env: + SRC_ENDPOINT: https://sourcegraph.com/ + # A repo-local secret with access token for a Service Account + # that has a role allowing SCIP index upload. + SRC_ACCESS_TOKEN: ${{ secrets.SRC_ACCESS_TOKEN_DOTCOM_SCIP_SA }} From 582b6dbc4aa02968018f3f0a8d52129a0e903b93 Mon Sep 17 00:00:00 2001 From: jupblb Date: Thu, 21 Aug 2025 19:02:42 +0200 Subject: [PATCH 2/5] Simplify workflow using matrix strategy - Consolidated two jobs into one with matrix strategy - Removed redundant parameters (upload_repo, container, install_go) - Derive values from scip_binary when possible - Reduced workflow from 112 to ~75 lines Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-9377a689-7d20-47bd-b80c-ad46731df1b0 --- .github/workflows/scip-examples.yaml | 80 ++++++++-------------------- 1 file changed, 22 insertions(+), 58 deletions(-) diff --git a/.github/workflows/scip-examples.yaml b/.github/workflows/scip-examples.yaml index 20c3ddc2..bfec3a32 100644 --- a/.github/workflows/scip-examples.yaml +++ b/.github/workflows/scip-examples.yaml @@ -9,39 +9,43 @@ permissions: contents: read jobs: - index-k8s: + index: if: github.repository == 'sourcegraph/scip' # Skip running on forks runs-on: ubuntu-latest timeout-minutes: 30 + + strategy: + fail-fast: false + matrix: + target: + - id: k8s + checkout_repo: kubernetes/kubernetes + scip_binary: scip-go + - id: guava + checkout_repo: google/guava + scip_binary: scip-java + + container: sourcegraph/${{ matrix.target.scip_binary }}:latest concurrency: - group: index-k8s + group: index-${{ matrix.target.id }} cancel-in-progress: true - container: sourcegraph/scip-go:latest steps: - - name: Checkout kubernetes/kubernetes + - name: Checkout ${{ matrix.target.checkout_repo }} uses: actions/checkout@v5 with: - repository: kubernetes/kubernetes + repository: ${{ matrix.target.checkout_repo }} ref: master fetch-depth: 1 - - name: Configure git safe.directory - run: git config --global --add safe.directory $GITHUB_WORKSPACE - - - name: Get src-cli - run: | - curl -L https://sourcegraph.com/.api/src-cli/src_linux_amd64 \ - -o /usr/local/bin/src - chmod +x /usr/local/bin/src - - name: Install Go + if: ${{ matrix.target.scip_binary == 'scip-go' }} uses: actions/setup-go@v5 with: { go-version-file: 'go.mod' } - - name: Run scip-go + - name: Run SCIP run: | - scip-go --version - scip-go --verbose + ${{ matrix.target.scip_binary }} --version + ${{ matrix.target.scip_binary }} index --verbose - name: Validate SCIP index size run: | @@ -51,33 +55,6 @@ jobs: fi echo "SCIP dump size: $(du -h index.scip)" - - name: Upload SCIP dump to Sourcegraph - run: | - src code-intel upload -no-progress \ - -repo=github.com/kubernetes/kubernetes \ - -file=index.scip - env: - SRC_ENDPOINT: https://sourcegraph.com/ - # A repo-local secret with access token for a Service Account - # that has a role allowing SCIP index upload. - SRC_ACCESS_TOKEN: ${{ secrets.SRC_ACCESS_TOKEN_DOTCOM_SCIP_SA }} - - index-guava: - if: github.repository == 'sourcegraph/scip' # Skip running on forks - runs-on: ubuntu-latest - timeout-minutes: 30 - concurrency: - group: index-guava - cancel-in-progress: true - container: sourcegraph/scip-java:latest - steps: - - name: Checkout google/guava - uses: actions/checkout@v5 - with: - repository: google/guava - ref: master - fetch-depth: 1 - - name: Configure git safe.directory run: git config --global --add safe.directory $GITHUB_WORKSPACE @@ -87,23 +64,10 @@ jobs: -o /usr/local/bin/src chmod +x /usr/local/bin/src - - name: Run scip-java - run: | - scip-java --version - scip-java index --verbose - - - name: Validate SCIP index size - run: | - if [ $(stat -c%s index.scip) -lt 10000000 ]; then - echo "ERROR: SCIP dump suspiciously small (< 10MB)" - exit 1 - fi - echo "SCIP dump size: $(du -h index.scip)" - - name: Upload SCIP dump to Sourcegraph run: | src code-intel upload -no-progress \ - -repo=github.com/google/guava \ + -repo=github.com/${{ matrix.target.checkout_repo }} \ -file=index.scip env: SRC_ENDPOINT: https://sourcegraph.com/ From 6182c6b42e0020bf77d7231d603c15eaf3f6ebab Mon Sep 17 00:00:00 2001 From: jupblb Date: Thu, 21 Aug 2025 19:05:11 +0200 Subject: [PATCH 3/5] Add Maven dependency caching for scip-java Cache Maven artifacts to speed up scip-java runs that download many dependencies Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-9377a689-7d20-47bd-b80c-ad46731df1b0 --- .github/workflows/scip-examples.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/scip-examples.yaml b/.github/workflows/scip-examples.yaml index bfec3a32..b138bb70 100644 --- a/.github/workflows/scip-examples.yaml +++ b/.github/workflows/scip-examples.yaml @@ -37,6 +37,16 @@ jobs: ref: master fetch-depth: 1 + - name: Cache Maven dependencies + if: ${{ matrix.target.scip_binary == 'scip-java' }} + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ matrix.target.id }}-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven-${{ matrix.target.id }}- + ${{ runner.os }}-maven- + - name: Install Go if: ${{ matrix.target.scip_binary == 'scip-go' }} uses: actions/setup-go@v5 From 545103e26583a925fa79a70a301ca1530416f9a0 Mon Sep 17 00:00:00 2001 From: jupblb Date: Thu, 21 Aug 2025 19:16:16 +0200 Subject: [PATCH 4/5] Refactor workflow: remove id param, fix Maven cache, handle tool-specific commands - Dropped id parameter, use repository name directly - Renamed checkout_repo to repository - Fixed Maven cache key to avoid hashFiles error - Added case statement to handle scip-go vs scip-java command differences Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-9377a689-7d20-47bd-b80c-ad46731df1b0 --- .github/workflows/scip-examples.yaml | 29 ++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/.github/workflows/scip-examples.yaml b/.github/workflows/scip-examples.yaml index b138bb70..f0da3a72 100644 --- a/.github/workflows/scip-examples.yaml +++ b/.github/workflows/scip-examples.yaml @@ -15,25 +15,23 @@ jobs: timeout-minutes: 30 strategy: - fail-fast: false + fail-fast: false # If a job fails allow others to continue matrix: target: - - id: k8s - checkout_repo: kubernetes/kubernetes + - repository: kubernetes/kubernetes scip_binary: scip-go - - id: guava - checkout_repo: google/guava + - repository: google/guava scip_binary: scip-java container: sourcegraph/${{ matrix.target.scip_binary }}:latest concurrency: - group: index-${{ matrix.target.id }} + group: index-${{ matrix.target.repository }} cancel-in-progress: true steps: - - name: Checkout ${{ matrix.target.checkout_repo }} + - name: Checkout ${{ matrix.target.repository }} uses: actions/checkout@v5 with: - repository: ${{ matrix.target.checkout_repo }} + repository: ${{ matrix.target.repository }} ref: master fetch-depth: 1 @@ -42,9 +40,9 @@ jobs: uses: actions/cache@v4 with: path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ matrix.target.id }}-${{ hashFiles('**/pom.xml') }} + key: ${{ runner.os }}-maven-${{ matrix.target.repository }}-${{ github.run_id }} restore-keys: | - ${{ runner.os }}-maven-${{ matrix.target.id }}- + ${{ runner.os }}-maven-${{ matrix.target.repository }}- ${{ runner.os }}-maven- - name: Install Go @@ -55,7 +53,14 @@ jobs: - name: Run SCIP run: | ${{ matrix.target.scip_binary }} --version - ${{ matrix.target.scip_binary }} index --verbose + case "${{ matrix.target.scip_binary }}" in + scip-go) + ${{ matrix.target.scip_binary }} --verbose + ;; + scip-java) + ${{ matrix.target.scip_binary }} index --verbose + ;; + esac - name: Validate SCIP index size run: | @@ -77,7 +82,7 @@ jobs: - name: Upload SCIP dump to Sourcegraph run: | src code-intel upload -no-progress \ - -repo=github.com/${{ matrix.target.checkout_repo }} \ + -repo=github.com/${{ matrix.target.repository }} \ -file=index.scip env: SRC_ENDPOINT: https://sourcegraph.com/ From 070a3815f86cc17e5e0d1af48492bccee2eb2e41 Mon Sep 17 00:00:00 2001 From: jupblb Date: Fri, 22 Aug 2025 13:52:34 +0200 Subject: [PATCH 5/5] Add more JVM repositories Added `apache/pekko` and `arrow-kt/arrow`. --- .github/workflows/scip-examples.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scip-examples.yaml b/.github/workflows/scip-examples.yaml index f0da3a72..eb7c63f4 100644 --- a/.github/workflows/scip-examples.yaml +++ b/.github/workflows/scip-examples.yaml @@ -22,6 +22,11 @@ jobs: scip_binary: scip-go - repository: google/guava scip_binary: scip-java + # scip-java has problems with indexing Kotlin codebase + # - repository: arrow-kt/arrow + # scip_binary: scip-java + - repository: apache/pekko + scip_binary: scip-java container: sourcegraph/${{ matrix.target.scip_binary }}:latest concurrency: @@ -32,7 +37,6 @@ jobs: uses: actions/checkout@v5 with: repository: ${{ matrix.target.repository }} - ref: master fetch-depth: 1 - name: Cache Maven dependencies