Update Data #11
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update Data | |
| on: | |
| schedule: | |
| # Run daily at 6:00 AM UTC | |
| - cron: '0 6 * * *' | |
| workflow_dispatch: | |
| # Allow manual triggering | |
| env: | |
| CARGO_TERM_COLOR: always | |
| jobs: | |
| update: | |
| name: Fetch and Publish Data | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download CLI from latest release | |
| id: download | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| # Try to download the latest CLI release | |
| mkdir -p /tmp/which-llm-bin | |
| if gh release download --repo ${{ github.repository }} \ | |
| --pattern 'which-llm-x86_64-unknown-linux-gnu.tar.gz' \ | |
| --dir /tmp/which-llm-bin; then | |
| echo "Downloaded CLI from release" | |
| tar -xzf /tmp/which-llm-bin/which-llm-*.tar.gz -C /tmp/which-llm-bin --no-same-owner | |
| chmod +x /tmp/which-llm-bin/which-llm | |
| echo "cli_path=/tmp/which-llm-bin/which-llm" >> $GITHUB_OUTPUT | |
| echo "need_build=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "No release found, will build from source" | |
| echo "need_build=true" >> $GITHUB_OUTPUT | |
| fi | |
| # Fallback: Build from source if no release exists | |
| - name: Setup Rust (fallback) | |
| if: steps.download.outputs.need_build == 'true' | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Cache Cargo (fallback) | |
| if: steps.download.outputs.need_build == 'true' | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: "ubuntu-latest" | |
| - name: Build CLI (fallback) | |
| if: steps.download.outputs.need_build == 'true' | |
| run: | | |
| cargo build --release | |
| echo "cli_path=./target/release/which-llm" >> $GITHUB_OUTPUT | |
| - name: Fetch data from APIs | |
| env: | |
| ARTIFICIAL_ANALYSIS_API_KEY: ${{ secrets.ARTIFICIAL_ANALYSIS_API_KEY }} | |
| WHICH_LLM_CACHE_DIR: ${{ runner.temp }}/which-llm-data | |
| run: | | |
| set -euo pipefail | |
| mkdir -p "$WHICH_LLM_CACHE_DIR" | |
| # Determine CLI path | |
| if [ "${{ steps.download.outputs.need_build }}" == "true" ]; then | |
| CLI="./target/release/which-llm" | |
| else | |
| CLI="/tmp/which-llm-bin/which-llm" | |
| fi | |
| # Fetch all data using the refresh command with API access | |
| $CLI refresh --use-api --quiet | |
| # Verify at least benchmarks.parquet exists | |
| echo "Generated parquet files:" | |
| ls -la "$WHICH_LLM_CACHE_DIR"/*.parquet || true | |
| if [ ! -f "$WHICH_LLM_CACHE_DIR/benchmarks.parquet" ]; then | |
| echo "Error: benchmarks.parquet not found" | |
| exit 1 | |
| fi | |
| - name: Generate manifest | |
| env: | |
| WHICH_LLM_CACHE_DIR: ${{ runner.temp }}/which-llm-data | |
| run: | | |
| set -euo pipefail | |
| cd "$WHICH_LLM_CACHE_DIR" | |
| # Verify parquet files exist before generating manifest | |
| if ! ls *.parquet >/dev/null 2>&1; then | |
| echo "Error: No parquet files found" | |
| exit 1 | |
| fi | |
| # Create manifest with timestamps and file info | |
| cat > manifest.json << EOF | |
| { | |
| "generated_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)", | |
| "version": "1", | |
| "source": { | |
| "artificial_analysis": "https://artificialanalysis.ai", | |
| "models_dev": "https://models.dev" | |
| }, | |
| "files": { | |
| EOF | |
| # Add file info for each parquet file | |
| first=true | |
| for f in *.parquet; do | |
| if [ -f "$f" ]; then | |
| size=$(stat -c%s "$f" 2>/dev/null || stat -f%z "$f") | |
| checksum=$(sha256sum "$f" | cut -d' ' -f1) | |
| if [ "$first" = true ]; then | |
| first=false | |
| else | |
| echo "," >> manifest.json | |
| fi | |
| printf ' "%s": {"size": %s, "sha256": "%s"}' "$f" "$size" "$checksum" >> manifest.json | |
| fi | |
| done | |
| cat >> manifest.json << EOF | |
| }, | |
| "attribution": { | |
| "text": "Data provided by Artificial Analysis (https://artificialanalysis.ai) and models.dev (https://models.dev)", | |
| "url": "https://artificialanalysis.ai" | |
| } | |
| } | |
| EOF | |
| echo "Generated manifest:" | |
| cat manifest.json | |
| - name: Create or update release | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| WHICH_LLM_CACHE_DIR: ${{ runner.temp }}/which-llm-data | |
| run: | | |
| cd $WHICH_LLM_CACHE_DIR | |
| # Check if data/latest release exists | |
| if gh release view data/latest --repo ${{ github.repository }} > /dev/null 2>&1; then | |
| echo "Updating existing release..." | |
| # Delete existing assets | |
| for asset in $(gh release view data/latest --repo ${{ github.repository }} --json assets -q '.assets[].name'); do | |
| gh release delete-asset data/latest "$asset" --repo ${{ github.repository }} --yes || true | |
| done | |
| # Upload new assets | |
| gh release upload data/latest *.parquet manifest.json --repo ${{ github.repository }} --clobber | |
| # Update release notes | |
| gh release edit data/latest \ | |
| --repo ${{ github.repository }} \ | |
| --title "Latest Data" \ | |
| --notes "Pre-built benchmark data for which-llm CLI. | |
| **Last updated:** $(date -u +%Y-%m-%dT%H:%M:%SZ) | |
| This release is automatically updated daily with fresh data from: | |
| - [Artificial Analysis](https://artificialanalysis.ai) - Benchmarks, pricing, performance | |
| - [models.dev](https://models.dev) - Capabilities, limits, modalities | |
| ## Usage | |
| The which-llm CLI automatically fetches this data. No API key required! | |
| \`\`\`bash | |
| which-llm refresh | |
| which-llm query \"SELECT * FROM benchmarks LIMIT 10\" | |
| \`\`\` | |
| ## Attribution | |
| Data provided by [Artificial Analysis](https://artificialanalysis.ai). | |
| Capability data from [models.dev](https://models.dev)." | |
| else | |
| echo "Creating new release..." | |
| gh release create data/latest \ | |
| --repo ${{ github.repository }} \ | |
| --title "Latest Data" \ | |
| --notes "Pre-built benchmark data for which-llm CLI. | |
| **Last updated:** $(date -u +%Y-%m-%dT%H:%M:%SZ) | |
| This release is automatically updated daily with fresh data from: | |
| - [Artificial Analysis](https://artificialanalysis.ai) - Benchmarks, pricing, performance | |
| - [models.dev](https://models.dev) - Capabilities, limits, modalities | |
| ## Usage | |
| The which-llm CLI automatically fetches this data. No API key required! | |
| \`\`\`bash | |
| which-llm refresh | |
| which-llm query \"SELECT * FROM benchmarks LIMIT 10\" | |
| \`\`\` | |
| ## Attribution | |
| Data provided by [Artificial Analysis](https://artificialanalysis.ai). | |
| Capability data from [models.dev](https://models.dev)." \ | |
| --latest=false \ | |
| *.parquet manifest.json | |
| fi | |
| - name: Create dated release (for history) | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| WHICH_LLM_CACHE_DIR: ${{ runner.temp }}/which-llm-data | |
| run: | | |
| cd $WHICH_LLM_CACHE_DIR | |
| DATE_TAG="data/$(date -u +%Y-%m-%d)" | |
| # Only create if it doesn't exist (avoid duplicates on re-runs) | |
| if ! gh release view "$DATE_TAG" --repo ${{ github.repository }} > /dev/null 2>&1; then | |
| echo "Creating dated release: $DATE_TAG" | |
| gh release create "$DATE_TAG" \ | |
| --repo ${{ github.repository }} \ | |
| --title "Data $(date -u +%Y-%m-%d)" \ | |
| --notes "Historical snapshot of benchmark data. | |
| Generated: $(date -u +%Y-%m-%dT%H:%M:%SZ) | |
| Data provided by [Artificial Analysis](https://artificialanalysis.ai). | |
| Capability data from [models.dev](https://models.dev)." \ | |
| --latest=false \ | |
| *.parquet manifest.json | |
| else | |
| echo "Dated release $DATE_TAG already exists, skipping" | |
| fi |