imports.yml

name: PrincetonCourses Imports

on:
  workflow_dispatch:
    inputs:
      run_import_courses:
        description: "Run course import (OIT + registrar details)"
        type: boolean
        default: true
      run_import_departments:
        description: "Run departments import (OIT)"
        type: boolean
        default: true
      run_import_evals:
        description: "Run evaluations scraper (requires PHPSESSID)"
        type: boolean
        default: true
      run_backfill:
        description: "Run backfill for missing Quality of Course"
        type: boolean
        default: false
      run_setnew:
        description: "Run setNewCourseFlag"
        type: boolean
        default: false
      restart_heroku:
        description: "Restart Heroku dynos at end (refresh dept cache)"
        type: boolean
        default: false
      term:
        description: "Target term code (e.g., 1252)"
        type: string
        required: false
      subject:
        description: "Target subject code (e.g., COS). Leave blank for all"
        type: string
        required: false
      php_sessid:
        description: "Registrar PHPSESSID cookie (required for evals)"
        type: string
        required: false
      registrar_fe_api_token:
        description: "Registrar FE API token (paste per run; overrides scrape)"
        type: string
        required: false

concurrency:
  group: imports-${{ github.ref }}
  cancel-in-progress: false

jobs:
  import_courses:
    if: ${{ inputs.run_import_courses }}
    runs-on: ubuntu-latest
    timeout-minutes: 180
    defaults:
      run:
        shell: bash
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup Node
        uses: actions/setup-node@v4
        with:
          node-version: 18
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Install Python deps
        run: |
          python -m pip install --upgrade pip
          python -m pip install -r requirements.txt
      - name: Install Node deps
        run: npm ci
      - name: Load Heroku config vars
        env:
          HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
          HEROKU_APP_NAME: ${{ secrets.HEROKU_APP_NAME }}
        run: |
          echo "::add-mask::${HEROKU_API_KEY}"
          cfg=$(curl -sS \
            -H "Authorization: Bearer ${HEROKU_API_KEY}" \
            -H "Accept: application/vnd.heroku+json; version=3" \
            https://api.heroku.com/apps/${HEROKU_APP_NAME}/config-vars)
          echo "$cfg" > heroku_config.json
          for key in MONGODB_URI CONSUMER_KEY CONSUMER_SECRET CHATBOT_API_KEY HOST PORT; do
            val=$(node -e "const o=require('./heroku_config.json'); const k='${key}'; if (o[k]) console.log(o[k]);")
            if [ -n "$val" ]; then
              echo "::add-mask::${val}"
              echo "$key=$val" >> $GITHUB_ENV
            fi
          done
      - name: Build query arg
        id: q
        run: |
          q=""
          if [ -n "${{ inputs.term }}" ] && [ -n "${{ inputs.subject }}" ]; then
            q="term=${{ inputs.term }}&subject=${{ inputs.subject }}"
          elif [ -n "${{ inputs.term }}" ]; then
            q="term=${{ inputs.term }}&subject=all"
          fi
          echo "query=${q}" >> $GITHUB_OUTPUT
      - name: Run course importer
        env:
          REGISTRAR_FE_API_TOKEN: ${{ inputs.registrar_fe_api_token }}
        run: |
          if [ -n "${{ inputs.registrar_fe_api_token }}" ]; then echo "::add-mask::${{ inputs.registrar_fe_api_token }}"; fi
          if [ -n "${{ steps.q.outputs.query }}" ]; then
            node importers/importBasicCourseDetails.js "${{ steps.q.outputs.query }}"
          else
            node importers/importBasicCourseDetails.js
          fi

  import_departments:
    if: ${{ inputs.run_import_departments }}
    needs: [import_courses]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    defaults:
      run:
        shell: bash
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup Node
        uses: actions/setup-node@v4
        with:
          node-version: 18
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Install Python deps
        run: |
          python -m pip install --upgrade pip
          python -m pip install -r requirements.txt
      - name: Install Node deps
        run: npm ci
      - name: Load Heroku config vars
        env:
          HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
          HEROKU_APP_NAME: ${{ secrets.HEROKU_APP_NAME }}
        run: |
          echo "::add-mask::${HEROKU_API_KEY}"
          cfg=$(curl -sS \
            -H "Authorization: Bearer ${HEROKU_API_KEY}" \
            -H "Accept: application/vnd.heroku+json; version=3" \
            https://api.heroku.com/apps/${HEROKU_APP_NAME}/config-vars)
          echo "$cfg" > heroku_config.json
          for key in MONGODB_URI CONSUMER_KEY CONSUMER_SECRET; do
            val=$(node -e "const o=require('./heroku_config.json'); const k='${key}'; if (o[k]) console.log(o[k]);")
            if [ -n "$val" ]; then
              echo "::add-mask::${val}"
              echo "$key=$val" >> $GITHUB_ENV
            fi
          done
      - name: Run departments importer
        run: node importers/importDepartments.js

  scrape_evals:
    if: ${{ inputs.run_import_evals }}
    needs: [import_courses]
    runs-on: ubuntu-latest
    timeout-minutes: 240
    defaults:
      run:
        shell: bash
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup Node
        uses: actions/setup-node@v4
        with:
          node-version: 18
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Install Python deps
        run: |
          python -m pip install --upgrade pip
          python -m pip install -r requirements.txt
      - name: Install Node deps
        run: npm ci
      - name: Load Heroku config vars
        env:
          HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
          HEROKU_APP_NAME: ${{ secrets.HEROKU_APP_NAME }}
        run: |
          echo "::add-mask::${HEROKU_API_KEY}"
          cfg=$(curl -sS \
            -H "Authorization: Bearer ${HEROKU_API_KEY}" \
            -H "Accept: application/vnd.heroku+json; version=3" \
            https://api.heroku.com/apps/${HEROKU_APP_NAME}/config-vars)
          echo "$cfg" > heroku_config.json
          for key in MONGODB_URI; do
            val=$(node -e "const o=require('./heroku_config.json'); const k='${key}'; if (o[k]) console.log(o[k]);")
            if [ -n "$val" ]; then
              echo "::add-mask::${val}"
              echo "$key=$val" >> $GITHUB_ENV
            fi
          done
      - name: Run evaluation scraper (non-interactive)
        env:
          EVAL_SCRAPE_DELAY_MS: "600"
          EVAL_SCRAPE_MAX_RETRIES: "3"
          EVAL_SCRAPE_RETRY_BACKOFF_MS: "1000"
          EVAL_RANDOMIZE_ORDER: "true"
          INPUT_TERM: ${{ inputs.term }}
          INPUT_SUBJECT: ${{ inputs.subject }}
          PHPSESSID: ${{ inputs.php_sessid }}
        run: |
          if [ -z "${PHPSESSID}" ]; then
            echo "php_sessid input is required to run evaluations" >&2
            exit 1
          fi
          echo "::add-mask::${PHPSESSID}"
          export EVAL_QUERY=$(node -e '
            const term = process.env.INPUT_TERM;
            const subj = process.env.INPUT_SUBJECT;
            const q = { "scores.Quality of Course": { "$exists": false } };
            if (term) q.semester = Number(term);
            if (subj) q.department = String(subj).toUpperCase();
            console.log(JSON.stringify(q));
          ')
          node importers/scrapeEvaluations.js --skip

  backfill_scores:
    if: ${{ inputs.run_backfill }}
    needs: [scrape_evals]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    defaults:
      run:
        shell: bash
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup Node
        uses: actions/setup-node@v4
        with:
          node-version: 18
      - name: Install Node deps
        run: npm ci
      - name: Load Heroku config vars
        env:
          HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
          HEROKU_APP_NAME: ${{ secrets.HEROKU_APP_NAME }}
        run: |
          echo "::add-mask::${HEROKU_API_KEY}"
          cfg=$(curl -sS \
            -H "Authorization: Bearer ${HEROKU_API_KEY}" \
            -H "Accept: application/vnd.heroku+json; version=3" \
            https://api.heroku.com/apps/${HEROKU_APP_NAME}/config-vars)
          echo "$cfg" > heroku_config.json
          val=$(node -e "const o=require('./heroku_config.json'); const k='MONGODB_URI'; if (o[k]) console.log(o[k]);")
          if [ -n "$val" ]; then
            echo "::add-mask::${val}"
            echo "MONGODB_URI=$val" >> $GITHUB_ENV
          fi
      - name: Run backfill
        run: |
          if [ -n "${{ inputs.term }}" ]; then
            node importers/insertMostRecentScoreIntoUnevaluatedSemesters.js "${{ inputs.term }}"
          else
            node importers/insertMostRecentScoreIntoUnevaluatedSemesters.js
          fi

  set_new_flag:
    if: ${{ inputs.run_setnew }}
    needs: [scrape_evals]
    runs-on: ubuntu-latest
    timeout-minutes: 30
    defaults:
      run:
        shell: bash
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup Node
        uses: actions/setup-node@v4
        with:
          node-version: 18
      - name: Install Node deps
        run: npm ci
      - name: Load Heroku config vars
        env:
          HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
          HEROKU_APP_NAME: ${{ secrets.HEROKU_APP_NAME }}
        run: |
          echo "::add-mask::${HEROKU_API_KEY}"
          cfg=$(curl -sS \
            -H "Authorization: Bearer ${HEROKU_API_KEY}" \
            -H "Accept: application/vnd.heroku+json; version=3" \
            https://api.heroku.com/apps/${HEROKU_APP_NAME}/config-vars)
          echo "$cfg" > heroku_config.json
          val=$(node -e "const o=require('./heroku_config.json'); const k='MONGODB_URI'; if (o[k]) console.log(o[k]);")
          if [ -n "$val" ]; then
            echo "::add-mask::${val}"
            echo "MONGODB_URI=$val" >> $GITHUB_ENV
          fi
      - name: Run setNewCourseFlag
        run: node importers/setNewCourseFlag.js

  restart_heroku:
    if: ${{ inputs.restart_heroku && always() }}
    needs: [import_departments]
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Restart Heroku dynos
        env:
          HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
          HEROKU_APP_NAME: ${{ secrets.HEROKU_APP_NAME }}
        run: |
          echo "::add-mask::${HEROKU_API_KEY}"
          curl -sS -X DELETE \
            -H "Accept: application/vnd.heroku+json; version=3" \
            -H "Authorization: Bearer ${HEROKU_API_KEY}" \
            "https://api.heroku.com/apps/${HEROKU_APP_NAME}/dynos"