github-etl/docker-compose.yml at main · mozilla-conduit/github-etl · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
services:
  # Mock GitHub API service for testing without rate limits
  mock-github-api:
    build:
      context: .
      dockerfile: Dockerfile.mock
    ports:
      - "5000:5000"
    networks:
      - github_etl

  # BigQuery emulator for local testing
  bigquery-emulator:
    image: ghcr.io/goccy/bigquery-emulator:latest
    platform: linux/amd64
    ports:
      - "9050:9050"
      - "9060:9060"
    volumes:
      - ./data.yml:/data.yml
    command: |
      --project=test --data-from-yaml=/data.yml --log-level=debug
    networks:
      - github_etl

  # GitHub ETL service
  github-etl:
    build: .
    depends_on:
      - mock-github-api
      - bigquery-emulator
    environment:
      # GitHub Configuration
      GITHUB_REPOS: "mozilla-firefox/firefox"
      GITHUB_TOKEN: ""  # Not needed for mock API

      # Use mock GitHub API instead of real API
      GITHUB_API_URL: "http://mock-github-api:5000"

      # BigQuery Configuration
      BIGQUERY_PROJECT: "test"
      BIGQUERY_DATASET: "github_etl"

      # Point to the BigQuery emulator
      BIGQUERY_EMULATOR_HOST: "http://bigquery-emulator:9050"
    volumes:
      - ./main.py:/app/main.py
    networks:
      - github_etl

networks:
  github_etl:
    driver: bridge