added docker runner and reduced persited files

Author: ihlec

.dockerignore

@@ -0,0 +1,5 @@
+.git
+.gitignore
+tracker-data
+*.log
+docker-compose.yml

.github/workflows/build.yml

@@ -0,0 +1,49 @@
+name: Build
+
+on:
+  push:
+    branches: [main]
+    tags: ['v*']
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: write
+  packages: write
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: docker/setup-buildx-action@v4
+
+      - name: Log in to GitHub Container Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@v6
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha
+
+      - name: Build and push
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.gitignore

@@ -13,4 +13,5 @@ ipfs-archive-tracker
 *.tmp
 
 # local docs and notes
-docs
+docs
+docker-compose.yml

Dockerfile

@@ -0,0 +1,12 @@
+FROM golang:1.25-alpine AS build
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN CGO_ENABLED=0 go build -o /ipfs-archive-tracker .
+
+FROM alpine:3.21
+RUN apk add --no-cache ca-certificates poppler-utils
+COPY --from=build /ipfs-archive-tracker /usr/local/bin/ipfs-archive-tracker
+EXPOSE 8384 8385
+ENTRYPOINT ["ipfs-archive-tracker"]

README.md

@@ -11,6 +11,33 @@
 go build -o ipfs-archive-tracker .
 ```
 
+## Docker / Compose
+
+A pre-built image is published to GHCR on every push to `main` and on version tags:
+
+```sh
+docker pull ghcr.io/gipplab/ipfs-archive-tracker:main
+```
+
+Or build locally:
+
+```sh
+docker build -t ipfs-archive-tracker .
+```
+
+Run container (persist data and expose only public API):
+
+```sh
+docker run -d --name ipfs-archive-tracker \
+  -p 8385:8385 \
+  -v "$(pwd)/tracker-data:/data" \
+  -v "$(pwd)/.api_key:/data/.api_key:ro" \
+  ghcr.io/gipplab/ipfs-archive-tracker:main \
+  -o /data -public-port 8385 -port 8384
+```
+
+See `docker-compose.yml` for a full Compose example. If Kubo runs in another container/network, set `-kubo` to that service URL instead.
+
 ## API key
 
 Indexing needs an API key: `.api_key` in `-o` or cwd, or `SAIA_API_KEY`.
@@ -30,22 +57,24 @@ Two servers by default: internal (Web UI: 127.0.0.1:8384) and public (Archives A
 ./ipfs-archive-tracker -o ./index-data -port 9000 -public-port 9001
 ```
 
-## Flags
-
-| Flag | Default | Description |
-|------|---------|-------------|
-| `-o` | `.` | Output directory for index and data files |
-| `-gateway` | `https://ipfs.io` | IPFS gateway base URL |
-| `-kubo` | `http://localhost:5001` | Kubo API URL for IPNS resolution |
-| `-workers` | `4` | Number of concurrent processing workers |
-| `-model` | `meta-llama-3.1-8b-instruct` | LLM model for keyword extraction |
-| `-fallback-model` | `llama-3.3-70b-instruct` | Model to try if primary returns 429 |
-| `-api-base` | `https://chat-ai.academiccloud.de/v1` | OpenAI-compatible API base URL |
-| `-spacing` | `100ms` | Minimum delay between dispatching CIDs |
-| `-cli` | `false` | Index pending CIDs from archives and exit (no web UI) |
-| `-port` | `8384` | Web UI port (localhost only) |
-| `-public-port` | `8385` | Public API port (archives only, bind all interfaces) |
-| `-refresh` | `10m` | Interval to refresh IPNS for all archives (0 to disable) |
+## Configuration
+
+All settings can be provided as CLI flags or environment variables. Flags take precedence.
+
+| Flag | Env var | Default | Description |
+|------|---------|---------|-------------|
+| `-o` | `TRACKER_DATA_DIR` | `.` | Output directory for index files |
+| `-gateway` | `TRACKER_GATEWAY` | `https://ipfs.io` | IPFS gateway base URL |
+| `-kubo` | `KUBO_API` | `http://localhost:5001` | Kubo API URL for IPNS resolution |
+| `-workers` | `TRACKER_WORKERS` | `4` | Number of concurrent processing workers |
+| `-model` | `TRACKER_MODEL` | `meta-llama-3.1-8b-instruct` | LLM model for keyword extraction |
+| `-fallback-model` | `TRACKER_FALLBACK_MODEL` | `llama-3.3-70b-instruct` | Model to try if primary returns 429 |
+| `-api-base` | `TRACKER_API_BASE` | `https://chat-ai.academiccloud.de/v1` | OpenAI-compatible API base URL |
+| `-spacing` | `TRACKER_SPACING` | `100ms` | Minimum delay between dispatching CIDs |
+| `-cli` | `TRACKER_CLI` | `false` | Index pending CIDs from archives and exit (no web UI) |
+| `-port` | `TRACKER_PORT` | `8384` | Web UI port (localhost only) |
+| `-public-port` | `TRACKER_PUBLIC_PORT` | `8385` | Public API port (archives only, bind all interfaces) |
+| `-refresh` | `TRACKER_REFRESH` | `10m` | Interval to refresh IPNS for all archives (0 to disable) |
 
 With the default `-api-base` (Chat AI / Academic Cloud), rate limits from the API are 1000 req/min, 10000/hour, 50002/day. Current models and exact API IDs: see [docs/chat-ai-api.md](docs/chat-ai-api.md) or `GET https://chat-ai.academiccloud.de/v1/models` (with your API key).
 
@@ -74,9 +103,7 @@ Expose only port 8385 (not 8384).
 
 | File | Description |
 |------|-------------|
-| `keyword_index.json` | Indexed metadata keyed by CID |
-| `keyword_failures.json` | Permanently failed CIDs with error details |
-| `archives.json` | Archive ID → name, CIDs (source of all known CIDs) |
+| `keyword_index.json` | Indexed metadata keyed by CID (only persisted file) |
 
 Example entry in `keyword_index.json`:
 

main.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"sync"
 	"syscall"
@@ -29,20 +30,50 @@ type PipelineConfig struct {
 	KuboHTTP      *http.Client
 }
 
+func envStr(key, fallback string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return fallback
+}
+
+func envInt(key string, fallback int) int {
+	if v := os.Getenv(key); v != "" {
+		if n, err := strconv.Atoi(v); err == nil {
+			return n
+		}
+	}
+	return fallback
+}
+
+func envDuration(key string, fallback time.Duration) time.Duration {
+	if v := os.Getenv(key); v != "" {
+		if d, err := time.ParseDuration(v); err == nil {
+			return d
+		}
+	}
+	return fallback
+}
+
+func envBool(key string) bool {
+	v := strings.ToLower(os.Getenv(key))
+	return v == "1" || v == "true"
+}
+
 func main() {
 	var (
-		outputDir       = flag.String("o", ".", "output directory for index and failure files")
-		gateway         = flag.String("gateway", "https://ipfs.io", "IPFS gateway base URL")
-		kuboAPI         = flag.String("kubo", "http://localhost:5001", "Kubo API URL for IPNS resolution")
-		workers         = flag.Int("workers", 4, "number of concurrent processing workers")
-		model           = flag.String("model", defaultModel, "LLM model for keyword extraction")
-		fallbackModel   = flag.String("fallback-model", defaultFallbackModel, "fallback model if primary is rate limited")
-		apiBase         = flag.String("api-base", defaultAPIBase, "OpenAI-compatible API base URL")
-		spacing         = flag.Duration("spacing", 100*time.Millisecond, "minimum delay between dispatching CIDs to workers")
-		cli             = flag.Bool("cli", false, "run in CLI mode (index CIDs and exit, no web UI)")
-		port            = flag.Int("port", defaultPort, "web UI port (localhost only)")
-		publicPort      = flag.Int("public-port", defaultPublicPort, "public API port (archives only, bind all interfaces)")
-		refreshInterval = flag.Duration("refresh", 10*time.Minute, "interval to refresh IPNS for all archives (0 to disable)")
+		outputDir       = flag.String("o", envStr("TRACKER_DATA_DIR", "."), "output directory for index files")
+		gateway         = flag.String("gateway", envStr("TRACKER_GATEWAY", "https://ipfs.io"), "IPFS gateway base URL")
+		kuboAPI         = flag.String("kubo", envStr("KUBO_API", "http://localhost:5001"), "Kubo API URL for IPNS resolution")
+		workers         = flag.Int("workers", envInt("TRACKER_WORKERS", 4), "number of concurrent processing workers")
+		model           = flag.String("model", envStr("TRACKER_MODEL", defaultModel), "LLM model for keyword extraction")
+		fallbackModel   = flag.String("fallback-model", envStr("TRACKER_FALLBACK_MODEL", defaultFallbackModel), "fallback model if primary is rate limited")
+		apiBase         = flag.String("api-base", envStr("TRACKER_API_BASE", defaultAPIBase), "OpenAI-compatible API base URL")
+		spacing         = flag.Duration("spacing", envDuration("TRACKER_SPACING", 100*time.Millisecond), "minimum delay between dispatching CIDs to workers")
+		cli             = flag.Bool("cli", envBool("TRACKER_CLI"), "run in CLI mode (index CIDs and exit, no web UI)")
+		port            = flag.Int("port", envInt("TRACKER_PORT", defaultPort), "web UI port (localhost only)")
+		publicPort      = flag.Int("public-port", envInt("TRACKER_PUBLIC_PORT", defaultPublicPort), "public API port (archives only, bind all interfaces)")
+		refreshInterval = flag.Duration("refresh", envDuration("TRACKER_REFRESH", 10*time.Minute), "interval to refresh IPNS for all archives (0 to disable)")
 	)
 	flag.Usage = func() {
 		fmt.Fprintf(os.Stderr, "Usage: %s [flags]\n\n", os.Args[0])
@@ -53,6 +84,8 @@ func main() {
 		fmt.Fprintf(os.Stderr, "Use -cli to index once from archives and exit (no web server).\n\n")
 		fmt.Fprintf(os.Stderr, "API key (required for indexing):\n")
 		fmt.Fprintf(os.Stderr, "  Read from .api_key file (in -o dir, then cwd), or SAIA_API_KEY env var.\n\n")
+		fmt.Fprintf(os.Stderr, "All flags can also be set via environment variables (TRACKER_*).\n")
+		fmt.Fprintf(os.Stderr, "Flags take precedence over environment variables.\n\n")
 		fmt.Fprintf(os.Stderr, "Flags:\n")
 		flag.PrintDefaults()
 	}