Skip to content

Commit 95a8895

Browse files
henrik242henrik242
committed
Merge fra+til into one popular-stops.csv
Fetch script now hits both PostHog insights, slurps the responses, and sums aggregated_value per id. Workflow drops the fra/til matrix and uploads a single popular-stops.csv. USAGE_URL in prod/dev configs points at the merged file. Old popular-stops-fra.csv / popular-stops-til.csv in GCS are now orphaned and can be removed manually if desired.
1 parent d5ca6a6 commit 95a8895

4 files changed

Lines changed: 50 additions & 45 deletions

File tree

‎.github/scripts/fetch-posthog-popular-stops.sh‎

Lines changed: 37 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,48 @@
11
#!/usr/bin/env bash
2-
# Fetch a PostHog insight and emit a semicolon-separated `id;name;usage` CSV
3-
# (with header) on stdout. Used to feed the nominatim-converter --usage flag.
2+
# Fetch both PostHog insights (boardings + alightings) and emit a merged
3+
# semicolon-separated `id;name;usage` CSV (with header) on stdout. `usage`
4+
# is the sum across both directions per id. Used to feed the
5+
# nominatim-converter --usage flag.
46
#
5-
# Usage: ./fetch-posthog-popular-stops.sh {fra|til}
6-
# Env: TOKEN - PostHog personal API key (insight:read + query:read)
7+
# Env: TOKEN - PostHog personal API key (insight:read + query:read)
78

89
set -euo pipefail
910

1011
: "${TOKEN:?set TOKEN to your PostHog personal API key}"
1112

12-
case "${1:-}" in
13-
fra) SHORT_ID=hd0beH5A ;;
14-
til) SHORT_ID=LePQhnOg ;;
15-
*) echo "Usage: $0 {fra|til}" >&2; exit 1 ;;
16-
esac
17-
1813
PROJ=2283
1914
LIMIT=1000
2015

21-
QUERY=$(curl -sG "https://eu.posthog.com/api/projects/$PROJ/insights/" \
22-
-H "Authorization: Bearer $TOKEN" \
23-
--data-urlencode "short_id=$SHORT_ID" \
24-
| jq ".results[0].query | .source.breakdownFilter.breakdown_limit = $LIMIT")
16+
# Fetch one insight by short_id and emit its query result JSON to stdout.
17+
fetch() {
18+
local short_id=$1
19+
local query
20+
query=$(curl -sG "https://eu.posthog.com/api/projects/$PROJ/insights/" \
21+
-H "Authorization: Bearer $TOKEN" \
22+
--data-urlencode "short_id=$short_id" \
23+
| jq ".results[0].query | .source.breakdownFilter.breakdown_limit = $LIMIT")
24+
curl -s "https://eu.posthog.com/api/projects/$PROJ/query/" \
25+
-H "Authorization: Bearer $TOKEN" \
26+
-H "Content-Type: application/json" \
27+
-d "{\"query\": $query}"
28+
}
2529

26-
curl -s "https://eu.posthog.com/api/projects/$PROJ/query/" \
27-
-H "Authorization: Bearer $TOKEN" \
28-
-H "Content-Type: application/json" \
29-
-d "{\"query\": $QUERY}" \
30-
| jq -r '["id","name","usage"],
31-
(.results[]
32-
| select(.breakdown_value[0] != "$$_posthog_breakdown_null_$$")
33-
| [.breakdown_value[0], .breakdown_value[1], .aggregated_value])
34-
| map(tostring) | join(";")'
30+
# Fetch both insights, slurp into one stream, sum usage per id.
31+
{
32+
fetch hd0beH5A # fra (boardings)
33+
fetch LePQhnOg # til (alightings)
34+
} | jq -rs '
35+
[.[].results[]]
36+
| map(select(.breakdown_value[0] != "$$_posthog_breakdown_null_$$"))
37+
| map(.breakdown_value[0] |= sub("^OSM:TopographicPlace:"; "OSM:PointOfInterest:"))
38+
| group_by(.breakdown_value[0])
39+
| map({
40+
id: .[0].breakdown_value[0],
41+
name: .[0].breakdown_value[1],
42+
usage: (map(.aggregated_value) | add)
43+
})
44+
| sort_by(-.usage)
45+
| (["id","name","usage"]),
46+
(.[] | [.id, .name, .usage])
47+
| map(tostring) | join(";")
48+
'

‎.github/workflows/cache-data-sources.yml‎

Lines changed: 11 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -73,35 +73,27 @@ jobs:
7373
} >> "$GITHUB_STEP_SUMMARY"
7474
7575
posthog-popular-stops:
76-
name: Cache popular stops (${{ matrix.mode }})
76+
name: Cache popular stops
7777
runs-on: ubuntu-latest
7878
permissions:
7979
contents: read
8080
id-token: write
81-
strategy:
82-
fail-fast: false
83-
matrix:
84-
include:
85-
- mode: fra
86-
filename: popular-stops-fra.csv
87-
- mode: til
88-
filename: popular-stops-til.csv
81+
env:
82+
OUT: popular-stops.csv
8983

9084
steps:
9185
- uses: actions/checkout@v5
9286

93-
- name: Fetch ${{ matrix.mode }} from PostHog
87+
- name: Fetch from PostHog
9488
env:
9589
# Token needs insight:read and query:read scopes.
9690
TOKEN: ${{ secrets.POSTHOG_TOKEN }}
9791
run: |
98-
.github/scripts/fetch-posthog-popular-stops.sh "${{ matrix.mode }}" > "${{ matrix.filename }}"
99-
echo "Wrote $(wc -l < "${{ matrix.filename }}") lines to ${{ matrix.filename }}"
100-
head -3 "${{ matrix.filename }}"
92+
.github/scripts/fetch-posthog-popular-stops.sh > "$OUT"
93+
echo "Wrote $(wc -l < "$OUT") lines to $OUT"
94+
head -3 "$OUT"
10195
10296
- name: Verify non-empty
103-
env:
104-
OUT: ${{ matrix.filename }}
10597
run: |
10698
ROWS=$(wc -l < "$OUT")
10799
echo "Rows: $ROWS"
@@ -123,15 +115,14 @@ jobs:
123115
env:
124116
CLOUDSDK_STORAGE_PARALLEL_COMPOSITE_UPLOAD_ENABLED: "False"
125117
run: |
126-
gcloud storage cp "${{ matrix.filename }}" \
127-
"gs://ent-geocoder-prd/data-sources/${{ matrix.filename }}"
118+
gcloud storage cp "$OUT" "gs://ent-geocoder-prd/data-sources/$OUT"
128119
129120
- name: Summary
130121
run: |
131122
{
132-
echo "### popular stops (${{ matrix.mode }})"
133-
echo "**Cached:** \`gs://ent-geocoder-prd/data-sources/${{ matrix.filename }}\`"
134-
echo "**Rows:** $(wc -l < "${{ matrix.filename }}")"
123+
echo "### popular stops"
124+
echo "**Cached:** \`gs://ent-geocoder-prd/data-sources/$OUT\`"
125+
echo "**Rows:** $(wc -l < "$OUT")"
135126
} >> "$GITHUB_STEP_SUMMARY"
136127
137128
notify-slack:

‎photon/import/config/sources-dev.conf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,4 @@ STOPPLACE_URL=https://storage.googleapis.com/marduk-production/tiamat/Current_la
66
OSM_URL=https://storage.googleapis.com/ror-osmdata-prd/osm-data/norway-latest.osm.pbf
77
POI_URL=https://raw.githubusercontent.com/entur/geocoder-data/refs/heads/main/festivals_netex_poi.xml
88
POI2_URL=https://raw.githubusercontent.com/entur/geocoder-data/refs/heads/main/events_norway_poi.xml
9-
USAGE_URL=https://storage.googleapis.com/ent-geocoder-prd/data-sources/popular-stops-fra.csv
9+
USAGE_URL=https://storage.googleapis.com/ent-geocoder-prd/data-sources/popular-stops.csv

‎photon/import/config/sources-prod.conf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@ STEDSNAVN_URL=https://storage.googleapis.com/ent-geocoder-prd/data-sources/Basis
55
STOPPLACE_URL=https://storage.googleapis.com/marduk-production/tiamat/Current_latest.zip
66
OSM_URL=https://storage.googleapis.com/ror-osmdata-prd/osm-data/norway-latest.osm.pbf
77
POI_URL=https://raw.githubusercontent.com/entur/geocoder-data/refs/heads/main/festivals_netex_poi.xml
8-
USAGE_URL=https://storage.googleapis.com/ent-geocoder-prd/data-sources/popular-stops-fra.csv
8+
USAGE_URL=https://storage.googleapis.com/ent-geocoder-prd/data-sources/popular-stops.csv

0 commit comments

Comments
 (0)