Skip to content

Commit fe95f48

Browse files
committed
Merge branch 'main' into feat/pt-126
2 parents 30f5dcd + 15f05d3 commit fe95f48

File tree

17 files changed

+465
-52
lines changed

17 files changed

+465
-52
lines changed

.github/workflows/web-app-deployer.yml

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ on:
3838
description: 1Password GitHub token secret reference
3939
type: string
4040
required: false
41+
OP_SENTRY_DSN:
42+
description: Reference to the 1Password Sentry DSN secret
43+
type: string
44+
required: false
4145
ENABLE_QUALITY_CHECKS:
4246
description: When true the tests and lint checks are performed(default), false will skip. This is only valid in protopype and demo deployments
4347
type: boolean
@@ -172,6 +176,7 @@ jobs:
172176
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
173177
SLACK_WEBHOOK_URL: ${{inputs.OP_SLACK_WEBHOOK_URL}}
174178
ADD_FEED_FORM_GITHUB_TOKEN: ${{inputs.OP_ADD_FEED_FORM_GITHUB_TOKEN}}
179+
SENTRY_DSN: ${{inputs.OP_SENTRY_DSN}}
175180

176181
- name: Authenticate to Google Cloud DEV
177182
if: ${{ inputs.FIREBASE_PROJECT == 'dev' }}
@@ -235,6 +240,10 @@ jobs:
235240
echo "REACT_APP_REMOTE_CONFIG_MINIMUM_FETCH_INTERVAL_MILLI=3600000" >> $GITHUB_ENV
236241
echo "REACT_APP_FEED_API_BASE_URL=https://api.mobilitydatabase.org" >> $GITHUB_ENV
237242
echo "REACT_APP_GBFS_VALIDATOR_API_BASE_URL=https://dev.gbfs.api.mobilitydatabase.org" >> $GITHUB_ENV
243+
echo "REACT_APP_SENTRY_DSN=${{ env.SENTRY_DSN }}" >> $GITHUB_ENV
244+
echo "REACT_APP_SENTRY_REPLAY_SESSION_SAMPLE_RATE=0.1" >> $GITHUB_ENV
245+
echo "REACT_APP_SENTRY_REPLAY_ERROR_SAMPLE_RATE=0.1" >> $GITHUB_ENV
246+
echo "REACT_APP_SENTRY_TRACES_SAMPLE_RATE=0.05" >> $GITHUB_ENV
238247
else
239248
echo "Setting FIREBASE_PROJECT to 'dev'"
240249
echo "FIREBASE_PROJECT=dev" >> $GITHUB_ENV
@@ -253,7 +262,7 @@ jobs:
253262
- name: Populate Variables
254263
working-directory: web-app
255264
run: |
256-
../scripts/replace-variables.sh -in_file src/.env.rename_me -out_file src/.env.${{ inputs.FIREBASE_PROJECT }} -variables REACT_APP_FIREBASE_API_KEY,REACT_APP_FIREBASE_AUTH_DOMAIN,REACT_APP_FIREBASE_PROJECT_ID,REACT_APP_FIREBASE_STORAGE_BUCKET,REACT_APP_FIREBASE_MESSAGING_SENDER_ID,REACT_APP_FIREBASE_APP_ID,REACT_APP_RECAPTCHA_SITE_KEY,REACT_APP_GOOGLE_ANALYTICS_ID,REACT_APP_REMOTE_CONFIG_MINIMUM_FETCH_INTERVAL_MILLI,REACT_APP_FEED_API_BASE_URL,REACT_APP_GBFS_VALIDATOR_API_BASE_URL
265+
../scripts/replace-variables.sh -in_file src/.env.rename_me -out_file src/.env.${{ inputs.FIREBASE_PROJECT }} -variables REACT_APP_FIREBASE_API_KEY,REACT_APP_FIREBASE_AUTH_DOMAIN,REACT_APP_FIREBASE_PROJECT_ID,REACT_APP_FIREBASE_STORAGE_BUCKET,REACT_APP_FIREBASE_MESSAGING_SENDER_ID,REACT_APP_FIREBASE_APP_ID,REACT_APP_RECAPTCHA_SITE_KEY,REACT_APP_GOOGLE_ANALYTICS_ID,REACT_APP_REMOTE_CONFIG_MINIMUM_FETCH_INTERVAL_MILLI,REACT_APP_FEED_API_BASE_URL,REACT_APP_GBFS_VALIDATOR_API_BASE_URL -optional_variables REACT_APP_SENTRY_DSN,REACT_APP_SENTRY_TRACES_SAMPLE_RATE,REACT_APP_SENTRY_REPLAY_SESSION_SAMPLE_RATE,REACT_APP_SENTRY_REPLAY_ERROR_SAMPLE_RATE
257266
258267
- name: Run Install for Functions
259268
if: ${{ inputs.DEPLOY_FIREBASE_FUNCTIONS }}

.github/workflows/web-prod.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,5 @@ jobs:
1313
FEED_SUBMIT_GOOGLE_SHEET_ID: "10eIUxWVtLmc2EATiwivgXBf4bOMErOnq7GFIoRedXHU"
1414
OP_SLACK_WEBHOOK_URL: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/Slack webhook URLs/rdpfgrmnbxqaelgi5oky3lryz4/internal-add-feeds"
1515
OP_ADD_FEED_FORM_GITHUB_TOKEN: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/cwzlqlspbw7goqjsdqu4b7matq/credential"
16+
OP_SENTRY_DSN: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/Sentry DSN - MobilityDatabase PROD/Sentry DSN PROD"
1617
secrets: inherit

functions-python/reverse_geolocation/src/reverse_geolocation_batch.py

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,16 @@ def get_feeds_data(
6565
return [d for d in data if d["stops_url"]]
6666

6767

68-
def parse_request_parameters(request: flask.Request) -> Tuple[List[str], bool]:
69-
"""Parse the request parameters to get the country codes and whether to include only unprocessed feeds."""
68+
def parse_request_parameters(request: flask.Request) -> Tuple[List[str], bool, bool]:
69+
"""
70+
Parse the request parameters.
71+
72+
Returns:
73+
Tuple[List[str], bool, bool]: A tuple containing:
74+
- country_codes: List of country codes to filter feeds
75+
- include_only_unprocessed: Whether to include only unprocessed feeds
76+
- use_cache: Whether to use cache for reverse geolocation
77+
"""
7078
json_request = request.get_json()
7179
country_codes = json_request.get("country_codes", "").split(",")
7280
country_codes = [code.strip().upper() for code in country_codes if code]
@@ -78,13 +86,16 @@ def parse_request_parameters(request: flask.Request) -> Tuple[List[str], bool]:
7886
include_only_unprocessed = (
7987
json_request.get("include_only_unprocessed", True) is True
8088
)
81-
return country_codes, include_only_unprocessed
89+
use_cache = bool(json_request.get("use_cache", True))
90+
return country_codes, include_only_unprocessed, use_cache
8291

8392

8493
def reverse_geolocation_batch(request: flask.Request) -> Tuple[str, int]:
8594
"""Batch function to trigger reverse geolocation for feeds."""
8695
try:
87-
country_codes, include_only_unprocessed = parse_request_parameters(request)
96+
country_codes, include_only_unprocessed, use_cache = parse_request_parameters(
97+
request
98+
)
8899
feeds_data = get_feeds_data(country_codes, include_only_unprocessed)
89100
logging.info("Valid feeds with latest dataset: %s", len(feeds_data))
90101

@@ -93,6 +104,7 @@ def reverse_geolocation_batch(request: flask.Request) -> Tuple[str, int]:
93104
stable_id=feed["stable_id"],
94105
dataset_id=feed["dataset_id"],
95106
stops_url=feed["stops_url"],
107+
use_cache=use_cache,
96108
)
97109
return f"Batch function triggered for {len(feeds_data)} feeds.", 200
98110
except Exception as e:
@@ -104,13 +116,19 @@ def create_http_processor_task(
104116
stable_id: str,
105117
dataset_id: str,
106118
stops_url: str,
119+
use_cache: bool = True,
107120
) -> None:
108121
"""
109122
Create a task to process a group of points.
110123
"""
111124
client = tasks_v2.CloudTasksClient()
112125
body = json.dumps(
113-
{"stable_id": stable_id, "stops_url": stops_url, "dataset_id": dataset_id}
126+
{
127+
"stable_id": stable_id,
128+
"stops_url": stops_url,
129+
"dataset_id": dataset_id,
130+
"use_cache": use_cache,
131+
}
114132
).encode()
115133
queue_name = os.getenv("QUEUE_NAME")
116134
project_id = os.getenv("PROJECT_ID")

functions-python/reverse_geolocation/tests/test_reverse_geolocation_batch.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,9 +104,12 @@ def test_parse_request_parameters(self):
104104
}.get(value, default)
105105
from reverse_geolocation_batch import parse_request_parameters
106106

107-
country_codes, include_only_unprocessed = parse_request_parameters(request)
107+
country_codes, include_only_unprocessed, use_cache = parse_request_parameters(
108+
request
109+
)
108110
self.assertEqual(["CA", "US"], country_codes)
109111
self.assertTrue(include_only_unprocessed)
112+
self.assertTrue(use_cache)
110113

111114
with pytest.raises(ValueError):
112115
request.get_json.return_value.get = lambda value, default: {
@@ -121,7 +124,7 @@ def test_reverse_geolocation_batch(self, mock_parse_request, mock_get_feeds, _):
121124
from reverse_geolocation_batch import reverse_geolocation_batch
122125

123126
request = MagicMock()
124-
mock_parse_request.return_value = ["CA", "US"]
127+
mock_parse_request.return_value = (["CA", "US"], False, False)
125128
mock_get_feeds.return_value = [
126129
{
127130
"stable_id": "test_feed",
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
22
"JP": [7],
33
"CA": [6, 8],
4-
"FR": []
4+
"FR": [8],
5+
"US": [5, 8]
56
}

functions-python/tasks_executor/src/tasks/data_import/transitfeeds/sync_transitfeeds.py

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -92,14 +92,30 @@ def _process_feeds(
9292
feed_stable_id,
9393
)
9494

95-
# Init-on-create (shared fields)
96-
if is_new:
97-
feed.name = row["Feed Name"]
98-
feed.externalids = [
95+
# Set transitfeeds Externalid
96+
existing_externalid = [
97+
eid
98+
for eid in feed.externalids
99+
if eid.source == "transitfeeds"
100+
and eid.associated_id == row["External Feed ID"]
101+
]
102+
if existing_externalid:
103+
logger.debug(
104+
"[%s] External ID for source 'transitfeeds' already set for %s: %s",
105+
feed_kind.upper(),
106+
feed_stable_id,
107+
existing_externalid[0].associated_id,
108+
)
109+
else:
110+
feed.externalids.append(
99111
Externalid(
100112
source="transitfeeds", associated_id=row["External Feed ID"]
101113
)
102-
]
114+
)
115+
feed.operational_status = "published"
116+
# Init-on-create (shared fields)
117+
if is_new:
118+
feed.name = row["Feed Name"]
103119
feed.provider = row["Provider"]
104120
feed.producer_url = row["Producer URL"]
105121
logger.debug(

functions/packages/feed-form/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
"main": "lib/index.js",
1919
"dependencies": {
2020
"axios": "^1.7.7",
21+
"countries-list": "^3.2.0",
2122
"firebase": "^10.6.0",
2223
"firebase-admin": "^11.8.0",
2324
"firebase-functions": "^4.3.1",

functions/packages/feed-form/src/impl/feed-form-impl.ts

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import * as logger from "firebase-functions/logger";
44
import {type FeedSubmissionFormRequestBody} from "./types";
55
import {type CallableRequest, HttpsError} from "firebase-functions/v2/https";
66
import axios from "axios";
7+
import {countries, continents, type TCountryCode} from "countries-list";
78

89
const SCOPES = [
910
"https://www.googleapis.com/auth/spreadsheets",
@@ -321,13 +322,21 @@ async function createGithubIssue(
321322
issueTitle += " - Official Feed";
322323
}
323324
const issueBody = buildGithubIssueBody(formData, spreadsheetId);
325+
326+
const labels = ["feed submission"];
327+
if (formData.country && formData.country in countries) {
328+
const country = countries[formData.country as TCountryCode];
329+
const continent = continents[country.continent].toLowerCase();
330+
if (continent != null) labels.push(continent);
331+
}
332+
324333
try {
325334
const response = await axios.post(
326335
githubRepoUrlIssue,
327336
{
328337
title: issueTitle,
329338
body: issueBody,
330-
labels: ["feed submission"],
339+
labels,
331340
},
332341
{
333342
headers: {

liquibase/changes/feat_1249-3.sql

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
-- Set operational_status to published for tfs feeds
2+
UPDATE feed
3+
SET operational_status = 'published'
4+
WHERE stable_id like 'tfs-%';

scripts/replace-variables.sh

Lines changed: 46 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,19 @@
2323
# For an example of a valid input file, check `../infra/vars.tfvars.rename_me`.
2424
# All variables need to be set to the environment previous running the script.
2525
# Parameters:
26-
# -variables <VARIABLES_LIST> Comma separated list of variables names.
27-
# -in_file <INPUT_FULL_PATH_NAME> Full path and file name of the input file.
28-
# -out_file <OUTPUT_FULL_PATH_NAME> Full path and file name of the output file.
29-
# -no_quotes Option to disable enclosing variable in double quotes during substitution
26+
# -variables <VARIABLES_LIST> Comma separated list of REQUIRED variable names.
27+
# -optional_variables <OPTIONAL_LIST> Comma separated list of OPTIONAL variable names (may be unset or empty).
28+
# -in_file <INPUT_FULL_PATH_NAME> Full path and file name of the input file.
29+
# -out_file <OUTPUT_FULL_PATH_NAME> Full path and file name of the output file.
30+
# -no_quotes Option to disable enclosing variable in double quotes during substitution
3031

3132
display_usage() {
3233
printf "\nThis script replaces variables from an input file creating/overriding the content of on an output file"
3334
printf "\nScript Usage:\n"
3435
echo "Usage: $0 [options]"
3536
echo "Options:"
36-
echo " -variables <VARIABLES_LIST> Comma separated list of variables names."
37+
echo " -variables <VARIABLES_LIST> Comma separated list of REQUIRED variable names."
38+
echo " -optional_variables <OPTIONAL_LIST> Comma separated list of OPTIONAL variable names."
3739
echo " -in_file <INPUT_FULL_PATH_NAME> Full path and file name of the input file."
3840
echo " -out_file <OUTPUT_FULL_PATH_NAME> Full path and file name of the output file."
3941
echo " -no_quotes Do not enclose variable values with quotes."
@@ -42,6 +44,7 @@ display_usage() {
4244
}
4345

4446
VARIABLES=""
47+
OPTIONAL_VARIABLES=""
4548
INPUT_FILE=""
4649
OUT_FILE=""
4750
ADD_QUOTES="true"
@@ -55,6 +58,11 @@ while [[ $# -gt 0 ]]; do
5558
shift # past argument
5659
shift # past value
5760
;;
61+
-optional_variables)
62+
OPTIONAL_VARIABLES="$2"
63+
shift # past argument
64+
shift # past value
65+
;;
5866
-in_file)
5967
IN_FILE="$2"
6068
shift # past argument
@@ -96,30 +104,46 @@ then
96104
fi
97105

98106
list=$(echo "$VARIABLES" | tr "," "\n")
107+
optional_list=$(echo "$OPTIONAL_VARIABLES" | tr "," "\n")
99108

100-
# Check if all variables are set
101-
for varname in $list
102-
do
103-
if [[ -z "${!varname}" ]]; then
104-
echo "Missing required variable value with name: $varname."
105-
echo "Script will not execute variables replacement, bye for now."
106-
exit 1
107-
fi
109+
# Check required variables (optional ones may be unset or empty)
110+
for varname in $list; do
111+
if [[ -z "${!varname+x}" ]]; then
112+
echo "Missing required variable (unset) with name: $varname."
113+
echo "Script will not execute variables replacement, bye for now."
114+
exit 1
115+
fi
116+
if [[ -z "${!varname}" ]]; then
117+
echo "Missing required variable (empty value) with name: $varname."
118+
echo "Script will not execute variables replacement, bye for now."
119+
exit 1
120+
fi
108121
done
109122

110123
# Reads from input setting the first version of the output.
111124
output=$(<"$IN_FILE")
112125

113126
# Replace variables and create output file
114-
for varname in $list
115-
do
116-
# shellcheck disable=SC2001
117-
# shellcheck disable=SC2016
118-
if [[ "$ADD_QUOTES" == "true" ]]; then
119-
output=$(echo "$output" | sed 's|{{'"$varname"'}}|'\""${!varname}"\"'|g')
120-
else
121-
output=$(echo "$output" | sed 's|{{'"$varname"'}}|'"${!varname}"'|g')
122-
fi
127+
for varname in $list; do
128+
# Required variables are guaranteed non-empty here
129+
value="${!varname}"
130+
# shellcheck disable=SC2001
131+
# shellcheck disable=SC2016
132+
if [[ "$ADD_QUOTES" == "true" ]]; then
133+
output=$(echo "$output" | sed 's|{{'"$varname"'}}|'"\"$value\""'|g')
134+
else
135+
output=$(echo "$output" | sed 's|{{'"$varname"'}}|'"$value"'|g')
136+
fi
137+
done
138+
139+
# Substitute optional variables
140+
for varname in $optional_list; do
141+
value="${!varname}"
142+
if [[ "$ADD_QUOTES" == "true" ]]; then
143+
output=$(echo "$output" | sed 's|{{'"$varname"'}}|'"\"$value\""'|g')
144+
else
145+
output=$(echo "$output" | sed 's|{{'"$varname"'}}|'"$value"'|g')
146+
fi
123147
done
124148

125149
echo "$output" > "$OUT_FILE"

0 commit comments

Comments
 (0)