Skip to content

Commit 628211c

Browse files
davidgamezdavidgamez
committed
Merge branch 'main' into feat/gbfs_bounding_box
2 parents d094861 + 2d988ff commit 628211c

File tree

29 files changed

+2279
-847
lines changed

29 files changed

+2279
-847
lines changed

‎.github/workflows/build-test.yml‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ on:
1818
env:
1919
python_version: '3.11'
2020
java_version: '11' # needed by setup-openapi-generator.sh
21+
liquibase_version: '4.33.0'
2122

2223
jobs:
2324
build-test:
@@ -66,17 +67,21 @@ jobs:
6667
scripts/lint-tests.sh
6768
6869
- name: Install Liquibase
69-
run: |
70-
wget -O- https://repo.liquibase.com/liquibase.asc | gpg --dearmor > liquibase-keyring.gpg && \
71-
cat liquibase-keyring.gpg | sudo tee /usr/share/keyrings/liquibase-keyring.gpg > /dev/null && \
72-
echo 'deb [trusted=yes arch=amd64 signed-by=/usr/share/keyrings/liquibase-keyring.gpg] https://repo.liquibase.com stable main' | sudo tee /etc/apt/sources.list.d/liquibase.list
73-
74-
sudo apt-get update
75-
sudo apt-get install liquibase=4.25.1
76-
70+
env:
71+
LIQUIBASE_VERSION: ${{ env.liquibase_version }}
72+
run: |
73+
curl -sSL https://github.com/liquibase/liquibase/releases/download/v${LIQUIBASE_VERSION}/liquibase-${LIQUIBASE_VERSION}.tar.gz -o liquibase.tar.gz
74+
rm -rf liquibase-dist
75+
mkdir liquibase-dist
76+
tar -xzf liquibase.tar.gz -C liquibase-dist
77+
sudo rm -rf /usr/local/liquibase
78+
sudo mv liquibase-dist /usr/local/liquibase
79+
sudo ln -sf /usr/local/liquibase/liquibase /usr/local/bin/liquibase
80+
liquibase --version
81+
7782
- name: Run Liquibase on Python functions test DB
83+
working-directory: ${{ github.workspace }}/liquibase
7884
run: |
79-
export LIQUIBASE_CLASSPATH="liquibase"
8085
export LIQUIBASE_COMMAND_CHANGELOG_FILE="changelog.xml"
8186
export LIQUIBASE_COMMAND_URL=jdbc:postgresql://localhost:54320/MobilityDatabaseTest
8287
export LIQUIBASE_COMMAND_USERNAME=postgres

‎.github/workflows/datasets-batch-deployer.yml‎

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ on:
4141

4242
env:
4343
python_version: '3.11'
44+
liquibase_version: '4.33.0'
4445

4546
jobs:
4647
terraform:
@@ -88,23 +89,21 @@ jobs:
8889
python-version: ${{ env.python_version }}
8990

9091
- name: Install Liquibase
91-
run: |
92-
wget -O- https://repo.liquibase.com/liquibase.asc | gpg --dearmor > liquibase-keyring.gpg && \
93-
cat liquibase-keyring.gpg | sudo tee /usr/share/keyrings/liquibase-keyring.gpg > /dev/null && \
94-
echo 'deb [trusted=yes arch=amd64 signed-by=/usr/share/keyrings/liquibase-keyring.gpg] https://repo.liquibase.com stable main' | sudo tee /etc/apt/sources.list.d/liquibase.list
92+
env:
93+
LIQUIBASE_VERSION: ${{ env.liquibase_version }}
94+
run: |
95+
curl -sSL https://github.com/liquibase/liquibase/releases/download/v${LIQUIBASE_VERSION}/liquibase-${LIQUIBASE_VERSION}.tar.gz -o liquibase.tar.gz
96+
rm -rf liquibase-dist
97+
mkdir liquibase-dist
98+
tar -xzf liquibase.tar.gz -C liquibase-dist
99+
sudo rm -rf /usr/local/liquibase
100+
sudo mv liquibase-dist /usr/local/liquibase
101+
sudo ln -sf /usr/local/liquibase/liquibase /usr/local/bin/liquibase
102+
liquibase --version
95103
96-
sudo apt-get update
97-
sudo apt-get install liquibase=4.25.1
98-
99-
# Uncomment the following block to test the local databases connections
100-
# - name: Test Database Connection
101-
# run: |
102-
# sudo apt-get update && sudo apt-get install -y postgresql-client
103-
# PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d MobilityDatabase -c "SELECT version();"
104-
105104
- name: Run Liquibase on Python functions DB
105+
working-directory: ${{ github.workspace }}/liquibase
106106
run: |
107-
export LIQUIBASE_CLASSPATH="liquibase"
108107
export LIQUIBASE_COMMAND_CHANGELOG_FILE="changelog.xml"
109108
export LIQUIBASE_COMMAND_URL=jdbc:postgresql://localhost:5432/MobilityDatabase
110109
export LIQUIBASE_COMMAND_USERNAME=postgres

‎.github/workflows/db-update.yml‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ on:
7070

7171
env:
7272
python_version: '3.11'
73+
liquibase_version: '4.33.0'
7374

7475
jobs:
7576
db-schema-update:
@@ -111,22 +112,27 @@ jobs:
111112
sudo apt-get update && sudo apt-get install -y postgresql-client
112113
PGPASSWORD=${{ secrets.DB_USER_PASSWORD }} psql -h localhost -p 5432 -U ${{ secrets.DB_USER_NAME }} -d ${{ inputs.DB_NAME }} -c "SELECT version();"
113114
115+
- name: Install Liquibase
116+
env:
117+
LIQUIBASE_VERSION: ${{ env.liquibase_version }}
118+
run: |
119+
curl -sSL https://github.com/liquibase/liquibase/releases/download/v${LIQUIBASE_VERSION}/liquibase-${LIQUIBASE_VERSION}.tar.gz -o liquibase.tar.gz
120+
rm -rf liquibase-dist
121+
mkdir liquibase-dist
122+
tar -xzf liquibase.tar.gz -C liquibase-dist
123+
sudo rm -rf /usr/local/liquibase
124+
sudo mv liquibase-dist /usr/local/liquibase
125+
sudo ln -sf /usr/local/liquibase/liquibase /usr/local/bin/liquibase
126+
liquibase --version
127+
114128
- name: Run Liquibase
129+
working-directory: ${{ github.workspace }}/liquibase
115130
run: |
116-
wget -O- https://repo.liquibase.com/liquibase.asc | gpg --dearmor > liquibase-keyring.gpg && \
117-
cat liquibase-keyring.gpg | sudo tee /usr/share/keyrings/liquibase-keyring.gpg > /dev/null && \
118-
echo 'deb [trusted=yes arch=amd64 signed-by=/usr/share/keyrings/liquibase-keyring.gpg] https://repo.liquibase.com stable main' | sudo tee /etc/apt/sources.list.d/liquibase.list
119-
120-
sudo apt-get update
121-
sudo apt-get install liquibase=4.25.1
122-
123-
export LIQUIBASE_CLASSPATH="liquibase"
124131
export LIQUIBASE_COMMAND_CHANGELOG_FILE="changelog.xml"
125132
export LIQUIBASE_COMMAND_URL=jdbc:postgresql://localhost:5432/${{ inputs.DB_NAME }}
126133
export LIQUIBASE_COMMAND_USERNAME=${{ secrets.DB_USER_NAME }}
127134
export LIQUIBASE_COMMAND_PASSWORD=${{ secrets.DB_USER_PASSWORD }}
128135
export LIQUIBASE_LOG_LEVEL=FINE
129-
130136
liquibase update
131137
132138
db-content-update:

‎.github/workflows/integration-tests-pr.yml‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ env:
1919
python_version: '3.11'
2020
java_version: '11' # needed by setup-openapi-generator.sh
2121
API_URL: 'http://localhost:8080'
22+
liquibase_version: '4.33.0'
2223

2324
jobs:
2425
integration-tests-pr:
@@ -61,17 +62,21 @@ jobs:
6162
working-directory: ${{ github.workspace }}
6263

6364
- name: Install Liquibase
65+
env:
66+
LIQUIBASE_VERSION: ${{ env.liquibase_version }}
6467
run: |
65-
wget -O- https://repo.liquibase.com/liquibase.asc | gpg --dearmor > liquibase-keyring.gpg && \
66-
cat liquibase-keyring.gpg | sudo tee /usr/share/keyrings/liquibase-keyring.gpg > /dev/null && \
67-
echo 'deb [trusted=yes arch=amd64 signed-by=/usr/share/keyrings/liquibase-keyring.gpg] https://repo.liquibase.com stable main' | sudo tee /etc/apt/sources.list.d/liquibase.list
68-
69-
sudo apt-get update
70-
sudo apt-get install liquibase=4.25.1
68+
curl -sSL https://github.com/liquibase/liquibase/releases/download/v${LIQUIBASE_VERSION}/liquibase-${LIQUIBASE_VERSION}.tar.gz -o liquibase.tar.gz
69+
rm -rf liquibase-dist
70+
mkdir liquibase-dist
71+
tar -xzf liquibase.tar.gz -C liquibase-dist
72+
sudo rm -rf /usr/local/liquibase
73+
sudo mv liquibase-dist /usr/local/liquibase
74+
sudo ln -sf /usr/local/liquibase/liquibase /usr/local/bin/liquibase
75+
liquibase --version
7176
7277
- name: Run Liquibase on API local DB
78+
working-directory: ${{ github.workspace }}/liquibase
7379
run: |
74-
export LIQUIBASE_CLASSPATH="liquibase"
7580
export LIQUIBASE_COMMAND_CHANGELOG_FILE="changelog.xml"
7681
export LIQUIBASE_COMMAND_URL=jdbc:postgresql://localhost:5432/MobilityDatabase
7782
export LIQUIBASE_COMMAND_USERNAME=postgres

‎api/src/shared/common/db_utils.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ def get_all_gtfs_feeds(
164164
165165
:return: The GTFS feeds in an iterator.
166166
"""
167-
batch_size = os.getenv("BATCH_SIZE", 100)
167+
batch_size = int(os.getenv("BATCH_SIZE", "500"))
168168
batch_query = db_session.query(Gtfsfeed).order_by(Gtfsfeed.stable_id).yield_per(batch_size)
169169
if published_only:
170170
batch_query = batch_query.filter(Gtfsfeed.operational_status == "published")
@@ -182,7 +182,6 @@ def get_all_gtfs_feeds(
182182
contains_eager(Gtfsfeed.gtfsdatasets)
183183
.joinedload(Gtfsdataset.validation_reports)
184184
.joinedload(Validationreport.features),
185-
joinedload(Gtfsfeed.visualization_dataset),
186185
*get_joinedload_options(include_extracted_location_entities=True),
187186
)
188187
)

‎functions-python/export_csv/src/main.py‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,6 @@ def get_gtfs_feed_csv_data(
208208
:return: Dictionary with feed data formatted for CSV output.
209209
"""
210210
joined_features = ""
211-
validated_at = None
212211
bounding_box = None
213212

214213
# First extract the common feed data
@@ -240,18 +239,16 @@ def get_gtfs_feed_csv_data(
240239
if features
241240
else ""
242241
)
243-
if latest_report.validated_at:
244-
validated_at = latest_report.validated_at
245-
if latest_dataset.bounding_box:
246-
shape = to_shape(latest_dataset.bounding_box)
247-
if shape and shape.bounds:
248-
bounding_box = BoundingBox(
249-
minimum_latitude=shape.bounds[1],
250-
maximum_latitude=shape.bounds[3],
251-
minimum_longitude=shape.bounds[0],
252-
maximum_longitude=shape.bounds[2],
253-
extracted_on=validated_at,
254-
)
242+
if feed.bounding_box:
243+
shape = to_shape(feed.bounding_box)
244+
if shape and shape.bounds:
245+
bounding_box = BoundingBox(
246+
minimum_latitude=shape.bounds[1],
247+
maximum_latitude=shape.bounds[3],
248+
minimum_longitude=shape.bounds[0],
249+
maximum_longitude=shape.bounds[2],
250+
extracted_on=feed.bounding_box_dataset.downloaded_at,
251+
)
255252

256253
# Keep the bounding box for that GTFS feed so it can be used in associated real-time feeds, if any
257254
if bounding_box:

‎functions-python/export_csv/tests/__init__.py‎

Whitespace-only changes.

‎functions-python/export_csv/tests/conftest.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ def populate_database(db_session):
156156
hosted_url=f"https://url_prefix/{feed_stable_id}/dataset-{i}_some_fake_hosted_url",
157157
note=f"dataset-{i} Some fake note",
158158
hash=fake.sha256(),
159-
downloaded_at=datetime.utcnow(),
159+
downloaded_at=datetime(2025, 1, 12),
160160
stable_id=f"dataset-{i}",
161161
)
162162
validation_report = Validationreport(
@@ -175,6 +175,9 @@ def populate_database(db_session):
175175
gtfs_dataset.locations = locations
176176

177177
active_gtfs_feeds[feed_index].gtfsdatasets.append(gtfs_dataset)
178+
db_session.flush()
179+
active_gtfs_feeds[feed_index].bounding_box = gtfs_dataset.bounding_box
180+
active_gtfs_feeds[feed_index].bounding_box_dataset_id = gtfs_dataset.id
178181
active_gtfs_feeds[0].locations = locations
179182
active_gtfs_feeds[1].locations = locations
180183

‎functions-python/export_csv/tests/test_export_csv_main.py‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
# limitations under the License.
1515
#
1616
import io
17+
import unittest
1718

1819
import pandas as pd
1920
import pandas.testing as pdt
@@ -34,13 +35,14 @@
3435
""" # noqa
3536

3637

37-
def test_export_csv():
38-
csv_file_path = "./output.csv"
39-
main.export_csv(csv_file_path)
40-
df_actual = pd.read_csv(csv_file_path)
41-
print(f"Collected data for {len(df_actual)} feeds.")
38+
class TestExportCSV(unittest.TestCase):
39+
def test_export_csv(self):
40+
csv_file_path = "./output.csv"
41+
main.export_csv(csv_file_path)
42+
df_actual = pd.read_csv(csv_file_path)
43+
print(f"Collected data for {len(df_actual)} feeds.")
4244

43-
df_expected = pd.read_csv(io.StringIO(expected_csv))
45+
df_expected = pd.read_csv(io.StringIO(expected_csv))
4446

45-
pdt.assert_frame_equal(df_actual, df_expected)
46-
print("DataFrames are equal.")
47+
pdt.assert_frame_equal(df_actual, df_expected)
48+
print("DataFrames are equal.")

‎functions-python/pmtiles_builder/src/csv_cache.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
import csv
1717
import logging
1818
import os
19+
import subprocess
20+
from pathlib import Path
1921
from typing import TypedDict, List, Dict
2022

2123
from gtfs import stop_txt_is_lat_log_required
@@ -39,6 +41,40 @@ class ShapeTrips(TypedDict):
3941
trip_ids: List[str]
4042

4143

44+
def get_volume_size(mountpoint: str):
45+
"""
46+
Returns the total size of the specified filesystem mount point in a human-readable format.
47+
48+
This function uses the `df` command-line utility to determine the size of the filesystem
49+
mounted at the path specified by `mountpoint`. If the mount point does not exist, the function
50+
prints an error message to the standard error and returns "N/A".
51+
52+
Parameters:
53+
mountpoint: str
54+
The filesystem mount point path to check.
55+
56+
Returns:
57+
str
58+
The total size of the specified filesystem mount point in human-readable format. If the
59+
mount point is not found, returns "N/A".
60+
"""
61+
mp = Path(mountpoint)
62+
if not mp.exists():
63+
logging.warning("Mountpoint not found: %s", mountpoint)
64+
return "N/A"
65+
cmd = [
66+
"bash",
67+
"-c",
68+
"df -h \"$1\" | awk 'NR==2 {print $2}'",
69+
"_", # $0 placeholder (ignored)
70+
str(mp), # $1
71+
]
72+
result = subprocess.run(cmd, capture_output=True, text=True, check=True)
73+
size = result.stdout.strip()
74+
75+
return size
76+
77+
4278
class CsvCache:
4379
"""
4480
CsvCache provides cached access to GTFS CSV files in a specified working directory.
@@ -68,6 +104,7 @@ def __init__(
68104
self.trips_no_shapes_per_route: Dict[str, List[str]] = {}
69105

70106
self.logger.info("Using work directory: %s", self.workdir)
107+
self.logger.info("Size of workdir: %s", get_volume_size(self.workdir))
71108

72109
def debug_log_size(self, label: str, obj: object) -> None:
73110
"""Log the deep size of an object in bytes when DEBUG is enabled."""

0 commit comments

Comments
 (0)