Merge pull request #29 from praekeltfoundation/use_polars_in_aaqv2

Use Polars LazyFrames in Turn BQ and AAQV2 ingestion

Author: TuscanyBothaReach

.github/workflows/lint.yaml

@@ -16,7 +16,7 @@ jobs:
       - name: Install dependencies
         id: install-deps
         run: |
-          uv sync --only-group dev
+          uv sync --group dev --extra polars
       - name: run linters
         run: |
           chmod +x .ci/lint.sh

.github/workflows/test.yaml

@@ -16,7 +16,7 @@ jobs:
       - name: Install dependencies
         id: install-deps
         run: |
-          uv sync --dev
+          uv sync --dev --extra polars
       - name: test with pytest
         run: |
           uv run pytest -v

README.md

@@ -7,7 +7,7 @@
     <img src="https://github.com/praekeltfoundation/rdw-ingestion-tools/workflows/lint/badge.svg" width="120" />
     <img src="https://github.com/praekeltfoundation/rdw-ingestion-tools/workflows/release/badge.svg" width="145" />
     <img src="https://github.com/praekeltfoundation/rdw-ingestion-tools/workflows/test/badge.svg" width="120" />
-    <img src="https://img.shields.io/badge/version-2.0.2.dev0-orange" width="145" />
+    <img src="https://img.shields.io/badge/version-2.0.2-orange" width="145" />
     <img src="https://img.shields.io/badge/license-MIT-blue" width="100" />
   </p>
 </div>
@@ -116,9 +116,9 @@ pyS3.s3.get_filenames(bucket=bucket, prefix=prefix)
 ```
 
 ### Running an example locally
-1. `uv sync`
+1. `uv sync` or `uv sync --extra polars` for Turn BQ and AAQV2 examples
 2. `uv run --env-file .env examples/{path}` e.g. `uv run --env-file .env examples/turn_bq/cards.py`
 
 #### Running tests
-1. `uv sync --dev`
+1. `uv sync --dev --extra polars`
 2. `uv run pytest -vv --cov`

examples/aaqv2/contents.py

@@ -2,4 +2,4 @@
 
 contents = pyAAQV2().contents.get_contents()
 
-print(contents.head(5))
+print(contents.collect().head(5))

examples/aaqv2/queries.py

@@ -5,4 +5,5 @@
 
 queries = pyAAQV2().queries.get_queries(start_date=start_date, end_date=end_date)
 
-print(queries.head(5))
+
+print(queries.collect().head(5))

examples/aaqv2/urgency_queries.py

@@ -7,4 +7,4 @@
     start_date=start_date, end_date=end_date
 )
 
-print(urgency_queries.head(5))
+print(urgency_queries.collect().head(5))

examples/aaqv2/urgency_rules.py

@@ -2,4 +2,4 @@
 
 urgency_rules = pyAAQV2().urgency_rules.get_urgency_rules()
 
-print(urgency_rules.head(5))
+print(urgency_rules.collect().head(5))

examples/turn_bq/cards.py

@@ -2,8 +2,8 @@
 
 cards = pyTurnBQ().cards.get_cards()
 
-print(cards.head(5))
+print(cards.collect().head(5))
 
 cards_by_id = pyTurnBQ().cards.get_cards_by_id(card_id=817938)
 
-print(cards_by_id)
+print(cards_by_id.collect())

examples/turn_bq/chats.py

@@ -2,10 +2,10 @@
 
 chats = pyTurnBQ().chats.get_chats_by_id(chat_id=1531207597)
 
-print(chats.head(5))
+print(chats.collect().head(5))
 
 chats = pyTurnBQ().chats.get_chats_by_updated_at(
-    from_timestamp="2023-10-01T00:00:00", to_timestamp="2023-10-10T00:00:00"
+    from_timestamp="2023-10-01T00:00:00", to_timestamp="2023-10-01T00:50:00"
 )
 
-print(chats.head(5))
+print(chats.collect().head(5))

examples/turn_bq/contacts.py

@@ -2,10 +2,10 @@
 
 contacts = pyTurnBQ().contacts.get_contacts_by_id(contact_id=2350649091)
 
-print(contacts.head(5))
+print(contacts.collect().head(5))
 
 contacts = pyTurnBQ().contacts.get_contacts_by_updated_at(
-    from_timestamp="2023-10-01T00:00:00", to_timestamp="2023-10-10T00:00:00"
+    from_timestamp="2023-10-01T00:00:00", to_timestamp="2023-10-01T00:00:30"
 )
 
-print(contacts.head(5))
+print(contacts.collect().head(5))