Publisher fixups (#60)

* fixup publisher

* one batch

* StorageClient batch sizes cannot exceed 1000

Author: ewdurbin

main.py

@@ -32,8 +32,8 @@
 SIMPLE_TABLE = os.environ.get("BIGQUERY_SIMPLE_TABLE")
 DOWNLOAD_TABLE = os.environ.get("BIGQUERY_DOWNLOAD_TABLE")
 MAX_BLOBS_PER_RUN = int(
-    os.environ.get("MAX_BLOBS_PER_RUN", "5000")
-)  # Cannot exceed 10,000
+    os.environ.get("MAX_BLOBS_PER_RUN", "1000")
+)  # Cannot exceed 10,000 per load, or 1,000 per batch call to delete blobs
 
 prefix = {Simple.__name__: "simple_requests", Download.__name__: "file_downloads"}
 
@@ -187,5 +187,6 @@ def load_processed_files_into_bigquery(event, context):
     with storage_client.batch():
         for blob in download_source_blobs:
             blob.delete()
+    with storage_client.batch():
         for blob in simple_source_blobs:
             blob.delete()

test_functions.py

@@ -212,8 +212,8 @@ def fake_batch(*a, **kw):
         pretend.call(RESULT_BUCKET),
     ]
     assert bucket_stub.list_blobs.calls == [
-        pretend.call(prefix=f"processed/{partition}/downloads-", max_results=5000),
-        pretend.call(prefix=f"processed/{partition}/simple-", max_results=5000),
+        pretend.call(prefix=f"processed/{partition}/downloads-", max_results=1000),
+        pretend.call(prefix=f"processed/{partition}/simple-", max_results=1000),
     ]
     assert (
         load_job_stub.result.calls