Merge pull request #2838 from fedspendingtransparency/qat

Sprint 118 Production Deploy

Author: tony-sappe

usaspending_api/common/helpers/generic_helper.py

@@ -14,6 +14,7 @@
     OVERLAY_VIEWS,
     DEPENDENCY_FILEPATH,
     MATERIALIZED_VIEWS,
+    CHUNKED_MATERIALIZED_VIEWS,
     MATVIEW_GENERATOR_FILE,
     DEFAULT_MATIVEW_DIR,
 )
@@ -22,6 +23,7 @@
 
 logger = logging.getLogger(__name__)
 TEMP_SQL_FILES = [DEFAULT_MATIVEW_DIR / val["sql_filename"] for val in MATERIALIZED_VIEWS.values()]
+TEMP_SQL_FILES += [DEFAULT_MATIVEW_DIR / val["sql_filename"] for val in CHUNKED_MATERIALIZED_VIEWS.values()]
 
 
 def read_text_file(filepath):

usaspending_api/common/management/commands/matview_runner.py

@@ -3,17 +3,21 @@
 import psycopg2
 import subprocess
 
+from django.core.management import call_command
 from django.core.management.base import BaseCommand
 from pathlib import Path
 
 from usaspending_api.common.data_connectors.async_sql_query import async_run_creates
 from usaspending_api.common.helpers.timing_helpers import ConsoleTimer as Timer
 from usaspending_api.common.matview_manager import (
+    CHUNKED_MATERIALIZED_VIEWS,
     DEFAULT_MATIVEW_DIR,
+    DEFAULT_CHUNKED_MATIVEW_DIR,
     DEPENDENCY_FILEPATH,
     DROP_OLD_MATVIEWS,
     MATERIALIZED_VIEWS,
     MATVIEW_GENERATOR_FILE,
+    CHUNKED_MATVIEW_GENERATOR_FILE,
     OVERLAY_VIEWS,
 )
 from usaspending_api.common.helpers.sql_helpers import get_database_dsn_string
@@ -27,12 +31,15 @@ class Command(BaseCommand):
 
     def faux_init(self, args):
         self.matviews = MATERIALIZED_VIEWS
+        self.chunked_matviews = CHUNKED_MATERIALIZED_VIEWS
         if args["only"]:
             self.matviews = {args["only"]: MATERIALIZED_VIEWS[args["only"]]}
         self.matview_dir = args["temp_dir"]
+        self.matview_chunked_dir = args["temp_chunked_dir"]
         self.no_cleanup = args["leave_sql"]
         self.remove_matviews = not args["leave_old"]
         self.run_dependencies = args["dependencies"]
+        self.chunk_count = args["chunk_count"]
 
     def add_arguments(self, parser):
         parser.add_argument("--only", choices=list(MATERIALIZED_VIEWS.keys()))
@@ -52,9 +59,18 @@ def add_arguments(self, parser):
             help="Choose a non-default directory to store materialized view SQL files.",
             default=DEFAULT_MATIVEW_DIR,
         )
+        parser.add_argument(
+            "--temp-chunked-dir",
+            type=Path,
+            help="Choose a non-default directory to store materialized view SQL files.",
+            default=DEFAULT_CHUNKED_MATIVEW_DIR,
+        )
         parser.add_argument(
             "--dependencies", action="store_true", help="Run the SQL dependencies before the materialized view SQL."
         )
+        parser.add_argument(
+            "--chunk-count", default=10, help="Number of chunks to split chunked matviews into", type=int
+        )
 
     def handle(self, *args, **options):
         """Overloaded Command Entrypoint"""
@@ -74,25 +90,52 @@ def generate_matview_sql(self):
             recursive_delete(self.matview_dir)
         self.matview_dir.mkdir()
 
+        if self.matview_chunked_dir.exists():
+            logger.warning("Clearing dir {}".format(self.matview_chunked_dir))
+            recursive_delete(self.matview_chunked_dir)
+        self.matview_chunked_dir.mkdir()
+
         # IF using this for operations, DO NOT LEAVE hardcoded `python3` in the command
-        exec_str = "python3 {} --quiet --dest={}/ --batch_indexes=3".format(MATVIEW_GENERATOR_FILE, self.matview_dir)
+        # Create main list of Matview SQL files
+        exec_str = f"python3 {MATVIEW_GENERATOR_FILE} --quiet --dest={self.matview_dir}/ --batch_indexes=3"
         subprocess.call(exec_str, shell=True)
 
+        # Create SQL files for Chunked Universal Transaction Matviews
+        for matview, config in self.chunked_matviews.items():
+            exec_str = f"python3 {CHUNKED_MATVIEW_GENERATOR_FILE} --quiet  --file {config['json_filepath']} --chunk-count {self.chunk_count}"
+            subprocess.call(exec_str, shell=True)
+
     def cleanup(self):
         """Cleanup files after run"""
         recursive_delete(self.matview_dir)
 
     def create_views(self):
         loop = asyncio.new_event_loop()
         tasks = []
+
+        # Create Matviews
         for matview, config in self.matviews.items():
-            logger.info("Creating Future for {}".format(matview))
+            logger.info(f"Creating Future for matview {matview}")
             sql = (self.matview_dir / config["sql_filename"]).read_text()
             tasks.append(asyncio.ensure_future(async_run_creates(sql, wrapper=Timer(matview)), loop=loop))
 
+        # Create Chunked Matviews
+        for matview, config in self.chunked_matviews.items():
+            for current_chunk in range(self.chunk_count):
+                chunked_matview = f"{matview}_{current_chunk}"
+                logger.info(f"Creating Future for chunked matview {chunked_matview}")
+                sql = (self.matview_chunked_dir / f"{chunked_matview}.sql").read_text()
+                tasks.append(asyncio.ensure_future(async_run_creates(sql, wrapper=Timer(chunked_matview)), loop=loop))
+
         loop.run_until_complete(asyncio.gather(*tasks))
         loop.close()
 
+        if "universal_transaction_matview" in self.chunked_matviews:
+            logger.info("Inserting data from universal_transaction_matview chunks into single table.")
+            call_command(
+                "combine_universal_transaction_matview_chunks", chunk_count=self.chunk_count, index_concurrency=20,
+            )
+
         for view in OVERLAY_VIEWS:
             run_sql(view.read_text(), "Creating Views")
 

usaspending_api/common/matview_manager.py

@@ -7,9 +7,13 @@
 import usaspending_api.search.models as mv
 
 DEFAULT_MATIVEW_DIR = settings.REPO_DIR.parent / "matviews"
+DEFAULT_CHUNKED_MATIVEW_DIR = settings.REPO_DIR.parent / "chunked_matviews"
 DEPENDENCY_FILEPATH = settings.APP_DIR / "database_scripts" / "matviews" / "functions_and_enums.sql"
-JSON_DIR = settings.APP_DIR / "database_scripts" / "matview_sql_generator"
+JSON_DIR = settings.APP_DIR / "database_scripts" / "matview_generator"
 MATVIEW_GENERATOR_FILE = settings.APP_DIR / "database_scripts" / "matview_generator" / "matview_sql_generator.py"
+CHUNKED_MATVIEW_GENERATOR_FILE = (
+    settings.APP_DIR / "database_scripts" / "matview_generator" / "chunked_matview_sql_generator.py"
+)
 OVERLAY_VIEWS = [
     settings.APP_DIR / "database_scripts" / "matviews" / "vw_award_search.sql",
     settings.APP_DIR / "database_scripts" / "matviews" / "vw_es_award_search.sql",
@@ -113,6 +117,10 @@
                 "sql_filename": "tas_autocomplete_matview.sql",
             },
         ),
+    ]
+)
+CHUNKED_MATERIALIZED_VIEWS = OrderedDict(
+    [
         (
             "universal_transaction_matview",
             {

usaspending_api/conftest_helpers.py

@@ -34,6 +34,12 @@ def __init__(self, index_type):
         self.client = Elasticsearch([settings.ES_HOSTNAME], timeout=settings.ES_TIMEOUT)
         self.template = retrieve_index_template("{}_template".format(self.index_type[:-1]))
         self.mappings = json.loads(self.template)["mappings"]
+        self.etl_config = {
+            "index_name": self.index_name,
+            "query_alias_prefix": self.alias_prefix,
+            "verbose": False,
+            "write_alias": self.index_name + "-alias",
+        }
 
     def delete_index(self):
         self.client.indices.delete(self.index_name, ignore_unavailable=True)
@@ -46,7 +52,7 @@ def update_index(self, **options):
         """
         self.delete_index()
         self.client.indices.create(index=self.index_name, body=self.template)
-        create_aliases(self.client, self.index_name, self.index_type, True)
+        create_aliases(self.client, self.etl_config)
         self._add_contents(**options)
 
     def _add_contents(self, **options):

usaspending_api/data/dabs_submission_window_schedule.csv

@@ -1,23 +1,23 @@
 id,period_start_date,period_end_date,submission_start_date,certification_due_date,submission_due_date,submission_reveal_date,submission_fiscal_year,submission_fiscal_quarter,submission_fiscal_month,is_quarter
-2017031,2016-10-01 00:00:00Z,2016-12-31 00:00:00Z,2017-01-19 00:00:00Z,2017-02-19 00:00:00Z,2017-02-19 00:00:00Z,2017-02-20 00:00:00Z,2017,1,3,True
-2017061,2017-01-01 00:00:00Z,2017-03-31 00:00:00Z,2017-04-19 00:00:00Z,2017-05-19 00:00:00Z,2017-05-19 00:00:00Z,2017-05-20 00:00:00Z,2017,2,6,True
-2017091,2017-04-01 00:00:00Z,2017-06-30 00:00:00Z,2017-07-19 00:00:00Z,2017-08-14 00:00:00Z,2017-08-14 00:00:00Z,2017-08-15 00:00:00Z,2017,3,9,True
-2017121,2017-07-01 00:00:00Z,2017-09-30 00:00:00Z,2017-10-06 00:00:00Z,2017-11-30 00:00:00Z,2017-11-30 00:00:00Z,2017-12-01 00:00:00Z,2017,4,12,True
-2018031,2017-10-01 00:00:00Z,2017-12-31 00:00:00Z,2018-01-19 00:00:00Z,2018-02-14 00:00:00Z,2018-02-14 00:00:00Z,2018-02-15 00:00:00Z,2018,1,3,True
-2018061,2018-01-01 00:00:00Z,2018-03-31 00:00:00Z,2018-04-19 00:00:00Z,2018-05-15 00:00:00Z,2018-05-15 00:00:00Z,2018-05-16 00:00:00Z,2018,2,6,True
-2018091,2018-04-01 00:00:00Z,2018-06-30 00:00:00Z,2018-07-19 00:00:00Z,2018-08-14 00:00:00Z,2018-08-14 00:00:00Z,2018-08-15 00:00:00Z,2018,3,9,True
-2018121,2018-07-01 00:00:00Z,2018-09-30 00:00:00Z,2018-10-19 00:00:00Z,2018-11-14 00:00:00Z,2018-11-14 00:00:00Z,2018-11-15 00:00:00Z,2018,4,12,True
-2019031,2018-10-01 00:00:00Z,2018-12-31 00:00:00Z,2019-02-21 00:00:00Z,2019-03-20 00:00:00Z,2019-03-20 00:00:00Z,2019-03-21 00:00:00Z,2019,1,3,True
-2019061,2019-01-01 00:00:00Z,2019-03-31 00:00:00Z,2019-04-19 00:00:00Z,2019-05-15 00:00:00Z,2019-05-15 00:00:00Z,2019-05-16 00:00:00Z,2019,2,6,True
-2019091,2019-04-01 00:00:00Z,2019-06-30 00:00:00Z,2019-07-19 00:00:00Z,2019-08-14 00:00:00Z,2019-08-14 00:00:00Z,2019-08-15 00:00:00Z,2019,3,9,True
-2019121,2019-07-01 00:00:00Z,2019-09-30 00:00:00Z,2019-10-18 00:00:00Z,2019-11-14 00:00:00Z,2019-11-14 00:00:00Z,2019-11-15 00:00:00Z,2019,4,12,True
-2020031,2019-10-01 00:00:00Z,2019-12-31 00:00:00Z,2020-01-17 00:00:00Z,2020-02-14 00:00:00Z,2020-02-14 00:00:00Z,2020-02-15 00:00:00Z,2020,1,3,True
-2020061,2020-01-01 00:00:00Z,2020-03-31 00:00:00Z,2020-04-17 00:00:00Z,2020-05-15 00:00:00Z,2020-05-15 00:00:00Z,2020-05-16 00:00:00Z,2020,2,6,True
-2020091,2020-04-01 00:00:00Z,2020-06-30 00:00:00Z,2020-07-17 00:00:00Z,2020-08-14 00:00:00Z,2020-07-30 00:00:00Z,2020-07-31 00:00:00Z,2020,3,9,True
-2020121,2020-07-01 00:00:00Z,2020-09-30 00:00:00Z,2020-10-19 00:00:00Z,2020-11-16 00:00:00Z,2020-11-16 00:00:00Z,2020-11-17 00:00:00Z,2020,4,12,True
-2020070,2020-04-01 00:00:00Z,2020-04-30 00:00:00Z,2020-07-17 00:00:00Z,2020-08-14 00:00:00Z,2020-07-30 00:00:00Z,2020-07-31 00:00:00Z,2020,3,7,False
-2020080,2020-05-01 00:00:00Z,2020-05-31 00:00:00Z,2020-07-17 00:00:00Z,2020-08-14 00:00:00Z,2020-07-30 00:00:00Z,2020-07-31 00:00:00Z,2020,3,8,False
-2020090,2020-06-01 00:00:00Z,2020-06-30 00:00:00Z,2020-07-17 00:00:00Z,2020-08-14 00:00:00Z,2020-07-30 00:00:00Z,2020-07-31 00:00:00Z,2020,3,9,False
-2020100,2020-07-01 00:00:00Z,2020-07-31 00:00:00Z,2020-08-19 00:00:00Z,2020-11-16 00:00:00Z,2020-08-28 00:00:00Z,2020-08-29 00:00:00Z,2020,4,10,False
-2020110,2020-08-01 00:00:00Z,2020-08-31 00:00:00Z,2020-09-18 00:00:00Z,2020-11-16 00:00:00Z,2020-09-29 00:00:00Z,2020-09-30 00:00:00Z,2020,4,11,False
-2020120,2020-09-01 00:00:00Z,2020-09-30 00:00:00Z,2020-10-19 00:00:00Z,2020-11-16 00:00:00Z,2020-11-16 00:00:00Z,2020-11-17 00:00:00Z,2020,4,12,False
+2017031,2016-10-01 00:00:00Z,2016-12-31 00:00:00Z,2017-01-19 00:00:00Z,2017-02-19 00:00:00Z,2017-02-19 00:00:00Z,2017-02-19 00:00:00Z,2017,1,3,True
+2017061,2017-01-01 00:00:00Z,2017-03-31 00:00:00Z,2017-04-19 00:00:00Z,2017-05-19 00:00:00Z,2017-05-19 00:00:00Z,2017-05-19 00:00:00Z,2017,2,6,True
+2017091,2017-04-01 00:00:00Z,2017-06-30 00:00:00Z,2017-07-19 00:00:00Z,2017-08-14 00:00:00Z,2017-08-14 00:00:00Z,2017-08-14 00:00:00Z,2017,3,9,True
+2017121,2017-07-01 00:00:00Z,2017-09-30 00:00:00Z,2017-10-06 00:00:00Z,2017-11-30 00:00:00Z,2017-11-30 00:00:00Z,2017-11-30 00:00:00Z,2017,4,12,True
+2018031,2017-10-01 00:00:00Z,2017-12-31 00:00:00Z,2018-01-19 00:00:00Z,2018-02-14 00:00:00Z,2018-02-14 00:00:00Z,2018-02-14 00:00:00Z,2018,1,3,True
+2018061,2018-01-01 00:00:00Z,2018-03-31 00:00:00Z,2018-04-19 00:00:00Z,2018-05-15 00:00:00Z,2018-05-15 00:00:00Z,2018-05-15 00:00:00Z,2018,2,6,True
+2018091,2018-04-01 00:00:00Z,2018-06-30 00:00:00Z,2018-07-19 00:00:00Z,2018-08-14 00:00:00Z,2018-08-14 00:00:00Z,2018-08-14 00:00:00Z,2018,3,9,True
+2018121,2018-07-01 00:00:00Z,2018-09-30 00:00:00Z,2018-10-19 00:00:00Z,2018-11-14 00:00:00Z,2018-11-14 00:00:00Z,2018-11-14 00:00:00Z,2018,4,12,True
+2019031,2018-10-01 00:00:00Z,2018-12-31 00:00:00Z,2019-02-21 00:00:00Z,2019-03-20 00:00:00Z,2019-03-20 00:00:00Z,2019-03-20 00:00:00Z,2019,1,3,True
+2019061,2019-01-01 00:00:00Z,2019-03-31 00:00:00Z,2019-04-19 00:00:00Z,2019-05-15 00:00:00Z,2019-05-15 00:00:00Z,2019-05-15 00:00:00Z,2019,2,6,True
+2019091,2019-04-01 00:00:00Z,2019-06-30 00:00:00Z,2019-07-19 00:00:00Z,2019-08-14 00:00:00Z,2019-08-14 00:00:00Z,2019-08-14 00:00:00Z,2019,3,9,True
+2019121,2019-07-01 00:00:00Z,2019-09-30 00:00:00Z,2019-10-18 00:00:00Z,2019-11-14 00:00:00Z,2019-11-14 00:00:00Z,2019-11-14 00:00:00Z,2019,4,12,True
+2020031,2019-10-01 00:00:00Z,2019-12-31 00:00:00Z,2020-01-17 00:00:00Z,2020-02-14 00:00:00Z,2020-02-14 00:00:00Z,2020-02-14 00:00:00Z,2020,1,3,True
+2020061,2020-01-01 00:00:00Z,2020-03-31 00:00:00Z,2020-04-17 00:00:00Z,2020-05-15 00:00:00Z,2020-05-15 00:00:00Z,2020-05-15 00:00:00Z,2020,2,6,True
+2020091,2020-04-01 00:00:00Z,2020-06-30 00:00:00Z,2020-07-17 00:00:00Z,2020-08-14 00:00:00Z,2020-07-30 00:00:00Z,2020-07-30 00:00:00Z,2020,3,9,True
+2020121,2020-07-01 00:00:00Z,2020-09-30 00:00:00Z,2020-10-19 00:00:00Z,2020-11-16 00:00:00Z,2020-11-16 00:00:00Z,2020-11-16 00:00:00Z,2020,4,12,True
+2020070,2020-04-01 00:00:00Z,2020-04-30 00:00:00Z,2020-07-17 00:00:00Z,2020-08-14 00:00:00Z,2020-07-30 00:00:00Z,2020-07-30 00:00:00Z,2020,3,7,False
+2020080,2020-05-01 00:00:00Z,2020-05-31 00:00:00Z,2020-07-17 00:00:00Z,2020-08-14 00:00:00Z,2020-07-30 00:00:00Z,2020-07-30 00:00:00Z,2020,3,8,False
+2020090,2020-06-01 00:00:00Z,2020-06-30 00:00:00Z,2020-07-17 00:00:00Z,2020-08-14 00:00:00Z,2020-07-30 00:00:00Z,2020-07-30 00:00:00Z,2020,3,9,False
+2020100,2020-07-01 00:00:00Z,2020-07-31 00:00:00Z,2020-08-19 00:00:00Z,2020-11-16 00:00:00Z,2020-08-28 00:00:00Z,2020-08-28 00:00:00Z,2020,4,10,False
+2020110,2020-08-01 00:00:00Z,2020-08-31 00:00:00Z,2020-09-18 00:00:00Z,2020-11-16 00:00:00Z,2020-09-29 00:00:00Z,2020-09-29 00:00:00Z,2020,4,11,False
+2020120,2020-09-01 00:00:00Z,2020-09-30 00:00:00Z,2020-10-19 00:00:00Z,2020-11-16 00:00:00Z,2020-11-16 00:00:00Z,2020-11-16 00:00:00Z,2020,4,12,False

usaspending_api/data/multiprocessing_worker_names.json

@@ -0,0 +1,153 @@
+{
+  "subjects": [
+    "Agent",
+    "Ant",
+    "Archer",
+    "Armadillo",
+    "Assassin",
+    "Bandit",
+    "Beetle",
+    "Boss",
+    "Brain",
+    "Captian",
+    "Champion",
+    "Commando",
+    "Conjuror",
+    "Crusher",
+    "Dart",
+    "Defender",
+    "Dragon",
+    "Electron",
+    "Enchanter",
+    "Eye",
+    "Falcon",
+    "Fox",
+    "Gargoyle",
+    "Genius",
+    "Golem",
+    "Guard",
+    "Guardian",
+    "Hammer",
+    "Heart",
+    "Hunter",
+    "Jackal",
+    "Juggernaut",
+    "Karma",
+    "Knight",
+    "Magician",
+    "Mamba",
+    "Mantis",
+    "Martian",
+    "Mastermind",
+    "Mecha",
+    "Minion",
+    "Monarch",
+    "Mongoose",
+    "Moth",
+    "Nightmare",
+    "Nutron",
+    "Omen",
+    "Phoenix",
+    "Protector",
+    "Proton",
+    "Puma",
+    "Ranger",
+    "Robot",
+    "Rocket",
+    "Saber",
+    "Scythe",
+    "Seer",
+    "Sentinel",
+    "Shadow",
+    "Shepherd",
+    "Slayer",
+    "Smasher",
+    "Spectacle",
+    "Spectre",
+    "Spirit",
+    "Spy",
+    "Storm",
+    "Titan",
+    "Trident",
+    "UFO",
+    "Vector",
+    "Warrior",
+    "Watcher",
+    "Wing",
+    "Wizard",
+    "Wolf",
+    "Wonder"
+  ],
+  "attributes": [
+    "Amber",
+    "Artifical",
+    "Atomic",
+    "Bionic",
+    "Black",
+    "Blue",
+    "Capped",
+    "Captian",
+    "Colossal",
+    "Commander",
+    "Crazy",
+    "Curious",
+    "Dark",
+    "Doctor",
+    "Eager",
+    "Earth",
+    "Ethereal",
+    "Fabulous",
+    "Fallen",
+    "Fancy",
+    "Fantastic",
+    "Fearless",
+    "Fiery",
+    "Flying",
+    "Gentle",
+    "Giant",
+    "Glorious",
+    "Green",
+    "Grey",
+    "Heavy",
+    "Humble",
+    "Ice",
+    "Infamous",
+    "Intelligent",
+    "Invisible",
+    "Jade",
+    "Kind",
+    "Mega",
+    "Mighty",
+    "Mysterious",
+    "Nefarious",
+    "Night",
+    "Nocturnal",
+    "Orange",
+    "Prime",
+    "Professor",
+    "Purple",
+    "Quick",
+    "Red",
+    "Ruby",
+    "Sassy",
+    "Sauve",
+    "Scarlet",
+    "Sensitive",
+    "Silver",
+    "Smooth",
+    "Sneeky",
+    "Speedy",
+    "Super",
+    "Supreme",
+    "The",
+    "Thunder",
+    "Ultra",
+    "Unarmed",
+    "Universal",
+    "White",
+    "Wild",
+    "Winged",
+    "Wonder",
+    "Yellow"
+  ]
+}