Change DB Schema (#13)

* Begin setting up tests

* Tests for return format

* Update for new db

* Allow release ingestion to happen in parallel

* Remove restriction

* Add indexes

* Fix tests

* Schema

* Add beta compose

* Update container names

* Remove ports

* ports

* Parse effective date

* Update version

* Readme

Author: mustyoshi

.docker/docker-compose.beta.yml

@@ -0,0 +1,25 @@
+version: '3'
+services:
+  congress_parser_api:
+    container_name: beta_congress_parser_api
+    environment:
+    - STAGE=prod
+    - db_host=10.0.0.248:5432
+    - db_table=us_code_beta
+    ports: ["9091:9090"]
+  congress_viewer_app:
+    container_name: beta_congress_viewer_app
+    volumes:
+    - /var/www/congress-beta:/usr/src/app/build
+    entrypoint:
+    - "yarn"
+    command:
+    - "build"
+    ports: ["3001:3000"]
+  congress_postgres:
+    container_name: beta_congress_postgres
+    image: tianon/true
+networks:
+  parser:
+    external:
+      name: docker_parser

.docker/docker-compose.prod.yml

@@ -11,22 +11,8 @@ services:
     - "yarn"
     command:
     - "build"
-#  congress_nginx:
-#    container_name: congress_nginx
-#    build:
-#      context: ../
-#      dockerfile: .docker/nginx.dockerfile
-#    ports:
-#      - 80:80
-#      - 443:443
-#    networks:
-#      parser:
-#    volumes:
-#      - /etc/letsencrypt:/etc/letsencrypt
-#      - ../frontend/.nginx/:/etc/nginx/sites-enabled/
-#      - /var/www/congress:/var/www/congress
   congress_postgres:
-    image: tianon/true 
+    image: tianon/true
 networks:
   parser:
     external:

.gitignore

@@ -3,4 +3,5 @@
 frontend/yarn-error.log
 .vscode
 .docker/docker-compose.local.yml
+backend/venv/
 venv/

README.md

@@ -8,14 +8,13 @@ docker-compose .docker/docker-compose.yml up -d
 ```
 
 ## Loading the database
-From the backend folder, you will need to tell it to parse some files before you can view them. importers.releases will load 3 release points from the US Code website. ~10MB compressed and put it into the database ~500MB in the db.
-run_through will load the first 100 bills from the senate and the first 100 bills from the house (limited to 100 for speed), you can change it on line 671.
-A semi up to date postgres dump is available for [download](https://congress.dev/congress.backup)
+From the backend folder, you will need to tell it to parse some files before you can view them. importers.releases will load 1 release point from the US Code website. ~10MB compressed and put it into the database ~500MB in the db.
+A semi up to date postgres dump is available for [download](https://files.congress.dev/congress_beta.backup)
 
 ```bash
 docker exec -it docker_parser_api bash
 
 python3 -m billparser.importers.releases rp.json
-python3 -m billparser.run_through
+python3 -m billparser.importers.bills bills.json
 
 ```

backend/.docker/Dockerfile

@@ -9,7 +9,4 @@ RUN pip3 install -r requirements.txt
 COPY . /usr/src/app
 WORKDIR /usr/src/app
 
-EXPOSE 9090
-EXPOSE 80
-
 ENTRYPOINT ["python3", "-m", "billparser"]

backend/billparser/__main__.py

@@ -33,16 +33,17 @@
     get_versions,
     get_revisions,
     get_revision_diff,
+    get_latest_base
 )
 from billparser.db.models import (
-    Chapter,
-    Section,
-    Content,
-    ContentDiff,
+    USCChapter,
+    USCSection,
+    USCContent,
+    USCContentDiff,
     Version,
-    Bill,
-    BillVersion,
-    BillContent,
+    Legislation,
+    LegislationVersion,
+    LegislationContent,
 )
 from billparser.helpers import treeify
 
@@ -112,18 +113,19 @@ def bills() -> str:
     start = time.time()
     bill_by_number = defaultdict(lambda: {})
     for bill in get_bills(house, senate, query, incl, decl):
-        title = "{}-{}".format(bill.chamber[0], bill.bill_number)
-        obj = row2dict(bill)
+        title = "{}-{}".format(bill.chamber.value[0], bill.number)
+        #obj = row2dict(bill)
+        obj = bill.to_dict()
         obj["versions"] = [
-            row2dict(x)
+            x.to_dict()
             for x in bill.versions
             if (
                 True
                 or (  # Filter based on only the included versions?
                     incl == "" and decl == ""
                 )
-                or (incl != "" and x.bill_version in incl.split(","))
-                or (decl != "" and x.bill_version in decl.split(","))
+                or (incl != "" and x.legislation_version in incl.split(","))
+                or (decl != "" and x.legislation_version in decl.split(","))
             )
         ]
         bill_by_number[title] = obj
@@ -267,9 +269,8 @@ def sections(chapter_id: int) -> str:
     Returns:
         str: Stringifed array of the rows
     """
-    latest_base = (
-        current_session.query(Version).filter(Version.base_id == None).all()[0]
-    )
+    latest_base = get_latest_base()
+
     res = []
     for section in get_sections(str(chapter_id), latest_base.version_id):
         res.append(section.to_dict())
@@ -311,9 +312,7 @@ def contents(section_id: int) -> str:
     Returns:
         str: Stringified array of the content rows
     """
-    latest_base = (
-        current_session.query(Version).filter(Version.base_id == None).all()[0]
-    )
+    latest_base = get_latest_base()
     res = []
     for section in get_content(str(section_id), latest_base.version_id):
         res.append(section.to_dict())

backend/billparser/actions/__init__.py

@@ -83,7 +83,10 @@
     "INSERT-CHAPTER-AT-END": [
         r"Title (?P<title>\d\d?A?), (?P<document_title>.+), is amended by adding at the end the following new chapter:?"
     ],
-    "TERM-DEFINITION": [r"The term \"(?P<term>.+?)\" means (?P<term_def>.+?)."],
+    "TERM-DEFINITION": [
+        r"The term \"(?P<term>.+?)\" means (?P<term_def>.+?).",
+        r"The term (?P<term>.+?) means (?P<term_def>.+?).",
+    ],
 }
 
 SuchCodeRegex = re.compile(r"(Section|paragraph) (?P<section>\d*)\(", re.IGNORECASE)
@@ -155,6 +158,7 @@ def __init__(self, **kwargs):
         self.cited_content = kwargs.get("cited_content", None)
         self.last_title = kwargs.get("last_title", "")
         self.next = kwargs.get("next", None)
+        self.legislation_content = kwargs.get("legislation_content", None)
         # print(kwargs)
 
     def set_action(self, action):

backend/billparser/actions/insert.py

@@ -1,7 +1,7 @@
-from billparser.transformer import Session
+from billparser.db.handler import Session
 from billparser.translater import translate_paragraph
 
-from billparser.db.models import ContentDiff, Section, Content
+from billparser.db.models import USCContentDiff, USCSection, USCContent
 
 from billparser.actions import ActionObject
 
@@ -24,6 +24,7 @@ def recursive_content(
     version_id: int,
     last_ident: str,
     session: "Session",
+    legislation_id: int,
 ) -> None:
     """
     This is the function that "inserts" a new block of content from a bill.
@@ -67,10 +68,10 @@ def recursive_content(
                     " ".join(content_elem.itertext()).strip().replace("\n", " ")
                 )
         if "heading" in heading.tag:
-            content = Content(
+            content = USCContent(
                 content_type=search_element.tag,
                 usc_ident=ident,
-                section_id=section_id,
+                usc_section_id=section_id,
                 parent_id=content_id,
                 order_number=order,
                 version_id=version_id,
@@ -86,27 +87,28 @@ def recursive_content(
                 content_str = (
                     " ".join(content_elem.itertext()).strip().replace("\n", " ")
                 )
-            content = Content(
+            content = USCContent(
                 content_type=search_element.tag,
                 usc_ident=ident,
-                section_id=section_id,
+                usc_section_id=section_id,
                 parent_id=content_id,
                 order_number=order,
                 version_id=version_id,
             )
         session.add(content)
         session.flush()
-        diff = ContentDiff(
-            chapter_id=chapter_id,
+        diff = USCContentDiff(
+            usc_chapter_id=chapter_id,
             version_id=version_id,
-            content_id=content.content_id,
-            section_id=section_id,
+            usc_content_id=content.usc_content_id,
+            usc_section_id=section_id,
             number=enum.attrib.get("value", ""),
             section_display=enum.text,
             content_str=content_str,
             heading=heading.text
             if heading is not None and heading.text != content_str
             else None,
+            legislation_content_id=legislation_id,
         )
         session.add(diff)
         session.commit()
@@ -116,12 +118,13 @@ def recursive_content(
                 recursive_content(
                     chapter_id,
                     section_id,
-                    content.content_id,
+                    content.usc_content_id,
                     elem,
                     order,
                     version_id,
                     ident,
                     session,
+                    legislation_id,
                 )
                 order = order + 1
 
@@ -135,24 +138,30 @@ def insert_section_after(action_obj: ActionObject, session: "Session") -> None:
         session (Session): DB session to insert into
     """
     cited_content = action_obj.cited_content
+    legislation_content = action_obj.legislation_content
+    if legislation_content is not None:
+        legislation_id = legislation_content.legislation_content_id
+    else:
+        legislation_id = None
     new_vers_id = action_obj.version_id
     if action_obj.next is not None:
         chapter = (
-            session.query(Section)
-            .filter(Section.section_id == cited_content.section_id)
+            session.query(USCSection)
+            .filter(USCSection.usc_section_id == cited_content.usc_section_id)
             .all()
         )
         if len(chapter) > 0:
-            chapter_id = chapter[0].chapter_id
+            chapter_id = chapter[0].usc_chapter_id
             recursive_content(
                 chapter_id,
-                cited_content.section_id,
+                cited_content.usc_section_id,
                 cited_content.parent_id,
                 translate_paragraph(action_obj.next)[0],
                 cited_content.order_number + 1,
                 new_vers_id,
                 "/".join(cited_content.usc_ident.split("/")[:-1]),
                 session,
+                legislation_id,
             )
             session.commit()
 
@@ -169,9 +178,9 @@ def insert_end(action_obj: ActionObject, session: "Session") -> None:
     cited_content = action_obj.cited_content
     if action_obj.next is not None:
         last_content = (
-            session.query(Content)
-            .filter(Content.parent_id == cited_content.content_id)
-            .order_by(Content.order_number.desc())
+            session.query(USCContent)
+            .filter(USCContent.parent_id == cited_content.usc_content_id)
+            .order_by(USCContent.order_number.desc())
             .limit(1)
             .all()
         )
@@ -193,43 +202,50 @@ def insert_text_end(action_obj: ActionObject, session: "Session") -> None:
     """
     action = action_obj.action
     cited_content = action_obj.cited_content
-    log.debug(cited_content.content_id)
+    log.debug(cited_content.usc_content_id)
     new_vers_id = action_obj.version_id
     to_replace = action.get("to_replace", "")
     chapter = (
-        session.query(Section)
-        .filter(Section.section_id == cited_content.section_id)
+        session.query(USCSection)
+        .filter(USCSection.usc_section_id == cited_content.usc_section_id)
         .limit(1)
         .all()
     )
     diff = None
+    legislation_content = action_obj.legislation_content
+    if legislation_content is not None:
+        legislation_id = legislation_content.legislation_content_id
+    else:
+        legislation_id = None
     if len(chapter) > 0:
-        chapter_id = chapter[0].chapter_id
+        chapter_id = chapter[0].usc_chapter_id
         if cited_content.heading is not None:
             heading_diff = cited_content.heading + " " + to_replace
-            if heading_diff != cited_content.heading:
-                diff = ContentDiff(
-                    content_id=cited_content.content_id,
-                    section_id=cited_content.section_id,
-                    chapter_id=chapter_id,
+            if heading_diff != cited_content.heading and heading_diff != " ":
+                diff = USCContentDiff(
+                    usc_content_id=cited_content.usc_content_id,
+                    usc_section_id=cited_content.usc_section_id,
+                    usc_chapter_id=chapter_id,
                     version_id=new_vers_id,
                     heading=heading_diff,
+                    legislation_content_id=legislation_id,
                 )
         elif cited_content.content_str is not None:
             content_diff = cited_content.content_str + " " + to_replace
-            if content_diff != cited_content.content_str:
-                diff = ContentDiff(
-                    content_id=cited_content.content_id,
-                    section_id=cited_content.section_id,
-                    chapter_id=chapter_id,
+            if content_diff != cited_content.content_str and heading_diff != " ":
+                diff = USCContentDiff(
+                    usc_content_id=cited_content.usc_content_id,
+                    usc_section_id=cited_content.usc_section_id,
+                    usc_chapter_id=chapter_id,
                     version_id=new_vers_id,
                     content_str=content_diff,
+                    legislation_content_id=legislation_id,
                 )
     if diff is not None:
         log.debug("adding")
         session.add(diff)
         session.commit()
-        log.debug("Added diff", diff.diff_id)
+        log.debug("Added diff", diff.usd_content_diff_id)
 
 
 def insert_text_after(action_obj: ActionObject, session: "Session") -> None: