Add some examples

Author: glesica

README.md

@@ -55,7 +55,8 @@ to go for non-developers.
   1. Clone the whole repo, `cd` into the `pyrs990` directory
   1. Install dependencies - `poetry install`
   1. Run it, some very simple examples are below:
-      1. `poetry run python -m pyrs990 --zip 59801 --use-disk-cache`
+      1. `poetry run pyrs990 --zip 59801 --use-disk-cache`
+      1. `poetry run pyrs990 --load-filters examples/has-a-website.json --use-disk-cache`
       1. ...more examples coming soon
   1. Run the commands again, notice the cache speedup
   1. The cache is set to `./.pyrs990-cache/`
@@ -151,6 +152,57 @@ reduce the number of files you have to download.
 
 See the example queries for more information.
 
+### Index Fields
+
+The index fields available for filtering are listed below. Note that,
+in general, the BMF index may be a bit more reliable since it points
+directly at the filing data files and doesn't require joining on the
+EIN, which we haven't entirely figured out yet (there seem to be EIN
+values missing from one or the other index in some cases).
+
+**BMF Index:**
+
+  - EIN - used to join indices
+  - NAME
+  - ICO
+  - STREET
+  - CITY
+  - STATE
+  - ZIP
+  - GROUP
+  - SUBSECTION
+  - AFFILIATION
+  - CLASSIFICATION
+  - RULING
+  - DEDUCTIBILITY
+  - FOUNDATION
+  - ACTIVITY
+  - ORGANIZATION
+  - STATUS
+  - TAX_PERIOD
+  - ASSET_CD
+  - INCOME_CD
+  - FILING_REQ_CD
+  - PF_FILING_REQ_CD
+  - ACCT_PD
+  - ASSET_AMT
+  - INCOME_AMT
+  - REVENUE_AMT
+  - NTEE_CD
+  - SORT_NAME
+
+**Annual Index:**
+
+  - RETURN_ID
+  - FILING_TYPE
+  - EIN - used for joining indices
+  - TAX_PERIOD
+  - SUB_DATE
+  - TAXPAYER_NAME
+  - RETURN_TYPE
+  - DLN
+  - OBJECT_ID - points to filing data
+
 ### Sources
 
 The [IRS BMF index files](https://www.irs.gov/charities-non-profits/exempt-organizations-business-master-file-extract-eo-bmf)

examples/has-a-website.json

@@ -0,0 +1,8 @@
+{
+  "index": {
+    "zip": "59801"
+  },
+  "filing": {
+    "website_address": "^(?!N/A).*$"
+  }
+}

examples/words-in-name.json

@@ -0,0 +1,7 @@
+{
+  "index": {
+    "zip": "59801",
+    "taxpayer_name": "land|river"
+  },
+  "filing": {}
+}

pyrs990/filters.py

@@ -1,5 +1,5 @@
 import re
-from typing import Mapping
+from typing import List, Mapping, Pattern
 
 from .filing import Filing, FilingFilter
 from .index import IndexFilter, IndexRecord
@@ -13,16 +13,20 @@ def __init__(self, name: str):
 
 
 def filter_filings(filters: Mapping[str, str],) -> FilingFilter:
+    patterns: List[Pattern] = []
+    for field_name, filter_text in filters.items():
+        patterns.append(re.compile(filter_text, flags=re.IGNORECASE))
+
     def _filter(filing: Filing) -> bool:
         passed = True
-        for fieldName in filters:
-            filter_text = filters[fieldName]
-            if not hasattr(filing, fieldName):
-                raise FieldNotFound(fieldName)
-            value = getattr(filing, fieldName)
+        for i, f in enumerate(filters):
+            if not hasattr(filing, f):
+                raise FieldNotFound(f)
+            value = getattr(filing, f)
             if value is None:
                 return False
-            match = re.match(filter_text, str(value))
+            pattern = patterns[i]
+            match = pattern.search(str(value))
             if match is None:
                 passed = False
                 break
@@ -32,16 +36,20 @@ def _filter(filing: Filing) -> bool:
 
 
 def filter_index_record(filters: Mapping[str, str],) -> IndexFilter:
+    patterns: List[Pattern] = []
+    for field_name, filter_text in filters.items():
+        patterns.append(re.compile(filter_text, flags=re.IGNORECASE))
+
     def _filter(record: IndexRecord) -> bool:
         passed = True
-        for field_name in filters:
-            filter_text = filters[field_name]
-            if not record.has_field(field_name):
-                raise FieldNotFound(field_name)
-            value = record.get_field(field_name)
+        for i, f in enumerate(filters):
+            if not record.has_field(f):
+                raise FieldNotFound(f)
+            value = record.get_field(f)
             if value is None:
                 return False
-            match = re.match(filter_text, str(value))
+            pattern = patterns[i]
+            match = pattern.search(str(value))
             if match is None:
                 passed = False
                 break

pyrs990/index.py

@@ -25,6 +25,10 @@ class IndexRecord(NamedTuple):
     that both represent the same organization, as indicated by the EIN
     field. In all cases the two records here should have the same `ein`
     value.
+
+    TODO: EINs seem to be inconsistent, may want to use a heuristic
+    Maybe base it on the organization name + zip code or something?
+    But that would be kind of a pain to validate, so not sure.
     """
 
     annual_record: AnnualRecord