make user data persistent and standardized (#175)

* delete old notebooks

* delete old models dir+artifacts

* delete old pdfestrian test data artifacts

* add todo comments to study ranker model

this is unrelated to the current changes, i just want to commit it somewhere

* delete old citations test data

* delete old dedupe model artifacts

* delete old train deduper script

* docs: update todos deployment readme

* refactor: move ranker classes for readability

* remove old ranker model

* tests: delete old ranker model tests

* fix: use consistent ranker model dir

* refactor: avoid hard-coded ranker col names

* include bigrams in study ranker features

* add sranker prop to get num texts learned

and retrain from scratch every 100 texts learned

* change+hide file handling in study ranker api

* build: add persistent file data volume

* update file storage dirs in app config

* start storing raw citation upload files to storage

* tests: update fs config in conftest

* fix: create/remove fs dirs in cli+api

* ci: add fs root dir to checks env

* use filesystem in study ranker io

* tests: update study ranker init calls

Author: bdewilde

.env.example

@@ -16,11 +16,15 @@ COLANDR_MAIL_USE_TLS=0
 COLANDR_MAIL_USE_SSL=0
 # filesystem storage
 COLANDR_APP_DIR="/app"
-COLANDR_FILESYSTEM_PROTOCOL="file"  # in prod, use "gcs"
-COLANDR_FILESYSTEM_ROOT_DIR="/tmp"  # in prod, use "<GCS_BUCKET>"
-COLANDR_FILESYSTEM_GCS_PROJECT="<GCS_PROJECT>"
-COLANDR_FILESYSTEM_GCS_TOKEN="<GCS_TOKEN>"
-COLANDR_FILESYSTEM_GCS_ENDPOINT_URL="http://colandr-gcs:4443"  # dev-only!
+# file-based filesystem
+COLANDR_FILESYSTEM_PROTOCOL="file"
+COLANDR_FILESYSTEM_ROOT_DIR="/app/data"
+COLANDR_FILESYSTEM_GCS_ENDPOINT_URL="http://colandr-gcs:4443"  # dev-only hack for gcs
+# gcs-based filesystem
+# COLANDR_FILESYSTEM_PROTOCOL="gcs"
+# COLANDR_FILESYSTEM_ROOT_DIR="<GCS_BUCKET>"
+# COLANDR_FILESYSTEM_GCS_PROJECT="<GCS_PROJECT>"
+# COLANDR_FILESYSTEM_GCS_TOKEN="<GCS_TOKEN>"
 # metadata extraction config
 COLANDR_METADATA_THRESHOLD=0.65
 COLANDR_METADATA_INCREASE_TO_RETRAIN=5

.github/workflows/checks.yml

@@ -17,6 +17,7 @@ env:
   COLANDR_DATABASE_URI: "postgresql+psycopg://colandr_app:password@localhost:5432/colandr"
   COLANDR_SECRET_KEY: "colandr_secret_key"
   COLANDR_APP_DIR: "/tmp"
+  COLANDR_FILESYSTEM_ROOT_DIR: "/tmp/data"
   COLANDR_MAIL_USERNAME: "colandr_mail_username"
   COLANDR_MAIL_PASSWORD: "colandr_mail_password"
   # COLANDR_FILESYSTEM_GCS_ENDPOINT_URL: "http://colandr-gcs:4443"

TODO.md

@@ -4,3 +4,5 @@
 - [ ] Improve and extend NLP functionality, just across the board
 - [ ] Enable https everywhere (via [let's encrypt](https://letsencrypt.org/)?)
 - [ ] Consider removing async tasks (and db columns?) for text content vectors
+- [ ] Add script to train a v2 (splink-based) deduper model
+- [ ] Add notebook demonstrating functionality of various NLP models

colandr/api/v1/routes/citation_imports.py

@@ -1,3 +1,4 @@
+import io
 import os
 import pathlib
 import typing as t
@@ -163,9 +164,13 @@ def post(self, files_data, query_data):
                 message=f"received invalid file type for citation import: '{fext}'"
             )
 
+        # unfortunately, we need to read the full file into memory rather than streaming
+        # so we can preprocess the citations and later save the raw file to disk
+        uploaded_data = uploaded_file.stream.read()
+
         try:
             citations_to_insert = _preprocess_citations(
-                uploaded_file.stream, fname, review_id
+                io.BytesIO(uploaded_data), fname, review_id
             )
         except ValueError as e:
             current_app.logger.exception(str(e))
@@ -216,6 +221,22 @@ def post(self, files_data, query_data):
             fname,
             review,
         )
+
+        fs = current_app.extensions["filesystem"]
+        # assign filename based an id, and full path
+        filename = f"{citations_import.id}{fext}"
+        filepath = os.path.join(
+            current_app.config["CITATION_UPLOADS_DIR"],
+            f"review_{review_id:08}",
+            filename,
+        )
+        # make review directory if doesn't already exist
+        fs.makedirs(os.path.dirname(filepath), exist_ok=True)
+        # save content to file on filesystem
+        with fs.open(filepath, mode="wb") as f:
+            # uploaded_file.save(f) may also work well
+            f.write(uploaded_data)
+
         # lastly, don't forget to deduplicate the citations and get their word2vecs
         tasks.get_citations_text_content_vectors.apply_async(
             args=[review_id], countdown=3

colandr/api/v1/routes/fulltext_uploads.py

@@ -155,14 +155,14 @@ def post(self, id, files_data):
                 message=f'invalid fulltext upload file type: "{ext}"'
             )
 
+        fs = current_app.extensions["filesystem"]
         # assign filename based an id, and full path
         filename = f"{id}{ext}"
         filepath = os.path.join(
             current_app.config["FULLTEXT_UPLOADS_DIR"],
             str(study.review_id),
             filename,
         )
-        fs = current_app.extensions["filesystem"]
         # make review directory if doesn't already exist
         fs.makedirs(os.path.dirname(filepath), exist_ok=True)
         # save content to file on filesystem

colandr/api/v1/routes/reviews.py

@@ -73,7 +73,8 @@ def delete(self, id):
         # remove directories on disk for review data
         dirnames = [
             os.path.join(current_app.config["FULLTEXT_UPLOADS_DIR"], str(id)),
-            os.path.join(current_app.config["RANKING_MODELS_DIR"], str(id)),
+            os.path.join(current_app.config["CITATION_UPLOADS_DIR"], f"review_{id:08}"),
+            os.path.join(current_app.config["RANKER_MODELS_DIR"], f"review_{id:08}"),
         ]
         for dirname in dirnames:
             shutil.rmtree(dirname, ignore_errors=True)
@@ -171,7 +172,12 @@ def post(self, json_data):
         # create directories on disk for review data
         dirnames = [
             os.path.join(current_app.config["FULLTEXT_UPLOADS_DIR"], str(review.id)),
-            os.path.join(current_app.config["RANKING_MODELS_DIR"], str(review.id)),
+            os.path.join(
+                current_app.config["CITATION_UPLOADS_DIR"], f"review_{review.id:08}"
+            ),
+            os.path.join(
+                current_app.config["RANKER_MODELS_DIR"], f"review_{review.id:08}"
+            ),
         ]
         for dirname in dirnames:
             try:

colandr/api/v1/routes/studies.py

@@ -395,9 +395,11 @@ def get(self, query_data):
 
             # best option: we have a trained study ranker model
             study_ranker = StudyRanker(
-                review_id, current_app.config["RANKER_MODELS_DIR"]
+                review_id,
+                current_app.config["RANKER_MODELS_DIR"],
+                current_app.extensions["filesystem"],
             )
-            if study_ranker.model_fpath.exists():
+            if study_ranker.model_exists:
                 records = (
                     {
                         "text": (

colandr/cli.py

@@ -95,7 +95,7 @@ def reset_db():
     current_app.logger.warning("resetting database ...")
     db.drop_all()
     db.create_all()
-    for dirkey in ("FULLTEXT_UPLOADS_DIR", "RANKING_MODELS_DIR"):
+    for dirkey in ("FULLTEXT_UPLOADS_DIR", "CITATION_UPLOADS_DIR", "RANKER_MODELS_DIR"):
         shutil.rmtree(current_app.config[dirkey], ignore_errors=True)
         os.makedirs(current_app.config[dirkey], exist_ok=True)
 

colandr/config.py

@@ -85,17 +85,17 @@
         "cache_timeout": 3600,
     },
 }
-FILESYSTEM_ROOT_DIR = os.environ.get("COLANDR_FILESYSTEM_ROOT_DIR", "/tmp")
-FULLTEXT_UPLOADS_DIR = os.path.join(FILESYSTEM_ROOT_DIR, "colandr_data", "fulltexts")
+FILESYSTEM_ROOT_DIR = os.environ.get("COLANDR_FILESYSTEM_ROOT_DIR", "/app/data")
+RANKER_MODELS_DIR = os.path.join(FILESYSTEM_ROOT_DIR, "ranker_models")
+CITATION_UPLOADS_DIR = os.path.join(FILESYSTEM_ROOT_DIR, "citations")
+FULLTEXT_UPLOADS_DIR = os.path.join(FILESYSTEM_ROOT_DIR, "fulltexts")
 ALLOWED_CITATION_UPLOAD_EXTENSIONS = {".ris", ".txt", ".bib", ".csv", ".tsv"}
 ALLOWED_FULLTEXT_UPLOAD_EXTENSIONS = {".txt", ".pdf"}
-# TODO: figure out root dir vs app dir
-COLANDR_APP_DIR = os.environ.get("COLANDR_APP_DIR", "/tmp")
+
+COLANDR_APP_DIR = os.environ.get("COLANDR_APP_DIR", "/app")
 DEDUPE_MODELS_DIR = os.path.join(
     COLANDR_APP_DIR, "colandr_data", "dedupe-v2", "model_202407"
 )
-RANKER_MODELS_DIR = os.path.join(COLANDR_APP_DIR, "colandr_data", "ranker_models")
-RANKING_MODELS_DIR = os.path.join(COLANDR_APP_DIR, "colandr_data", "ranking_models")
 
 # metadata extraction config
 METADATA_THRESHOLD = float(os.environ.get("COLANDR_METADATA_THRESHOLD", "0.65"))

colandr/lib/models/__init__.py

@@ -1,3 +1,2 @@
 from .deduper_v2 import DeduperV2
-from .ranker import Ranker
 from .study_ranker import StudyRanker