Move csv resources out of static folder

Should only be used to serve frontend assets and was causing issues with migrations on CI test build

Author: paulapreuss

app/dashboard/helpers.py

@@ -4,13 +4,13 @@
 import csv
 from copy import deepcopy
 
-from django.contrib.staticfiles.storage import staticfiles_storage
 from django.utils.translation import gettext_lazy as _
 from django.db.models import Value, Q, F, Case, When
 from django.db.models.functions import Concat, Replace
 from django.templatetags.static import static
 from numbers import Number
 
+from epa.settings import RESOURCES_DIR
 from projects.models import Viewer, Project
 import pickle
 from django.conf import settings as django_settings
@@ -93,10 +93,10 @@
 KPI_PARAMETERS = {}
 KPI_PARAMETERS_ASSETS = {}
 
-if os.path.exists(staticfiles_storage.path("MVS_kpis_list.csv")) is True:
-    with open(
-        staticfiles_storage.path("MVS_kpis_list.csv"), encoding="utf-8"
-    ) as csvfile:
+MVS_KPIS_LIST_PATH = RESOURCES_DIR / "MVS_kpis_list.csv"
+
+if MVS_KPIS_LIST_PATH.exists():
+    with open(MVS_KPIS_LIST_PATH, encoding="utf-8") as csvfile:
         csvreader = csv.reader(csvfile, delimiter=",", quotechar='"')
         CARRIER_PLACEHOLDER = ":carrier:"
         for i, row in enumerate(csvreader):
@@ -132,9 +132,7 @@
                 #                 {"name": _(verbose), "id": label, "unit": _(unit)}
                 #             )
 
-    with open(
-        staticfiles_storage.path("MVS_kpis_list.csv"), encoding="utf-8"
-    ) as csvfile:
+    with open(MVS_KPIS_LIST_PATH, encoding="utf-8") as csvfile:
         csvreader = csv.reader(csvfile, delimiter=",", quotechar='"')
         for i, row in enumerate(csvreader):
             if i == 0:

app/epa/settings.py

@@ -41,6 +41,9 @@
     "django.contrib.staticfiles.finders.AppDirectoriesFinder",
 ]
 
+# Resources dir (containing helper csv files)
+RESOURCES_DIR = BASE_DIR / "resources"
+
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = env.bool("DEBUG", default=False)
 PROD_ENV = env.bool("PROD_ENV", default=False)

app/projects/helpers.py

@@ -5,10 +5,11 @@
 import csv
 from openpyxl import load_workbook
 from django import forms
-from django.contrib.staticfiles.storage import staticfiles_storage
 from django.core.exceptions import ValidationError
 from django.utils.translation import gettext_lazy as _
 from django.utils.html import html_safe
+
+from epa.settings import RESOURCES_DIR
 from projects.dtos import convert_to_dto
 from projects.models import Timeseries, AssetType
 from projects.constants import MAP_MVS_EPA
@@ -19,11 +20,11 @@
 TS_MANUAL_TYPE = "manual"
 TS_INPUT_TYPES = (TS_MANUAL_TYPE, TS_SELECT_TYPE, TS_UPLOAD_TYPE)
 
+MVS_PARAMETERS_LIST_PATH = RESOURCES_DIR / "MVS_parameters_list.csv"
+
 PARAMETERS = {}
-if os.path.exists(staticfiles_storage.path("MVS_parameters_list.csv")) is True:
-    with open(
-        staticfiles_storage.path("MVS_parameters_list.csv"), encoding="utf-8"
-    ) as csvfile:
+if MVS_PARAMETERS_LIST_PATH.exists():
+    with open(MVS_PARAMETERS_LIST_PATH, encoding="utf-8") as csvfile:
         csvreader = csv.reader(csvfile, delimiter=",", quotechar='"')
         for i, row in enumerate(csvreader):
             if i == 0:

app/projects/management/commands/update_assettype.py

@@ -1,6 +1,8 @@
 from django.core.management.base import BaseCommand, CommandError
 import pandas as pd
 import json
+
+from epa.settings import RESOURCES_DIR
 from projects.models import *
 
 
@@ -15,7 +17,7 @@ def add_arguments(self, parser):
     def handle(self, *args, **options):
         update_assets = options["update"]
 
-        df = pd.read_csv("static/assettypes_list.csv")
+        df = pd.read_csv(RESOURCES_DIR / "assettypes_list.csv")
         assets = df.to_dict(orient="records")
 
         for asset_params in assets:

app/projects/management/commands/update_valuetype.py

@@ -1,5 +1,7 @@
 from django.core.management.base import BaseCommand, CommandError
 import pandas as pd
+
+from epa.settings import RESOURCES_DIR
 from projects.models import *
 
 
@@ -14,7 +16,7 @@ def add_arguments(self, parser):
     def handle(self, *args, **options):
         update_valuetypes = options["update"]
 
-        df = pd.read_csv("static/valuetypes_list.csv")
+        df = pd.read_csv(RESOURCES_DIR / "valuetypes_list.csv")
         valuetypes = df.to_dict(orient="records")
         for vt_params in valuetypes:
             qs = ValueType.objects.filter(type=vt_params["type"])