feat: Cleanup Review's importer | NPG-7782 (#557)

Mr-Leshiy · web-flow · commit 974dcb8840aa · 2023-09-11T11:55:33.000+03:00
# Description

Cleanup review's importer logic, updated configuration process, load
config from file
diff --git a/src/event-db/stage_data/dev/00002_testfund_ideascale_params.sql b/src/event-db/stage_data/dev/00002_testfund_ideascale_params.sql
@@ -14,6 +14,7 @@ INSERT INTO config (id, id2, id3, value) VALUES (
             "Question 3": "Auditability"
         },
         "stage_ids": [4684, 4685, 4686],
+        "anonymize_start_id": 5000,
         "proposals": {
             "field_mappings": {
                 "proposer_url": ["relevant_link_1", "website__github_repository__or_any_other_relevant_link__", "relevant_link_3"],
diff --git a/utilities/ideascale-importer/Earthfile b/utilities/ideascale-importer/Earthfile
@@ -76,18 +76,9 @@ docker:
 test:
     FROM +build
 
-    COPY ../../src/event-db+docker-compose/docker-compose.yml docker-compose.yml
-    WITH DOCKER \
-        --compose docker-compose.yml \
-        --pull postgres:14 \
-        --load migrations:latest=(../../containers/event-db-migrations+docker --data=test) \
-        --service migrations \
-        --allow-privileged
-        RUN --no-cache \
-            --secret IDEASCALE_EMAIL \
-            --secret IDEASCALE_PASSWORD \
-            --secret IDEASCALE_API_TOKEN \
-            EVENTDB_URL="postgres://catalyst-event-dev:CHANGE_ME@localhost/CatalystEventDev" \
-            IDEASCALE_API_URL="https://temp-cardano-sandbox.ideascale.com" \
-            poetry run pytest
-    END
+    RUN --no-cache \
+        --secret IDEASCALE_EMAIL \
+        --secret IDEASCALE_PASSWORD \
+        --secret IDEASCALE_API_TOKEN \
+        IDEASCALE_API_URL="https://temp-cardano-sandbox.ideascale.com" \
+        poetry run pytest
diff --git a/utilities/ideascale-importer/ideascale_importer/cli/reviews.py b/utilities/ideascale-importer/ideascale_importer/cli/reviews.py
@@ -16,11 +16,6 @@ def import_reviews(
         envvar="IDEASCALE_API_URL",
         help="IdeaScale API URL",
     ),
-    database_url: str = typer.Option(
-        ...,
-        envvar="EVENTDB_URL",
-        help="Postgres database URL"
-    ),
     email: str = typer.Option(
         ...,
         envvar="IDEASCALE_EMAIL",
@@ -31,15 +26,15 @@ def import_reviews(
         envvar="IDEASCALE_PASSWORD",
         help="Ideascale user's password (needs admin access)",
     ),
-    event_id: int = typer.Option(
-        ...,
-        help="Database row id of the event which data will be imported",
-    ),
     api_token: str = typer.Option(
         ...,
         envvar="IDEASCALE_API_TOKEN",
         help="IdeaScale API token"
     ),
+    config_path: str = typer.Option(
+        ..., 
+        help="config file"
+    ),
     allocations_path: str = typer.Option(
         ..., 
         help="allocations file"
@@ -65,11 +60,10 @@ def import_reviews(
     async def inner():
         importer = ReviewsManager(
             ideascale_url=ideascale_url,
-            database_url=database_url,
             email=email,
             password=password,
             api_token=api_token,
-            event_id=event_id,
+            config_path=config_path
         )
 
         try:
@@ -92,30 +86,15 @@ def prepare_allocations(
         envvar="IDEASCALE_API_URL",
         help="IdeaScale API URL",
     ),
-    database_url: str = typer.Option(
-        ...,
-        envvar="EVENTDB_URL",
-        help="Postgres database URL"
-    ),
-    email: str = typer.Option(
-        ...,
-        envvar="IDEASCALE_EMAIL",
-        help="Ideascale user's email address (needs admin access)",
-    ),
-    password: str = typer.Option(
-        ...,
-        envvar="IDEASCALE_PASSWORD",
-        help="Ideascale user's password (needs admin access)",
-    ),
-    event_id: int = typer.Option(
-        ...,
-        help="Database row id of the event which data will be imported",
-    ),
     api_token: str = typer.Option(
         ...,
         envvar="IDEASCALE_API_TOKEN",
         help="IdeaScale API token"
     ),
+    config_path: str = typer.Option(
+        ..., 
+        help="config file"
+    ),
     pas_path: str = typer.Option(
         ..., 
         help="PAs file"
@@ -141,11 +120,8 @@ def prepare_allocations(
     async def inner():
         importer = ReviewsManager(
             ideascale_url=ideascale_url,
-            database_url=database_url,
-            email=email,
-            password=password,
             api_token=api_token,
-            event_id=event_id,
+            config_path=config_path
         )
 
         try:
diff --git a/utilities/ideascale-importer/ideascale_importer/reviews_manager/manager.py b/utilities/ideascale-importer/ideascale_importer/reviews_manager/manager.py
@@ -4,6 +4,7 @@
 from typing import List, Dict
 import pydantic
 import tempfile
+import json
 
 from ideascale_importer import utils
 import ideascale_importer.db
@@ -65,44 +66,32 @@ class ReviewsManager:
     def __init__(
         self,
         ideascale_url,
-        database_url,
-        email,
-        password,
-        event_id,
         api_token,
+        config_path = None,
+        email = None,
+        password = None,
     ):
         self.ideascale_url = ideascale_url
-        self.database_url = database_url
         self.email = email
         self.password = password
-        self.event_id = event_id
         self.api_token = api_token
+        self.config_path = config_path
 
         self.frontend_client = None
         self.db = None
 
     async def connect(self):
-        if self.frontend_client is None:
+        if self.frontend_client is None and (self.email and self.password):
             logger.info("Connecting to the Ideascale frontend")
             self.frontend_client = FrontendClient(self.ideascale_url)
             await self.frontend_client.login(self.email, self.password)
-        if self.db is None:
-            logger.info("Connecting to the database")
-            self.db = await ideascale_importer.db.connect(self.database_url)
 
     async def __load_config(self):
         """Load the configuration setting from the event db."""
 
-        logger.info("Loading ideascale config from the event-db")
-
-        config = ideascale_importer.db.models.Config(row_id=0, id="ideascale", id2=f"{self.event_id}", id3="", value=None)
-        res = await ideascale_importer.db.select(self.db, config,  cond={
-            "id": f"= '{config.id}'",
-            "AND id2": f"= '{config.id2}'"
-            })
-        if len(res) == 0:
-            raise Exception("Cannot find ideascale config in the event-db database")
-        self.config = Config(**res[0].value)
+        logger.info(f"Loading ideascale config from {self.config_path}")
+        config_data = json.load(open(self.config_path))
+        self.config = Config(**config_data)
 
     async def __download_reviews(self, reviews_output_path: str):
         logger.info("Download reviews from Ideascale...")
@@ -113,14 +102,15 @@ async def __download_reviews(self, reviews_output_path: str):
     async def __prepare_allocations(self, pas_path: str, output_path: str):
         logger.info("Prepare allocations for proposal's reviews...")
 
-        self.allocations = await allocate(
-            nr_allocations=self.config.nr_allocations,
+        await allocate(
+            nr_allocations=self.config.allocations_number,
             pas_path=pas_path,
             ideascale_api_key=self.api_token,
             ideascale_api_url=self.ideascale_url,
             stage_ids=self.config.stage_ids,
             challenges_group_id=self.config.campaign_group_id,
             group_id=self.config.group_id,
+            anonymize_start_id=self.config.anonymize_start_id,
             output_path=output_path,
         )
     
@@ -133,7 +123,7 @@ async def __prepare_reviews(self, allocations_path: str, output_path: str):
             allocation_path=allocations_path,
             challenges_group_id=self.config.campaign_group_id,
             questions=self.config.questions,
-            fund=self.event_id,
+            fund=self.config.event_id,
             output_path=output_path
         )
 
@@ -159,14 +149,17 @@ async def import_reviews_run(self, allocations_path: str, output_path: str):
         reviews_dir.cleanup()
 
     async def close(self):
-        await self.frontend_client.close()
+        if self.frontend_client:
+            await self.frontend_client.close()
 
 class Config(pydantic.BaseModel):
     """Represents the available configuration fields."""
 
+    event_id: int
     group_id: int
     campaign_group_id: int
     review_stage_ids: List[int]
     stage_ids: List[int]
-    nr_allocations: List[int]
+    allocations_number: List[int]
     questions: Dict[str, str]
+    anonymize_start_id: int
diff --git a/utilities/ideascale-importer/ideascale_importer/reviews_manager/processing/prepare.py b/utilities/ideascale-importer/ideascale_importer/reviews_manager/processing/prepare.py
@@ -19,7 +19,7 @@ async def allocate(
     stage_ids: List[int],
     nr_allocations: List[int],
     output_path: str,
-    anonymize_start_id: int = 5000,
+    anonymize_start_id: int,
     seed: int = 7,
 ):
     """Run the allocator check."""
@@ -34,17 +34,13 @@ async def _allocate():
         await ideascale.import_com_revs(group_id=group_id, start_id=anonymize_start_id, historic_pas=importer.pas)
         allocator.allocate()
 
-        # This data is not used by our core system, it is only consumed by Ideascale
+        allocator.export_allocations(allocator.source.pas, f"{output_path}/allocations-by-pa.xlsx")
+        groups = allocator.generate_challenges_groups()
+        for group in groups:
+            allocator.export_allocations(group["pas"], f"{output_path}/{group['challenge']}.xlsx")
 
-        # allocator.export_allocations(allocator.source.pas, f"{output_path}/allocations-by-pa.xlsx")
-        # groups = allocator.generate_challenges_groups()
-        # for group in groups:
-        #     allocator.export_allocations(group["pas"], f"{output_path}/{group['challenge']}.xlsx")
-
-        file_name = f"{output_path}/allocations.csv"
-        allocator.export_single_allocations(allocator.allocations, file_name)
+        allocator.export_single_allocations(allocator.allocations, f"{output_path}/allocations.csv")
         allocator.allocation_stats()
-        return file_name
 
     def nr_allocations_map():
         res = {}
diff --git a/utilities/ideascale-importer/ideascale_importer/tests/reviews_test.py b/utilities/ideascale-importer/ideascale_importer/tests/reviews_test.py
@@ -12,18 +12,16 @@ def get_env_var(env_var):
 @pytest.fixture()
 def reviews_manager_mock():
     ideascale_url = get_env_var("IDEASCALE_API_URL")
-    database_url = get_env_var("EVENTDB_URL")
     email = get_env_var("IDEASCALE_EMAIL")
     password = get_env_var("IDEASCALE_PASSWORD")
-    event_id = 0
     api_token = get_env_var("IDEASCALE_API_TOKEN")
+    config_path = "./ideascale_importer/tests/test_data/config.json"
     return ReviewsManager(
             ideascale_url=ideascale_url,
-            database_url=database_url,
             email=email,
             password=password,
             api_token=api_token,
-            event_id=event_id,
+            config_path=config_path
         )
 
 @pytest.mark.asyncio
diff --git a/utilities/ideascale-importer/ideascale_importer/tests/test_data/config.json b/utilities/ideascale-importer/ideascale_importer/tests/test_data/config.json
@@ -0,0 +1,14 @@
+{  
+    "event_id": 0,
+    "group_id": 37429,
+    "review_stage_ids": [171],
+    "allocations_number": [1, 1],
+    "campaign_group_id": 88,
+    "questions": {
+        "Question 1": "Impact / Alignment",
+        "Question 2": "Feasibility",
+        "Question 3": "Auditability"
+    },
+    "stage_ids": [4684, 4685, 4686],
+    "anonymize_start_id": 5000
+}
