feat: Update Review's importer, remove allocation step | NPG-8040 (#545)

# Description

- Removed allocation step from review's importer, it will be moved as a
separate tool later
- Setup test in Earthly

---------

Co-authored-by: Joshua Gilman <[email protected]>

Author: Mr-Leshiy

.github/workflows/ci-test.yml

@@ -31,5 +31,7 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Run tests
+        env:
+          EARTHLY_SECRETS: "IDEASCALE_EMAIL=${{ secrets.IDEASCALE_EMAIL }}, IDEASCALE_PASSWORD=${{ secrets.IDEASCALE_PASSWORD }}, IDEASCALE_API_TOKEN=${{ secrets.IDEASCALE_API_TOKEN }}"
         run: |
           earthly -P --remote-cache=ghcr.io/${{ github.repository }}:cache +test

Earthfile

@@ -123,6 +123,7 @@ ci:
 test:
     BUILD ./src/event-db+test
     BUILD ./src/cat-data-service+test
+    BUILD ./utilities/ideascale-importer+test
 
 tag-workspace:
     ARG SVU_VERSION=1.10.2

utilities/ideascale-importer/Earthfile

@@ -73,6 +73,21 @@ docker:
     SAVE IMAGE --push ${registry}ideascale-importer:$tag
 
 # Run tests
-pytest:
+test:
     FROM +build
-    RUN poetry run pytest -v --doctest-modules
+
+    COPY ../../src/event-db+docker-compose/docker-compose.yml docker-compose.yml
+    WITH DOCKER \
+        --compose docker-compose.yml \
+        --pull postgres:14 \
+        --load migrations:latest=(../../containers/event-db-migrations+docker --data=test) \
+        --service migrations \
+        --allow-privileged
+        RUN --no-cache \
+            --secret IDEASCALE_EMAIL \
+            --secret IDEASCALE_PASSWORD \
+            --secret IDEASCALE_API_TOKEN \
+            EVENTDB_URL="postgres://catalyst-event-dev:CHANGE_ME@localhost/CatalystEventDev" \
+            IDEASCALE_API_URL="https://temp-cardano-sandbox.ideascale.com" \
+            poetry run pytest
+    END

utilities/ideascale-importer/ideascale_importer/cli/reviews.py

@@ -40,9 +40,9 @@ def import_reviews(
         envvar="IDEASCALE_API_TOKEN",
         help="IdeaScale API token"
     ),
-    pa_path: str = typer.Option(
+    allocations_path: str = typer.Option(
         ..., 
-        help="PAs file"
+        help="allocations file"
     ),
     output_path: str = typer.Option(
         ...,
@@ -70,7 +70,7 @@ async def inner():
             password=password,
             api_token=api_token,
             event_id=event_id,
-            pa_path=pa_path,
+            allocations_path=allocations_path,
             output_path=output_path
         )
 

utilities/ideascale-importer/ideascale_importer/reviews_importer/importer.py

@@ -29,14 +29,14 @@ async def login(self,  email, password):
             'rememberMe': 'true',
         }
 
-        await self.inner.post(f"{login}", data=data)
+        res = await self.inner.post(f"{login}", data=data)
 
     async def download_reviews(self, reviews_path, review_stage_ids):
         async def download_file(self, review_stage_id):
             export_endpoint = "/a/admin/workflow/survey-tools/assessment/report/statistic/export/assessment-details/"
             file_name = f"{reviews_path}/{review_stage_id}.xlsx"
 
-            content = await self.inner.get(f"{export_endpoint}{review_stage_id}") 
+            content = await self.inner.get(f"{export_endpoint}{review_stage_id}")
             tree = html.fromstring(content)
 
             # we are looking for '<div class="card panel export-result-progress" data-features="refresh-processing-item" data-processing-item-id="15622">'
@@ -70,7 +70,7 @@ def __init__(
         password,
         event_id,
         api_token,
-        pa_path,
+        allocations_path,
         output_path,
     ):
         self.ideascale_url = ideascale_url
@@ -80,11 +80,10 @@ def __init__(
         self.event_id = event_id
         self.api_token = api_token
 
-        self.pa_path = pa_path
+        self.allocations_path = allocations_path
         self.output_path = output_path
 
         self.reviews_dir = tempfile.TemporaryDirectory()
-        self.allocations_dir = tempfile.TemporaryDirectory()
 
         self.frontend_client = None
         self.db = None
@@ -117,19 +116,20 @@ async def download_reviews(self):
 
         self.reviews = await self.frontend_client.download_reviews(self.reviews_dir.name, self.config.review_stage_ids)
 
-    async def prepare_allocations(self):
-        logger.info("Prepare allocations for proposal's reviews...")
-
-        self.allocations_path = await allocate(
-            nr_allocations=self.config.nr_allocations,
-            pas_path=self.pa_path,
-            ideascale_api_key=self.api_token,
-            ideascale_api_url=self.ideascale_url,
-            stage_ids=self.config.stage_ids,
-            challenges_group_id=self.config.campaign_group_id,
-            group_id=self.config.group_id,
-            output_path=self.allocations_dir.name,
-        )
+    # This code will be moved as a separate cli command
+    # async def prepare_allocations(self):
+    #     logger.info("Prepare allocations for proposal's reviews...")
+
+    #     self.allocations_path = await allocate(
+    #         nr_allocations=self.config.nr_allocations,
+    #         pas_path=self.pa_path,
+    #         ideascale_api_key=self.api_token,
+    #         ideascale_api_url=self.ideascale_url,
+    #         stage_ids=self.config.stage_ids,
+    #         challenges_group_id=self.config.campaign_group_id,
+    #         group_id=self.config.group_id,
+    #         output_path=self.output_path,
+    #     )
 
     async def prepare_reviews(self):
         logger.info("Prepare proposal's reviews...")
@@ -155,12 +155,10 @@ async def run(self):
         await self.load_config()
 
         await self.download_reviews()
-        await self.prepare_allocations()
         await self.prepare_reviews()
 
     async def close(self):
         self.reviews_dir.cleanup()
-        self.allocations_dir.cleanup()
         await self.frontend_client.close()
 
 class Config(pydantic.BaseModel):

utilities/ideascale-importer/ideascale_importer/tests/pa.csv

@@ -1,27 +1,33 @@
 import pytest
-import tempfile
+import os
 
 from ideascale_importer.reviews_importer.importer import Importer
 
-@pytest.fixture
-def reviews_importer_mock():
-    ideascale_url = "https://temp-cardano-sandbox.ideascale.com"
-    database_url = "postgres://catalyst-event-dev:CHANGE_ME@localhost/CatalystEventDev"
-    email = ""
-    password = ""
+def get_env_var(env_var):
+    if os.environ.get(env_var):
+        return os.environ.get(env_var)
+    else:
+        raise Exception(f"could not find {env_var} env var")
+
+@pytest.fixture(scope="session")
+def reviews_importer_mock(tmpdir_factory):
+    ideascale_url = get_env_var("IDEASCALE_API_URL")
+    database_url = get_env_var("EVENTDB_URL")
+    email = get_env_var("IDEASCALE_EMAIL")
+    password = get_env_var("IDEASCALE_PASSWORD")
     event_id = 0
-    api_token = ""
-    pa_path = "./pa.csv"
-    output_path = tempfile.TemporaryDirectory()
+    api_token = get_env_var("IDEASCALE_API_TOKEN")
+    allocations_path = "./ideascale_importer/tests/test_data/allocations.csv"
+    output_path = tmpdir_factory.mktemp("output")
     return Importer(
             ideascale_url=ideascale_url,
             database_url=database_url,
             email=email,
             password=password,
             api_token=api_token,
             event_id=event_id,
-            pa_path=pa_path,
-            output_path=output_path.name
+            allocations_path=allocations_path,
+            output_path=output_path
         )
 
 @pytest.mark.asyncio

utilities/ideascale-importer/ideascale_importer/tests/reviews_importer_test.py

@@ -0,0 +1,2 @@
+pa_anon_id,pa_email,pa_level,proposal_id,proposal_url,proposal_title,proposal_challenge_id
+0,[email protected],0,109581,https://temp-cardano-sandbox.ideascale.com/c/idea/109581,Proposal 1,394