feat: Review's allocation generation | NPG-8066 (#552)

# Description

- Added new cli command for allocations generation

## Type of change

Please delete options that are not relevant.

- [ ] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] This change requires a documentation update

## How Has This Been Tested?

- [x] test_allocations_generator

## Checklist

- [ ] My code follows the style guidelines of this project
- [ ] I have performed a self-review of my code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [ ] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my
feature works
- [ ] New and existing unit tests pass locally with my changes
- [ ] Any dependent changes have been merged and published in downstream
modules

Author: Mr-Leshiy

src/event-db/stage_data/dev/00002_testfund_ideascale_params.sql

@@ -6,7 +6,7 @@ INSERT INTO config (id, id2, id3, value) VALUES (
     '{  
         "group_id": 37429,
         "review_stage_ids": [171],
-        "nr_allocations": [0, 0],
+        "nr_allocations": [1, 1],
         "campaign_group_id": 88,
         "questions": {
             "Question 1": "Impact / Alignment",

utilities/ideascale-importer/ideascale_importer/cli/reviews.py

@@ -4,12 +4,12 @@
 from typing import List
 from loguru import logger
 
-from ideascale_importer.reviews_importer.importer import Importer
+from ideascale_importer.reviews_manager.manager import ReviewsManager
 from ideascale_importer.utils import configure_logger
 
 app = typer.Typer(add_completion=False)
 
-@app.command(name="import")
+@app.command(name="import-reviews")
 def import_reviews(
     ideascale_url: str = typer.Option(
         ...,
@@ -59,24 +59,101 @@ def import_reviews(
         help="Log format",
     )
 ):
-    """Import all reviews data from IdeaScale for a given funnel."""
+    """Import all reviews data from IdeaScale for the provided event."""
     configure_logger(log_level, log_format)
 
     async def inner():
-        importer = Importer(
+        importer = ReviewsManager(
             ideascale_url=ideascale_url,
             database_url=database_url,
             email=email,
             password=password,
             api_token=api_token,
             event_id=event_id,
-            allocations_path=allocations_path,
-            output_path=output_path
         )
 
         try:
             await importer.connect()
-            await importer.run()
+            await importer.import_reviews_run(
+                allocations_path=allocations_path,
+                output_path=output_path
+                )
+            await importer.close()
+        except Exception as e:
+            traceback.print_exc()
+            logger.error(e)
+
+    asyncio.run(inner())
+
+@app.command(name="prepare-allocations")
+def prepare_allocations(
+    ideascale_url: str = typer.Option(
+        ...,
+        envvar="IDEASCALE_API_URL",
+        help="IdeaScale API URL",
+    ),
+    database_url: str = typer.Option(
+        ...,
+        envvar="EVENTDB_URL",
+        help="Postgres database URL"
+    ),
+    email: str = typer.Option(
+        ...,
+        envvar="IDEASCALE_EMAIL",
+        help="Ideascale user's email address (needs admin access)",
+    ),
+    password: str = typer.Option(
+        ...,
+        envvar="IDEASCALE_PASSWORD",
+        help="Ideascale user's password (needs admin access)",
+    ),
+    event_id: int = typer.Option(
+        ...,
+        help="Database row id of the event which data will be imported",
+    ),
+    api_token: str = typer.Option(
+        ...,
+        envvar="IDEASCALE_API_TOKEN",
+        help="IdeaScale API token"
+    ),
+    pas_path: str = typer.Option(
+        ..., 
+        help="PAs file"
+    ),
+    output_path: str = typer.Option(
+        ...,
+        help="output path"
+    ),
+    log_level: str = typer.Option(
+        "info",
+        envvar="REVIEWS_LOG_LEVEL",
+        help="Log level",
+    ),
+    log_format: str = typer.Option(
+        "json",
+        envvar="REVIEWS_LOG_FORMAT",
+        help="Log format",
+    )
+):
+    """Prepare allocations for the provided event."""
+    configure_logger(log_level, log_format)
+
+    async def inner():
+        importer = ReviewsManager(
+            ideascale_url=ideascale_url,
+            database_url=database_url,
+            email=email,
+            password=password,
+            api_token=api_token,
+            event_id=event_id,
+        )
+
+        try:
+            await importer.connect()
+            await importer.generate_allocations_run(
+                pas_path=pas_path,
+                output_path=output_path
+                )
             await importer.close()
         except Exception as e:
             traceback.print_exc()

…le_importer/reviews_importer/importer.py …cale_importer/reviews_manager/manager.pyutilities/ideascale-importer/ideascale_importer/reviews_importer/importer.py renamed to utilities/ideascale-importer/ideascale_importer/reviews_manager/manager.py utilities/ideascale-importer/ideascale_importer/reviews_importer/importer.py renamed to utilities/ideascale-importer/ideascale_importer/reviews_manager/manager.py

@@ -29,7 +29,7 @@ async def login(self,  email, password):
             'rememberMe': 'true',
         }
 
-        res = await self.inner.post(f"{login}", data=data)
+        await self.inner.post(f"{login}", data=data)
 
     async def download_reviews(self, reviews_path, review_stage_ids):
         async def download_file(self, review_stage_id):
@@ -61,7 +61,7 @@ async def download_file(self, review_stage_id):
             files.append(await download_file(self, review_stage_id))
         return files
 
-class Importer:
+class ReviewsManager:
     def __init__(
         self,
         ideascale_url,
@@ -70,8 +70,6 @@ def __init__(
         password,
         event_id,
         api_token,
-        allocations_path,
-        output_path,
     ):
         self.ideascale_url = ideascale_url
         self.database_url = database_url
@@ -80,15 +78,19 @@ def __init__(
         self.event_id = event_id
         self.api_token = api_token
 
-        self.allocations_path = allocations_path
-        self.output_path = output_path
-
-        self.reviews_dir = tempfile.TemporaryDirectory()
-
         self.frontend_client = None
         self.db = None
 
-    async def load_config(self):
+    async def connect(self):
+        if self.frontend_client is None:
+            logger.info("Connecting to the Ideascale frontend")
+            self.frontend_client = FrontendClient(self.ideascale_url)
+            await self.frontend_client.login(self.email, self.password)
+        if self.db is None:
+            logger.info("Connecting to the database")
+            self.db = await ideascale_importer.db.connect(self.database_url)
+
+    async def __load_config(self):
         """Load the configuration setting from the event db."""
 
         logger.info("Loading ideascale config from the event-db")
@@ -102,63 +104,61 @@ async def load_config(self):
             raise Exception("Cannot find ideascale config in the event-db database")
         self.config = Config(**res[0].value)
 
-    async def connect(self):
-        if self.frontend_client is None:
-            logger.info("Connecting to the Ideascale frontend")
-            self.frontend_client = FrontendClient(self.ideascale_url)
-            await self.frontend_client.login(self.email, self.password)
-        if self.db is None:
-            logger.info("Connecting to the database")
-            self.db = await ideascale_importer.db.connect(self.database_url)
-
-    async def download_reviews(self):
-        logger.info("Dowload reviews from Ideascale...")
+    async def __download_reviews(self, reviews_output_path: str):
+        logger.info("Download reviews from Ideascale...")
 
-        self.reviews = await self.frontend_client.download_reviews(self.reviews_dir.name, self.config.review_stage_ids)
+        self.reviews = await self.frontend_client.download_reviews(reviews_output_path, self.config.review_stage_ids)
 
     # This code will be moved as a separate cli command
-    # async def prepare_allocations(self):
-    #     logger.info("Prepare allocations for proposal's reviews...")
-
-    #     self.allocations_path = await allocate(
-    #         nr_allocations=self.config.nr_allocations,
-    #         pas_path=self.pa_path,
-    #         ideascale_api_key=self.api_token,
-    #         ideascale_api_url=self.ideascale_url,
-    #         stage_ids=self.config.stage_ids,
-    #         challenges_group_id=self.config.campaign_group_id,
-    #         group_id=self.config.group_id,
-    #         output_path=self.output_path,
-    #     )
+    async def __prepare_allocations(self, pas_path: str, output_path: str):
+        logger.info("Prepare allocations for proposal's reviews...")
+
+        self.allocations = await allocate(
+            nr_allocations=self.config.nr_allocations,
+            pas_path=pas_path,
+            ideascale_api_key=self.api_token,
+            ideascale_api_url=self.ideascale_url,
+            stage_ids=self.config.stage_ids,
+            challenges_group_id=self.config.campaign_group_id,
+            group_id=self.config.group_id,
+            output_path=output_path,
+        )
 
-    async def prepare_reviews(self):
+    async def __prepare_reviews(self, allocations_path: str, output_path: str):
         logger.info("Prepare proposal's reviews...")
         await process_ideascale_reviews(
             ideascale_xlsx_path=self.reviews,
             ideascale_api_url=self.ideascale_url,
             ideascale_api_key=self.api_token,
-            allocation_path=self.allocations_path,
+            allocation_path=allocations_path,
             challenges_group_id=self.config.campaign_group_id,
             questions=self.config.questions,
             fund=self.event_id,
-            output_path=self.output_path
+            output_path=output_path
         )
 
-    async def import_reviews(self):
-        logger.info("Import reviews into Event db")
+    async def __import_reviews_to_service(self):
+        logger.info("Import reviews into cat data service")
+
+    async def generate_allocations_run(self, pas_path: str, output_path: str):
+        """Run the Allocations generation."""
+
+        await self.__load_config()
+        await self.__prepare_allocations(pas_path=pas_path, output_path=output_path)
 
-    async def run(self):
-        """Run the importer."""
+    async def import_reviews_run(self, allocations_path: str, output_path: str):
+        """Run the reviews importer."""
         if self.frontend_client is None:
             raise Exception("Not connected to the ideascale")
 
-        await self.load_config()
+        await self.__load_config()
+        reviews_dir = tempfile.TemporaryDirectory()
+        await self.__download_reviews(reviews_output_path=reviews_dir.name)
+        await self.__prepare_reviews(allocations_path=allocations_path, output_path=output_path)
 
-        await self.download_reviews()
-        await self.prepare_reviews()
+        reviews_dir.cleanup()
 
     async def close(self):
-        self.reviews_dir.cleanup()
         await self.frontend_client.close()
 
 class Config(pydantic.BaseModel):

…/reviews_importer/processing/__init__.py …r/reviews_manager/processing/__init__.pyutilities/ideascale-importer/ideascale_importer/reviews_importer/processing/__init__.py renamed to utilities/ideascale-importer/ideascale_importer/reviews_manager/processing/__init__.py utilities/ideascale-importer/ideascale_importer/reviews_importer/processing/__init__.py renamed to utilities/ideascale-importer/ideascale_importer/reviews_manager/processing/__init__.py

@@ -29,14 +29,14 @@ def __init__(self, importer: Importer, nr_allocations: dict, seed: int, ideascal
     def __get_relevant_proposals(
         self, challenge_ids: List[int], source: List[models.Proposal], level: int, times_check: bool = True
     ) -> List[models.Proposal]:
+        if not challenge_ids:
+            return []
+
         times_picked = [len(p.allocations) for p in source]
         min_times_picked = min(times_picked)
         max_times_picked = max(times_picked)
         for x in range(min_times_picked, max_times_picked + 1):
-            if len(challenge_ids) > 0:
-                proposals = [p for p in source if (p.challenge_id in challenge_ids and (len(p.allocations) <= x and times_check))]
-            else:
-                proposals = [p for p in source if (len(p.allocations) <= x and times_check)]
+            proposals = [p for p in source if (p.challenge_id in challenge_ids and (len(p.allocations) <= x and times_check))]
             if len(proposals) >= self.nr_allocations[level]:
                 return proposals
         return proposals
@@ -48,17 +48,28 @@ def __generate_pa_challenges(self, pa: models.Pa, authors: dict) -> List[int]:
         If level 1, use preferences as base challenges.
         If level 0, use all challenges as base challenges.
         Filter out challenges as author.
-        If no challenges remain and pa is author in some challenge, use all challenges as base and filter the one as author.
+        If no challenges remain, use all challenges as base and filter the one as author.
         """
         if pa.id in authors.keys():
             challenges_as_author = authors[pa.id]
         else:
             challenges_as_author = []
-        base_challenges = pa.challenge_ids if pa.level == 1 else []
-        challenges_ids = list(filter(lambda c: c not in challenges_as_author, base_challenges))
-        if len(challenges_ids) == 0 and len(challenges_as_author) > 0:
-            challenges_ids = list(filter(lambda c: c not in challenges_as_author, [c.id for c in self.source.challenges]))
-        return challenges_ids
+        
+        base_challenges_ids = [c.id for c in self.source.challenges]
+        # If level 0, use all challenges as base challenges.
+        if pa.level == 0:
+            # Filter out challenges as author.
+            return list(filter(lambda c: c not in challenges_as_author, base_challenges_ids))
+        #  If level 1, use preferences as base challenges.
+        if pa.level == 1:
+            # Filter out challenges as author.
+            challenges_ids = list(filter(lambda c: c not in challenges_as_author, pa.challenge_ids))
+            # If no challenges remain, use all challenges as base and filter the one as author.
+            if len(challenges_ids) == 0:
+                return list(filter(lambda c: c not in challenges_as_author, base_challenges_ids))
+            return challenges_ids
+
+        raise Exception(f"Invalid pa level: {pa.level}")
 
     def __generate_authors_map(self):
         logger.info("Generating authors map...")