feat(ideascale): Update fields for F11 params, saves JSON artifacts | NPG-000 (#661)

# Description

Updates ideascale importer to use F11.
Saves JSON artifacts to output directory.

## Type of change

Please delete options that are not relevant.

- [ ] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] This change requires a documentation update

## How Has This Been Tested?

- [x] Run `ideascale-importer ideascale import-all` for single network
- [x] Run `ideascale-importer ideascale import-all` for `mainnet` and
`preprod`

## Checklist

- [x] My code follows the style guidelines of this project
- [x] I have performed a self-review of my code
- [x] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my
feature works
- [x] New and existing unit tests pass locally with my changes
- [ ] Any dependent changes have been merged and published in downstream
modules

Author: saibatizoku

utilities/ideascale-importer/ideascale_importer/cli/ideascale.py

@@ -1,7 +1,8 @@
 """IdeaScale CLI commands."""
 
 import asyncio
-from typing import Optional, List
+from pathlib import Path
+from typing import Optional
 import typer
 
 from ideascale_importer.ideascale.client import Client
@@ -39,6 +40,9 @@ def import_all(
         envvar="IDEASCALE_API_URL",
         help="IdeaScale API URL",
     ),
+    output_dir: Optional[str] = typer.Option(
+        default=None, envvar="IDEASCALE_OUTPUT_DIR", help="Output directory for generated files"
+    ),
 ):
     """Import all event data from IdeaScale for a given event."""
     configure_logger(log_level, log_format)
@@ -47,13 +51,23 @@ async def inner(
         event_id: int,
         proposals_scores_csv_path: Optional[str],
         ideascale_api_url: str,
+        output_dir: Optional[str]
     ):
+        # check if output_dir path exists, or create otherwise
+        if output_dir is None:
+            logger.info("No output directory was defined.")
+        else:
+            output_dir = Path(output_dir)
+            output_dir.mkdir(exist_ok=True, parents=True)
+            logger.info(f"Output directory for artifacts: {output_dir}")
+
         importer = Importer(
             api_token,
             database_url,
             event_id,
             proposals_scores_csv_path,
             ideascale_api_url,
+            output_dir
         )
 
         try:
@@ -63,4 +77,4 @@ async def inner(
         except Exception as e:
             logger.error(e)
 
-    asyncio.run(inner(event_id, proposals_scores_csv, ideascale_api_url))
+    asyncio.run(inner(event_id, proposals_scores_csv, ideascale_api_url, output_dir))

utilities/ideascale-importer/ideascale_importer/cli/snapshot.py

@@ -25,7 +25,7 @@ def import_snapshot(
     catalyst_toolbox_path: str = typer.Option(
         default="catalyst-toolbox", envvar="CATALYST_TOOLBOX_PATH", help="Path to the catalyst-toolbox"
     ),
-    gvc_api_url: str = typer.Option(..., envvar="GVC_API_URL", help="URL of the GVC API"),
+    gvc_api_url: str = typer.Option(default="", envvar="GVC_API_URL", help="DEPRECATED. URL of the GVC API"),
     raw_snapshot_file: str = typer.Option(
         None,
         help=(
@@ -102,7 +102,6 @@ async def inner():
                 network_ids=network_ids,
                 snapshot_tool_path=snapshot_tool_path,
                 catalyst_toolbox_path=catalyst_toolbox_path,
-                gvc_api_url=gvc_api_url,
                 raw_snapshot_file=raw_snapshot_file,
                 ssh_config=ssh_config,
             )

utilities/ideascale-importer/ideascale_importer/db/__init__.py

@@ -64,6 +64,7 @@ async def insert(conn: asyncpg.Connection, model: Model) -> Any:
         return ret[0]
     return None
 
+
 async def select(conn: asyncpg.Connection, model: Model, cond: Dict[str, str] = {}) -> List[Any]:
     """Select a single model."""
 
@@ -77,7 +78,7 @@ async def select(conn: asyncpg.Connection, model: Model, cond: Dict[str, str] =
         SELECT {cols_str}
         FROM {model.table()}
         {f' WHERE {cond_str}' if cond_str else ' '}
-    """.strip()  
+    """.strip()
 
     result = await conn.fetch(stmt_template)
 
@@ -123,9 +124,13 @@ async def upsert_many(
     pre_update_set_str = ",".join([f"{col} = {val}" for col, val in pre_update_cols.items()])
     pre_update_cond_str = " ".join([f"{col} {cond}" for col, cond in pre_update_cond.items()])
 
-    pre_update_template = f"""
+    pre_update_template = (
+        f"""
         WITH updated AS ({ f"UPDATE {models[0].table()} SET {pre_update_set_str} {f' WHERE {pre_update_cond_str}' if pre_update_cond_str else ' '}" })
-        """.strip() if pre_update_set_str else " "     
+        """.strip()
+        if pre_update_set_str
+        else " "
+    )
 
     stmt_template = f"""
         {pre_update_template}
@@ -172,6 +177,27 @@ async def event_exists(conn: asyncpg.Connection, id: int) -> bool:
     return row is not None
 
 
+class EventThesholdNotFound(Exception):
+    """Raised when the event's voting power threshold is not found."""
+
+    ...
+
+
+async def event_threshold(conn: asyncpg.Connection, row_id: int) -> int:
+    """Fetch the event's voting power threshold in ADA."""
+    res = await conn.fetchrow("SELECT voting_power_threshold FROM event WHERE row_id = $1", row_id)
+    if res is None:
+        raise EventThesholdNotFound()
+    threshold = int(res["voting_power_threshold"]/1000000)
+    return threshold
+
+async def update_event_description(conn: asyncpg.Connection, row_id: int, description: str):
+    """Update the event description.
+
+    NOTE: this field includes a JSON string used to inform other services."""
+    await conn.execute(f"UPDATE event SET description = '{description}' WHERE row_id = $1", row_id)
+
+
 class VoteOptionsNotFound(Exception):
     """Raised when a vote option is not found."""
 

utilities/ideascale-importer/ideascale_importer/db/models.py

@@ -219,6 +219,7 @@ def table() -> str:
         """Return the name of the table that this model is stored in."""
         return "snapshot"
 
+
 @dataclass
 class Config(Model):
     """Represents a database config."""

utilities/ideascale-importer/ideascale_importer/ideascale/artifacts.py

@@ -0,0 +1,94 @@
+from typing import Optional
+from ideascale_importer.db.models import Objective, Proposal
+from pydantic import BaseModel
+
+
+class ProposalJson(BaseModel):
+    """A proposal in JSON used for output artifacts."""
+
+    category_name: str
+    chain_vote_options: str
+    challenge_id: str
+    challenge_type: str
+    chain_vote_type: str
+    internal_id: str
+    proposal_funds: str
+    proposal_id: str
+    proposal_impact_score: str
+    proposal_summary: str
+    proposal_title: str
+    proposal_url: str
+    proposer_email: Optional[str] = None
+    proposer_name: Optional[str] = None
+    proposer_relevant_experience: Optional[str] = None
+    proposer_url: Optional[str] = None
+    proposal_solution: Optional[str] = None
+    files_url: str
+
+
+class ChallengesJson(BaseModel):
+    id: str
+    internal_id: int
+    title: str
+    challenge_type: str
+    challenge_url: str
+    description: str
+    fund_id: str
+    rewards_total: str
+    proposers_rewards: str
+
+
+def objective_to_challenge_json(obj: Objective, ideascale_url: str, idx: int = 0) -> ChallengesJson:
+    c_url = f"{ideascale_url}/c/campaigns/{obj.id}/"
+    return ChallengesJson.model_validate(
+        {
+            "id": f"{idx}",
+            "internal_id": obj.id,
+            "title": obj.title,
+            "challenge_type": obj.category.removeprefix("catalyst-"),
+            "challenge_url": c_url,
+            "description": obj.description,
+            "fund_id": f"{obj.event}",
+            "rewards_total": f"{obj.rewards_total}",
+            "proposers_rewards": f"{obj.proposers_rewards}",
+        }
+    )
+
+
+def json_from_proposal(prop: Proposal, challenge: ChallengesJson, fund_id: int, idx: int = 0) -> ProposalJson:
+    if prop.proposer_relevant_experience == "":
+        experience = None
+    else:
+        experience = prop.proposer_relevant_experience
+    if prop.extra is not None:
+        solution = prop.extra.get("solution", None)
+    else:
+        solution = None
+    return ProposalJson.model_validate(
+        {
+            "category_name": f"Fund {fund_id}",
+            "chain_vote_options": "blank,yes,no",
+            "challenge_id": challenge.id,
+            "challenge_type": challenge.challenge_type,
+            "chain_vote_type": "private",
+            "internal_id": f"{idx}",
+            "proposal_funds": f"{prop.funds}",
+            "proposal_id": f"{prop.id}",
+            "proposal_impact_score": f"{prop.impact_score}",
+            "proposal_summary": prop.summary,
+            "proposal_title": prop.title,
+            "proposal_url": prop.url,
+            "proposer_name": prop.proposer_name,
+            "proposer_relevant_experience": experience,
+            "proposal_solution": solution,
+            "files_url": prop.files_url,
+        }
+    )
+
+
+class FundsJson(BaseModel):
+    """Current Fund (Event) information in JSON used for output artifacts."""
+    id: int
+    goal: str
+    threshold: int
+    rewards_info: str = ""

utilities/ideascale-importer/ideascale_importer/ideascale/client.py

@@ -209,6 +209,16 @@ async def funnel(self, funnel_id: int) -> Funnel:
         res = await self._get(f"/a/rest/v1/funnels/{funnel_id}")
         return Funnel.model_validate(res)
 
+    async def event_themes(self, campaign_id: int, themes_custom_key: str) -> List[str]:
+        """Get the list of themes for this Fund,by IdeaScale `campaign_id`."""
+        try:
+            res = await self._get(f"/a/rest/v1/customFields/idea/campaigns/{campaign_id}")
+            themes_fields = [f for f in res if f["key"] and f["key"] == themes_custom_key]
+            themes = themes_fields[0]["options"].split("\r\n")
+            return themes
+        except Exception as e:
+            raise Exception(f"Unable to fetch themes: {e}")
+
     async def _get(self, path: str) -> Mapping[str, Any] | Iterable[Mapping[str, Any]]:
         """Execute a GET request on IdeaScale API."""
         headers = {"api_token": self.api_token}