Update the method to retrieve the list of modified packages

Resolves: AlmaLinux/build-system#438

Author: javihernandez

alws/crud/build.py

@@ -1,6 +1,8 @@
 import logging
 import typing
 
+import redis.asyncio as aioredis
+
 import sqlalchemy
 from sqlalchemy import delete
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -238,11 +240,13 @@ async def generate_query(count=False):
 
 
 async def get_module_preview(
+    redis: aioredis.client.Redis,
     platform: models.Platform,
     flavors: typing.List[models.PlatformFlavour],
     module_request: build_schema.ModulePreviewRequest,
 ) -> build_schema.ModulePreview:
     refs, modules, enabled_modules = await build_schema.get_module_refs(
+        redis=redis,
         task=module_request.ref,
         platform=platform,
         flavors=flavors,

alws/routers/builds.py

@@ -2,6 +2,7 @@
 
 from fastapi import APIRouter, Depends, HTTPException, status
 from fastapi_sqla import AsyncSessionDependency
+import redis.asyncio as aioredis
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from alws import models
@@ -10,7 +11,7 @@
 from alws.crud import build_node
 from alws.crud import platform as platform_crud
 from alws.crud import platform_flavors as flavors_crud
-from alws.dependencies import get_async_db_key
+from alws.dependencies import get_async_db_key, get_redis
 from alws.errors import BuildError, DataNotFoundError
 from alws.schemas import build_schema
 
@@ -82,6 +83,7 @@ async def get_builds_per_page(
 async def get_module_preview(
     module_request: build_schema.ModulePreviewRequest,
     db: AsyncSession = Depends(AsyncSessionDependency(key=get_async_db_key())),
+    redis: aioredis.Redis = Depends(get_redis),
 ):
     platform = await platform_crud.get_platform(
         db,
@@ -94,6 +96,7 @@ async def get_module_preview(
             ids=module_request.flavors,
         )
     return await build_crud.get_module_preview(
+        redis,
         platform,
         flavors,
         module_request,

alws/schemas/build_schema.py

@@ -16,6 +16,8 @@
 )
 from typing_extensions import Annotated
 
+import redis.asyncio as aioredis
+
 from alws import models
 from alws.config import settings
 from alws.constants import BuildTaskRefType
@@ -432,6 +434,7 @@ async def _get_module_ref(
 
 
 async def get_module_refs(
+    redis: aioredis.client.Redis,
     task: BuildTaskRef,
     platform: models.Platform,
     flavors: typing.List[models.PlatformFlavour],
@@ -448,9 +451,7 @@ async def get_module_refs(
 
     clean_dist_name = get_clean_distr_name(platform.name)
     distr_ver = platform.distr_version
-    modified_list = await get_modified_refs_list(
-        platform.modularity['modified_packages_url']
-    )
+    modified_list = await get_modified_refs_list(redis, platform.distr_version)
     template = await download_modules_yaml(
         task.url, task.git_ref, BuildTaskRefType.to_text(task.ref_type)
     )

alws/scripts/git_cacher/git_cacher.py

@@ -18,6 +18,7 @@ class Config(BaseSettings):
     git_cache_keys: typing.Dict[str, str] = {
         'rpms': 'rpms_gitea_cache',
         'modules': 'modules_gitea_cache',
+        'autopatch': 'autopatch_gitea_cache',
     }
     cacher_sentry_environment: str = "dev"
     cacher_sentry_dsn: str = ""
@@ -56,9 +57,12 @@ async def run(config, logger, redis_client, gitea_client, organization):
     to_index = []
     git_names = set()
     for repo in await gitea_client.list_repos(organization):
-        if repo['empty'] is True:
+        if repo['empty']:
             logger.warning(f"Skipping empty repo {repo['html_url']}")
             continue
+        if organization == 'autopatch' and repo['archived']:
+            logger.warning(f"Skipping archived repo {repo['html_url']}")
+            continue
         repo_name = repo['full_name']
         git_names.add(repo_name)
         repo_meta = {
@@ -79,6 +83,12 @@ async def run(config, logger, redis_client, gitea_client, organization):
     for result in results:
         cache_record = cache[result['repo_name']]
         cache_record['tags'] = [tag['name'] for tag in result['tags']]
+        if organization == 'autopatch':
+            cache_record['branches'] = [
+                branch for branch in result['branches']
+                if not branch['name'].endswith('-deprecated')
+            ]
+
         cache_record['branches'] = [
             branch['name'] for branch in result['branches']
         ]
@@ -104,8 +114,11 @@ async def main():
     while True:
         logger.info('Checking cache for updates')
         await asyncio.gather(
+            # projects git data live in these gitea orgs
             run(config, logger, redis_client, gitea_client, 'rpms'),
             run(config, logger, redis_client, gitea_client, 'modules'),
+            # almalinux modified packages live in autopatch gitea org
+            run(config, logger, redis_client, gitea_client, 'autopatch'),
         )
         logger.info(
             f'Cache has been updated, waiting for {wait} sec for next update'

alws/utils/modularity.py

@@ -11,11 +11,15 @@
 import gi
 import requests
 import yaml
+
 from pydantic import BaseModel
+import redis.asyncio as aioredis
 
 gi.require_version("Modulemd", "2.0")
 from gi.repository import Modulemd
 
+from alws.scripts.git_cacher.git_cacher import Config as GitCacherConfig
+from alws.scripts.git_cacher.git_cacher import load_redis_cache
 
 def calc_dist_macro(
     module_name: str,
@@ -32,14 +36,21 @@ def calc_dist_macro(
     return f".module_{dist_prefix}+{build_index}+{dist_hash}"
 
 
-async def get_modified_refs_list(platform_url: str):
+async def get_modified_refs_list(
+    redis: aioredis.client.Redis, distr_version: str
+):
     package_list = []
-    async with aiohttp.ClientSession() as session:
-        async with session.get(platform_url) as response:
-            yaml_body = await response.text()
-            response.raise_for_status()
-            package_list = yaml.safe_load(yaml_body)["modified_packages"]
-            return package_list
+    dist_prefix = f'a{distr_version}'
+    config = GitCacherConfig()
+
+    cache = await load_redis_cache(redis, config.git_cache_keys['autopatch'])
+    package_list = [
+        repo['name'] for repo in cache.values()
+        if any(
+            branch.startswith(dist_prefix) for branch in repo['branches']
+        )
+    ]
+    return package_list
 
 
 def get_modules_yaml_from_repo(repo_name: str):