Merge pull request #1121 from javihernandez/bs-438

Update the method to retrieve the list of modified packages

Author: anfimovdm

alws/crud/build.py

@@ -1,6 +1,7 @@
 import logging
 import typing
 
+import redis.asyncio as aioredis
 import sqlalchemy
 from sqlalchemy import delete
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -238,11 +239,13 @@ async def generate_query(count=False):
 
 
 async def get_module_preview(
+    redis: aioredis.client.Redis,
     platform: models.Platform,
     flavors: typing.List[models.PlatformFlavour],
     module_request: build_schema.ModulePreviewRequest,
 ) -> build_schema.ModulePreview:
     refs, modules, enabled_modules = await build_schema.get_module_refs(
+        redis=redis,
         task=module_request.ref,
         platform=platform,
         flavors=flavors,

alws/routers/builds.py

@@ -1,5 +1,6 @@
 import typing
 
+import redis.asyncio as aioredis
 from fastapi import APIRouter, Depends, HTTPException, status
 from fastapi_sqla import AsyncSessionDependency
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -10,7 +11,7 @@
 from alws.crud import build_node
 from alws.crud import platform as platform_crud
 from alws.crud import platform_flavors as flavors_crud
-from alws.dependencies import get_async_db_key
+from alws.dependencies import get_async_db_key, get_redis
 from alws.errors import BuildError, DataNotFoundError
 from alws.schemas import build_schema
 
@@ -82,6 +83,7 @@ async def get_builds_per_page(
 async def get_module_preview(
     module_request: build_schema.ModulePreviewRequest,
     db: AsyncSession = Depends(AsyncSessionDependency(key=get_async_db_key())),
+    redis: aioredis.Redis = Depends(get_redis),
 ):
     platform = await platform_crud.get_platform(
         db,
@@ -94,6 +96,7 @@ async def get_module_preview(
             ids=module_request.flavors,
         )
     return await build_crud.get_module_preview(
+        redis,
         platform,
         flavors,
         module_request,

alws/schemas/build_schema.py

@@ -7,6 +7,7 @@
 import urllib.parse
 
 import aiohttp.client_exceptions
+import redis.asyncio as aioredis
 from pydantic import (
     AfterValidator,
     AnyHttpUrl,
@@ -125,9 +126,7 @@ class BuildCreatePlatforms(BaseModel):
 
 class BuildCreate(BaseModel):
     platforms: conlist(BuildCreatePlatforms, min_length=1)
-    tasks: conlist(
-        typing.Union[BuildTaskRef, BuildTaskModuleRef], min_length=1
-    )
+    tasks: conlist(typing.Union[BuildTaskRef, BuildTaskModuleRef], min_length=1)
     linked_builds: typing.List[int] = []
     mock_options: typing.Optional[typing.Dict[str, typing.Any]] = None
     platform_flavors: typing.Optional[typing.List[int]] = None
@@ -432,6 +431,7 @@ async def _get_module_ref(
 
 
 async def get_module_refs(
+    redis: aioredis.client.Redis,
     task: BuildTaskRef,
     platform: models.Platform,
     flavors: typing.List[models.PlatformFlavour],
@@ -448,9 +448,7 @@ async def get_module_refs(
 
     clean_dist_name = get_clean_distr_name(platform.name)
     distr_ver = platform.distr_version
-    modified_list = await get_modified_refs_list(
-        platform.modularity['modified_packages_url']
-    )
+    modified_list = await get_modified_refs_list(redis, platform.distr_version)
     template = await download_modules_yaml(
         task.url, task.git_ref, BuildTaskRefType.to_text(task.ref_type)
     )

alws/scripts/git_cacher/git_cacher.py

@@ -18,6 +18,7 @@ class Config(BaseSettings):
     git_cache_keys: typing.Dict[str, str] = {
         'rpms': 'rpms_gitea_cache',
         'modules': 'modules_gitea_cache',
+        'autopatch': 'autopatch_gitea_cache',
     }
     cacher_sentry_environment: str = "dev"
     cacher_sentry_dsn: str = ""
@@ -56,9 +57,12 @@ async def run(config, logger, redis_client, gitea_client, organization):
     to_index = []
     git_names = set()
     for repo in await gitea_client.list_repos(organization):
-        if repo['empty'] is True:
+        if repo['empty']:
             logger.warning(f"Skipping empty repo {repo['html_url']}")
             continue
+        if organization == 'autopatch' and repo['archived']:
+            logger.warning(f"Skipping archived repo {repo['html_url']}")
+            continue
         repo_name = repo['full_name']
         git_names.add(repo_name)
         repo_meta = {
@@ -79,6 +83,13 @@ async def run(config, logger, redis_client, gitea_client, organization):
     for result in results:
         cache_record = cache[result['repo_name']]
         cache_record['tags'] = [tag['name'] for tag in result['tags']]
+        if organization == 'autopatch':
+            cache_record['branches'] = [
+                branch
+                for branch in result['branches']
+                if not branch['name'].endswith('-deprecated')
+            ]
+
         cache_record['branches'] = [
             branch['name'] for branch in result['branches']
         ]
@@ -104,11 +115,14 @@ async def main():
     while True:
         logger.info('Checking cache for updates')
         await asyncio.gather(
+            # projects git data live in these gitea orgs
             run(config, logger, redis_client, gitea_client, 'rpms'),
             run(config, logger, redis_client, gitea_client, 'modules'),
+            # almalinux modified packages live in autopatch gitea org
+            run(config, logger, redis_client, gitea_client, 'autopatch'),
         )
         logger.info(
-            f'Cache has been updated, waiting for {wait} sec for next update'
+            'Cache has been updated, waiting for %d secs for next update' % wait
         )
         await asyncio.sleep(wait)
 

alws/utils/modularity.py

@@ -9,13 +9,17 @@
 
 import aiohttp
 import gi
+import redis.asyncio as aioredis
 import requests
 import yaml
 from pydantic import BaseModel
 
 gi.require_version("Modulemd", "2.0")
 from gi.repository import Modulemd
 
+from alws.scripts.git_cacher.git_cacher import Config as GitCacherConfig
+from alws.scripts.git_cacher.git_cacher import load_redis_cache
+
 
 def calc_dist_macro(
     module_name: str,
@@ -32,14 +36,20 @@ def calc_dist_macro(
     return f".module_{dist_prefix}+{build_index}+{dist_hash}"
 
 
-async def get_modified_refs_list(platform_url: str):
+async def get_modified_refs_list(
+    redis: aioredis.client.Redis, distr_version: str
+):
     package_list = []
-    async with aiohttp.ClientSession() as session:
-        async with session.get(platform_url) as response:
-            yaml_body = await response.text()
-            response.raise_for_status()
-            package_list = yaml.safe_load(yaml_body)["modified_packages"]
-            return package_list
+    dist_prefix = f'a{distr_version}'
+    config = GitCacherConfig()
+
+    cache = await load_redis_cache(redis, config.git_cache_keys['autopatch'])
+    package_list = [
+        repo['name']
+        for repo in cache.values()
+        if any(branch.startswith(dist_prefix) for branch in repo['branches'])
+    ]
+    return package_list
 
 
 def get_modules_yaml_from_repo(repo_name: str):
@@ -240,9 +250,7 @@ def add_module_dependencies_from_mock_defs(self, enabled_modules: dict):
                     if stream:
                         new_deps.add_runtime_stream(name, stream)
                     else:
-                        new_deps.set_empty_runtime_dependencies_for_module(
-                            name
-                        )
+                        new_deps.set_empty_runtime_dependencies_for_module(name)
 
         for module in old_runtime:
             if module == "platform":