diff --git a/alws/scripts/git_cacher/git_cacher.py b/alws/scripts/git_cacher/git_cacher.py index bc580319..19d02ee9 100644 --- a/alws/scripts/git_cacher/git_cacher.py +++ b/alws/scripts/git_cacher/git_cacher.py @@ -25,14 +25,14 @@ class Config(BaseSettings): cacher_sentry_traces_sample_rate: float = 0.2 -async def load_redis_cache(redis, cache_key): +async def load_redis_cache(redis: aioredis.Redis, cache_key: str) -> dict: value = await redis.get(cache_key) if not value: return {} return json.loads(value) -async def save_redis_cache(redis, cache_key, cache): +async def save_redis_cache(redis: aioredis.Redis, cache_key: str, cache: dict): await redis.set(cache_key, json.dumps(cache)) @@ -49,7 +49,13 @@ def setup_logger(): return logger -async def run(config, logger, redis_client, gitea_client, organization): +async def run( + config: Config, + logger: logging.Logger, + redis_client: aioredis.Redis, + gitea_client: GiteaClient, + organization: str, +): cache = await load_redis_cache( redis_client, config.git_cache_keys[organization] ) @@ -114,15 +120,19 @@ async def main(): wait = 600 while True: logger.info('Checking cache for updates') - await asyncio.gather( - # projects git data live in these gitea orgs - run(config, logger, redis_client, gitea_client, 'rpms'), - run(config, logger, redis_client, gitea_client, 'modules'), - # almalinux modified packages live in autopatch gitea org - run(config, logger, redis_client, gitea_client, 'autopatch'), - ) + await asyncio.gather(*( + run(config, logger, redis_client, gitea_client, organization) + for organization in ( + # projects git data live in these gitea orgs + 'rpms', + 'modules', + # almalinux modified packages live in autopatch gitea org + 'autopatch', + ) + )) logger.info( - 'Cache has been updated, waiting for %d secs for next update' % wait + 'Cache has been updated, waiting for %d secs for next update', + wait, ) await asyncio.sleep(wait) diff --git a/alws/utils/gitea.py b/alws/utils/gitea.py index 8afbe239..7ac4e515 100644 --- a/alws/utils/gitea.py +++ b/alws/utils/gitea.py @@ -62,6 +62,12 @@ async def make_request(self, endpoint: str, params: dict = None): ) as response: response.raise_for_status() return await response.json() + except aiohttp.client_exceptions.ClientResponseError: + self.log.exception( + 'Request %s returned with an error', + full_url, + ) + return [] except ( aiohttp.client_exceptions.ClientConnectorError, aiohttp.client_exceptions.ServerDisconnectedError, @@ -99,6 +105,13 @@ async def list_tags(self, repo: str) -> typing.List: endpoint = f'repos/{repo}/tags' return await self._list_all_pages(endpoint) + async def list_tags_from_repo_refs(self, repo: str) -> list[dict]: + endpoint = f'repos/{repo}/git/refs/tags' + return [ + {'name': ref['ref'].replace('refs/tags/', '')} + for ref in await self.make_request(endpoint) + ] + async def list_branches(self, repo: str) -> typing.List: endpoint = f'repos/{repo}/branches' return await self._list_all_pages(endpoint) @@ -108,6 +121,6 @@ async def get_branch(self, repo: str, branch: str) -> typing.Dict: return await self.make_request(endpoint) async def index_repo(self, repo_name: str): - tags = await self.list_tags(repo_name) + tags = await self.list_tags_from_repo_refs(repo_name) branches = await self.list_branches(repo_name) return {'repo_name': repo_name, 'tags': tags, 'branches': branches}