diff --git a/alws/crud/errata.py b/alws/crud/errata.py index a515c106..c7771f87 100644 --- a/alws/crud/errata.py +++ b/alws/crud/errata.py @@ -573,8 +573,11 @@ def prepare_search_params( errata_record: Union[models.NewErrataRecord, BaseErrataRecord], ) -> DefaultDict[str, List[str]]: search_params = collections.defaultdict(list) + attrs = ["name", "version", "epoch"] + if errata_record.is_issued_by_almalinux: + attrs.append("release") for package in errata_record.packages: - for attr in ("name", "version", "epoch"): + for attr in attrs: value = str(getattr(package, attr)) if value in search_params[attr]: continue @@ -608,6 +611,7 @@ async def load_platform_packages( pkg_names=search_params["name"], pkg_versions=search_params["version"], pkg_epochs=search_params["epoch"], + pkg_releases=search_params.get("release", None), ) if not pkgs: @@ -675,6 +679,7 @@ async def get_matching_albs_packages( prod_repos_cache: typing.Dict, module: Optional[str] = None, build_id: Optional[int] = None, + is_issued_by_almalinux: Optional[bool] = False, ) -> Tuple[List[models.NewErrataToALBSPackage], dict]: package_type = {} items_to_insert = [] @@ -683,10 +688,12 @@ async def get_matching_albs_packages( # - my-pkg-2.0-2 # - my-pkg-2.0-20191233git # - etc + # Also, note that when processing erratas issued by almalinux, we don't + # clean the package release clean_package_name = "-".join(( errata_package.name, errata_package.version, - clean_release(errata_package.release), + clean_release(errata_package.release, is_issued_by_almalinux), )) # We add ErrataToALBSPackage if we find a matching package already # in production repositories. @@ -729,15 +736,11 @@ async def get_matching_albs_packages( models.BuildTaskArtifact.name.startswith(name_query), ] if build_id: - conditions.append( - models.BuildTask.build_id == build_id - ) + conditions.append(models.BuildTask.build_id == build_id) query = ( select(models.BuildTaskArtifact) .join(models.BuildTaskArtifact.build_task) - .where( - and_(*conditions) - ) + .where(and_(*conditions)) .options( selectinload(models.BuildTaskArtifact.build_task), ) @@ -775,6 +778,11 @@ async def get_matching_albs_packages( ] pulp_pkgs = get_rpm_packages_by_ids(pulp_pkg_ids, pkg_fields) errata_record_ids = set() + package_status = ( + ErrataPackageStatus.approved + if is_issued_by_almalinux + else ErrataPackageStatus.proposal + ) for package in result: pulp_rpm_package = pulp_pkgs.get(package.href) if not pulp_rpm_package: @@ -782,7 +790,7 @@ async def get_matching_albs_packages( clean_pulp_package_name = "-".join(( pulp_rpm_package.name, pulp_rpm_package.version, - clean_release(pulp_rpm_package.release), + clean_release(pulp_rpm_package.release, is_issued_by_almalinux), )) if ( pulp_rpm_package.arch not in (errata_package.arch, "noarch") @@ -791,7 +799,7 @@ async def get_matching_albs_packages( continue mapping = models.NewErrataToALBSPackage( albs_artifact_id=package.id, - status=ErrataPackageStatus.proposal, + status=package_status, name=pulp_rpm_package.name, version=pulp_rpm_package.version, release=pulp_rpm_package.release, @@ -919,7 +927,12 @@ async def process_new_errata_packages( db_packages.append(db_package) # Create ErrataToAlbsPackages matching_packages, pkg_type = await get_matching_albs_packages( - db, db_package, prod_repos_cache, db_errata.module, package.build_id + db, + db_package, + prod_repos_cache, + db_errata.module, + package.build_id, + errata.is_issued_by_almalinux, ) db_packages.extend(matching_packages) pkg_types.append(pkg_type) @@ -937,7 +950,9 @@ async def create_new_errata_record(errata: typing.Dict): platform = platform.scalars().first() items_to_insert = [] original_id = errata.id - oval_title = f"{errata.id}: {errata.title} ({errata.severity.capitalize()})" + oval_title = ( + f"{errata.id}: {errata.title} ({errata.severity.capitalize()})" + ) # Errata db record db_errata = models.NewErrataRecord( @@ -993,7 +1008,9 @@ async def create_new_errata_record(errata: typing.Dict): # References items_to_insert.extend( - await process_new_errata_references(session, errata, db_errata, platform) + await process_new_errata_references( + session, errata, db_errata, platform + ) ) # Errata Packages new_errata_packages, pkg_types = await process_new_errata_packages( @@ -1073,7 +1090,9 @@ async def create_errata_record(errata: typing.Dict): oval_title=get_oval_title( errata.title, alma_errata_id, errata.severity ), - original_title=get_verbose_errata_title(errata.title, errata.severity), + original_title=get_verbose_errata_title( + errata.title, errata.severity + ), contact_mail=platform.contact_mail, status=errata.status, version=errata.version, diff --git a/alws/models.py b/alws/models.py index 3c9d8986..bea259b3 100644 --- a/alws/models.py +++ b/alws/models.py @@ -4,8 +4,6 @@ from typing import Any, Dict, List, Literal, Optional import sqlalchemy -from sqlalchemy.orm import selectinload -from sqlalchemy.ext.asyncio import AsyncSession from fastapi_users.db import ( SQLAlchemyBaseOAuthAccountTable, SQLAlchemyBaseUserTable, @@ -18,13 +16,14 @@ AssociationProxy, association_proxy, ) -from sqlalchemy.ext.asyncio import create_async_engine +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine from sqlalchemy.orm import ( Mapped, declarative_mixin, declared_attr, mapped_column, relationship, + selectinload, ) from sqlalchemy.sql import func @@ -663,9 +662,7 @@ class BuildTaskArtifact(Base): name: Mapped[str] = mapped_column(sqlalchemy.Text, nullable=False) type: Mapped[str] = mapped_column(sqlalchemy.Text, nullable=False) href: Mapped[str] = mapped_column(sqlalchemy.Text, nullable=False) - meta: Mapped[Optional[Dict[str, Any]]] = mapped_column( - JSONB, nullable=True - ) + meta: Mapped[Optional[Dict[str, Any]]] = mapped_column(JSONB, nullable=True) build_task: Mapped["BuildTask"] = relationship( "BuildTask", back_populates="artifacts" ) @@ -1625,6 +1622,10 @@ def get_type(self): "EA": "enhancement", }[self.id[2:4]] + @property + def is_issued_by_almalinux(self) -> bool: + return bool(re.search(r"AL[BES]A-\d{4}:A", self.id)) + class NewErrataPackage(Base): __tablename__ = "new_errata_packages" diff --git a/alws/schemas/errata_schema.py b/alws/schemas/errata_schema.py index cddc4ed8..0d55a51f 100644 --- a/alws/schemas/errata_schema.py +++ b/alws/schemas/errata_schema.py @@ -2,9 +2,9 @@ import re from typing import Any, List, Optional -from pydantic import BaseModel, field_validator, computed_field +from pydantic import BaseModel, computed_field, field_validator -from alws.constants import ErrataReleaseStatus, ErrataPackageStatus +from alws.constants import ErrataPackageStatus, ErrataReleaseStatus class BaseErrataCVE(BaseModel): diff --git a/alws/utils/errata.py b/alws/utils/errata.py index 4857b1b9..b9b2c50b 100644 --- a/alws/utils/errata.py +++ b/alws/utils/errata.py @@ -89,9 +89,7 @@ def debrand_id(rec_id: str) -> str: return f'oval:{org}.{adv}:{data["rec_type"]}:{data["idx"]}' -def debrand_affected_cpe_list( - cpe_list: List[str], distro_version -) -> List[str]: +def debrand_affected_cpe_list(cpe_list: List[str], distro_version) -> List[str]: new_list = [] for cpe in cpe_list: cpe = cpe.replace("redhat:enterprise_linux", "almalinux:almalinux") diff --git a/alws/utils/parsing.py b/alws/utils/parsing.py index 2f89f72a..c0f70a2f 100644 --- a/alws/utils/parsing.py +++ b/alws/utils/parsing.py @@ -1,12 +1,11 @@ import re import typing -from tap import parser import hawkey +from tap import parser from alws.constants import TestCaseStatus - __all__ = [ 'clean_release', 'get_clean_distr_name', @@ -22,13 +21,16 @@ def slice_list( max_len: int, ) -> typing.Generator[typing.List[str], None, None]: return ( - source_list[i:i + max_len] + source_list[i : i + max_len] for i in range(0, len(source_list), max_len) ) -def clean_release(release: str) -> str: - release = re.sub(r'\.alma.*$', '', release) +def clean_release( + release: str, keep_alma_suffix: typing.Optional[bool] = False +) -> str: + if not keep_alma_suffix: + release = re.sub(r'\.alma.*$', '', release) latest = None raw_res = re.search(r'\.module.*', release) if raw_res: @@ -41,9 +43,15 @@ def clean_release(release: str) -> str: def get_clean_distr_name(distr_name: str) -> str: - clean_distr_name = re.search( - r'^(?P[^\d]*)', distr_name, re.IGNORECASE, - ).groupdict().get('dist_name', '') + clean_distr_name = ( + re.search( + r'^(?P[^\d]*)', + distr_name, + re.IGNORECASE, + ) + .groupdict() + .get('dist_name', '') + ) return clean_distr_name.strip('-') @@ -91,8 +99,9 @@ def parse_tap_output(text: bytes) -> list: def get_diagnostic(tap_item): diagnostics = [] index = raw_data.index(tap_item) + 1 - while index < len(raw_data) and \ - raw_data[index].category == "diagnostic": + while ( + index < len(raw_data) and raw_data[index].category == "diagnostic" + ): diagnostics.append(raw_data[index].text) index += 1 return u"\n".join(diagnostics) @@ -135,5 +144,7 @@ def tap_set_status(tap_results): True if all tests have passed, False otherwise """ - conditions = [item["status"] == TestCaseStatus.FAILED for item in tap_results] + conditions = [ + item["status"] == TestCaseStatus.FAILED for item in tap_results + ] return False if any(conditions) else True diff --git a/alws/utils/pulp_utils.py b/alws/utils/pulp_utils.py index 90511410..acc0930e 100644 --- a/alws/utils/pulp_utils.py +++ b/alws/utils/pulp_utils.py @@ -61,7 +61,9 @@ def get_rpm_module_packages_from_repository( return result module_name, module_stream = module.split(":") - devel_module_name = f"{module_name}-devel" if not module_name.endswith("-devel") else "" + devel_module_name = ( + f"{module_name}-devel" if not module_name.endswith("-devel") else "" + ) repo_modules = [] try: repo_index = IndexWrapper.from_template(repo_modules_yaml) @@ -201,9 +203,12 @@ def get_rpm_packages_from_repository( pkg_versions: typing.Optional[typing.List[str]] = None, pkg_epochs: typing.Optional[typing.List[str]] = None, pkg_arches: typing.Optional[typing.List[str]] = None, + pkg_releases: typing.Optional[typing.List[str]] = None, ) -> typing.List[RpmPackage]: first_subq = ( - select(CoreRepository.pulp_id).where(CoreRepository.pulp_id == repo_id).scalar_subquery() + select(CoreRepository.pulp_id) + .where(CoreRepository.pulp_id == repo_id) + .scalar_subquery() ) second_subq = ( select(CoreRepositoryContent.content_id) @@ -233,6 +238,8 @@ def get_rpm_packages_from_repository( conditions.append(RpmPackage.epoch.in_(pkg_epochs)) if pkg_arches: conditions.append(RpmPackage.arch.in_(pkg_arches)) + if pkg_releases: + conditions.append(RpmPackage.release.in_(pkg_releases)) query = select(RpmPackage).where(*conditions) with open_session(key="pulp") as pulp_db: