diff --git a/alws/crud/errata.py b/alws/crud/errata.py index 1f20ba15..633f57d2 100644 --- a/alws/crud/errata.py +++ b/alws/crud/errata.py @@ -4,6 +4,7 @@ import json import logging import re +import typing import uuid from typing import ( Any, @@ -33,7 +34,7 @@ ErrataReferenceType, ErrataReleaseStatus, ) -from alws.dependencies import get_async_db_key +from alws.dependencies import get_async_db_key, get_async_db_session from alws.pulp_models import ( RpmPackage, UpdateCollection, @@ -913,280 +914,282 @@ async def process_new_errata_packages( return packages, pkg_types -async def create_new_errata_record(db: AsyncSession, errata: BaseErrataRecord): - platform = await db.execute( - select(models.Platform) - .where(models.Platform.id == errata.platform_id) - .options(selectinload(models.Platform.repos)) - ) - platform = platform.scalars().first() - items_to_insert = [] - original_id = errata.id - oval_title = f"{errata.id}: {errata.title} ({errata.severity.capitalize()})" - - # Errata db record - db_errata = models.NewErrataRecord( - id=errata.id, - # TODO: BS-376 - freezed=errata.freezed, - platform_id=errata.platform_id, - module=errata.module, - devel_module=errata.devel_module, - release_status=ErrataReleaseStatus.NOT_RELEASED, - # TODO: BS-376 - # Not sure it's used, check and if not, remove it from data model - summary=None, - # TODO: BS-376 - # Not used AFAIK, maybe can be removed from data model - solution=None, - issued_date=errata.issued_date, - updated_date=errata.updated_date, - description=None, - original_description=errata.description, - title=None, - oval_title=oval_title, - # TODO: I'd prefer to keep the title without severity, and then present - # the severity differently in UI - original_title=f"{errata.severity.capitalize()}: {errata.title}", - contact_mail=platform.contact_mail, - status=errata.status, - version=errata.version, - severity=errata.severity, - rights=jinja2.Template(platform.copyright).render( - year=datetime.datetime.utcnow().year - ), - # All values set to None below will be set upon errata release - definition_id=None, - definition_version=errata.definition_version, - definition_class=errata.definition_class, - # TODO: Right now we're adding all repos cpes - # Ideally, we should generate affected cpes based on the repos - # where the affected packages live. - affected_cpe=None, - criteria=None, - original_criteria=errata.criteria, - tests=None, - original_tests=errata.tests, - objects=None, - original_objects=errata.objects, - states=None, - original_states=errata.states, - variables=None, - original_variables=errata.variables, - ) - items_to_insert.append(db_errata) - - # References - items_to_insert.extend( - await process_new_errata_references(db, errata, db_errata, platform) - ) - # Errata Packages - new_errata_packages, pkg_types = await process_new_errata_packages( - db, errata, db_errata, platform - ) - items_to_insert.extend(new_errata_packages) - - db.add_all(items_to_insert) - await db.flush() - await db.refresh(db_errata) - if not settings.github_integration_enabled: - return db_errata - - try: - github_client = await get_github_client() - await create_github_issue( - client=github_client, - title=errata.title, - description=errata.description, - advisory_id=errata.id, - original_id=original_id, - platform_name=platform.name, - severity=errata.severity.capitalize(), - packages=errata.packages, - platform_id=errata.platform_id, - find_packages_types=pkg_types, +async def create_new_errata_record(errata: typing.Dict): + async with get_async_db_session() as session: + platform = await session.execute( + select(models.Platform) + .where(models.Platform.id == errata.platform_id) + .options(selectinload(models.Platform.repos)) ) - except Exception as err: - logging.exception( - "Cannot create GitHub issue: %s", - err, + platform = platform.scalars().first() + items_to_insert = [] + original_id = errata.id + oval_title = f"{errata.id}: {errata.title} ({errata.severity.capitalize()})" + + # Errata db record + db_errata = models.NewErrataRecord( + id=errata.id, + # TODO: BS-376 + freezed=errata.freezed, + platform_id=errata.platform_id, + module=errata.module, + devel_module=errata.devel_module, + release_status=ErrataReleaseStatus.NOT_RELEASED, + # TODO: BS-376 + # Not sure it's used, check and if not, remove it from data model + summary=None, + # TODO: BS-376 + # Not used AFAIK, maybe can be removed from data model + solution=None, + issued_date=errata.issued_date, + updated_date=errata.updated_date, + description=None, + original_description=errata.description, + title=None, + oval_title=oval_title, + # TODO: I'd prefer to keep the title without severity, and then present + # the severity differently in UI + original_title=f"{errata.severity.capitalize()}: {errata.title}", + contact_mail=platform.contact_mail, + status=errata.status, + version=errata.version, + severity=errata.severity, + rights=jinja2.Template(platform.copyright).render( + year=datetime.datetime.utcnow().year + ), + # All values set to None below will be set upon errata release + definition_id=None, + definition_version=errata.definition_version, + definition_class=errata.definition_class, + # TODO: Right now we're adding all repos cpes + # Ideally, we should generate affected cpes based on the repos + # where the affected packages live. + affected_cpe=None, + criteria=None, + original_criteria=errata.criteria, + tests=None, + original_tests=errata.tests, + objects=None, + original_objects=errata.objects, + states=None, + original_states=errata.states, + variables=None, + original_variables=errata.variables, ) - return db_errata + items_to_insert.append(db_errata) - -async def create_errata_record(db: AsyncSession, errata: BaseErrataRecord): - platform = await db.execute( - select(models.Platform) - .where(models.Platform.id == errata.platform_id) - .options(selectinload(models.Platform.repos)) - ) - platform = platform.scalars().first() - items_to_insert = [] - original_id = errata.id - - # Rebranding RHEL -> AlmaLinux - for key in ("description", "title"): - setattr( - errata, - key, - debrand_description_and_title(getattr(errata, key)), + # References + items_to_insert.extend( + await process_new_errata_references(session, errata, db_errata, platform) ) - alma_errata_id = re.sub(r"^RH", "AL", errata.id) - - # Check if errata refers to a module - r = re.compile(r"Module ([\d\w\-\_]+:[\d\.\w]+) is enabled") - match = r.findall(str(errata.criteria)) - # Ensure we get a module and is not the -devel one - errata_module = None if not match else match[0].replace("-devel:", ":") - - # Errata db record - db_errata = models.NewErrataRecord( - id=alma_errata_id, - freezed=errata.freezed, - platform_id=errata.platform_id, - module=errata_module, - release_status=ErrataReleaseStatus.NOT_RELEASED, - summary=None, - solution=None, - issued_date=errata.issued_date, - updated_date=errata.updated_date, - description=None, - original_description=errata.description, - title=None, - oval_title=get_oval_title( - errata.title, alma_errata_id, errata.severity - ), - original_title=get_verbose_errata_title(errata.title, errata.severity), - contact_mail=platform.contact_mail, - status=errata.status, - version=errata.version, - severity=errata.severity, - rights=jinja2.Template(platform.copyright).render( - year=datetime.datetime.utcnow().year - ), - definition_id=errata.definition_id, - definition_version=errata.definition_version, - definition_class=errata.definition_class, - affected_cpe=errata.affected_cpe, - criteria=None, - original_criteria=errata.criteria, - tests=None, - original_tests=errata.tests, - objects=None, - original_objects=errata.objects, - states=None, - original_states=errata.states, - variables=None, - original_variables=errata.variables, - ) - items_to_insert.append(db_errata) + # Errata Packages + new_errata_packages, pkg_types = await process_new_errata_packages( + session, errata, db_errata, platform + ) + items_to_insert.extend(new_errata_packages) - # References - self_ref_exists = False - for ref in errata.references: - db_cve = None - if ref.cve: - db_cve = await db.execute( - select(models.ErrataCVE).where( - models.ErrataCVE.id == ref.cve.id - ) + session.add_all(items_to_insert) + await session.flush() + await session.refresh(db_errata) + if not settings.github_integration_enabled: + return db_errata + + try: + github_client = await get_github_client() + await create_github_issue( + client=github_client, + title=errata.title, + description=errata.description, + advisory_id=errata.id, + original_id=original_id, + platform_name=platform.name, + severity=errata.severity.capitalize(), + packages=errata.packages, + platform_id=errata.platform_id, + find_packages_types=pkg_types, ) - db_cve = db_cve.scalars().first() - if db_cve is None: - db_cve = models.ErrataCVE( - id=ref.cve.id, - cvss3=ref.cve.cvss3, - cwe=ref.cve.cwe, - impact=ref.cve.impact, - public=ref.cve.public, - ) - items_to_insert.append(db_cve) - ref_title = "" - if ref.ref_type in ( - ErrataReferenceType.cve.value, - ErrataReferenceType.rhsa.value, - ): - ref_title = ref.ref_id - db_reference = models.NewErrataReference( - href=ref.href, - ref_id=ref.ref_id, - ref_type=ref.ref_type, - title=ref_title, - cve=db_cve, + except Exception as err: + logging.exception( + "Cannot create GitHub issue: %s", + err, + ) + return db_errata + + +async def create_errata_record(errata: typing.Dict): + async with get_async_db_session() as session: + platform = await session.execute( + select(models.Platform) + .where(models.Platform.id == errata.platform_id) + .options(selectinload(models.Platform.repos)) ) - if ref.ref_type == ErrataReferenceType.self_ref.value: - self_ref_exists = True - db_errata.references.append(db_reference) - items_to_insert.append(db_reference) - if not self_ref_exists: - html_id = db_errata.id.replace(":", "-") - self_ref = models.NewErrataReference( - href=( - "https://errata.almalinux.org/" - f"{platform.distr_version}/{html_id}.html" + platform = platform.scalars().first() + items_to_insert = [] + original_id = errata.id + + # Rebranding RHEL -> AlmaLinux + for key in ("description", "title"): + setattr( + errata, + key, + debrand_description_and_title(getattr(errata, key)), + ) + alma_errata_id = re.sub(r"^RH", "AL", errata.id) + + # Check if errata refers to a module + r = re.compile(r"Module ([\d\w\-\_]+:[\d\.\w]+) is enabled") + match = r.findall(str(errata.criteria)) + # Ensure we get a module and is not the -devel one + errata_module = None if not match else match[0].replace("-devel:", ":") + + # Errata db record + db_errata = models.NewErrataRecord( + id=alma_errata_id, + freezed=errata.freezed, + platform_id=errata.platform_id, + module=errata_module, + release_status=ErrataReleaseStatus.NOT_RELEASED, + summary=None, + solution=None, + issued_date=errata.issued_date, + updated_date=errata.updated_date, + description=None, + original_description=errata.description, + title=None, + oval_title=get_oval_title( + errata.title, alma_errata_id, errata.severity ), - ref_id=db_errata.id, - ref_type=ErrataReferenceType.self_ref, - title=db_errata.id, + original_title=get_verbose_errata_title(errata.title, errata.severity), + contact_mail=platform.contact_mail, + status=errata.status, + version=errata.version, + severity=errata.severity, + rights=jinja2.Template(platform.copyright).render( + year=datetime.datetime.utcnow().year + ), + definition_id=errata.definition_id, + definition_version=errata.definition_version, + definition_class=errata.definition_class, + affected_cpe=errata.affected_cpe, + criteria=None, + original_criteria=errata.criteria, + tests=None, + original_tests=errata.tests, + objects=None, + original_objects=errata.objects, + states=None, + original_states=errata.states, + variables=None, + original_variables=errata.variables, ) - db_errata.references.append(self_ref) - items_to_insert.append(self_ref) + items_to_insert.append(db_errata) + + # References + self_ref_exists = False + for ref in errata.references: + db_cve = None + if ref.cve: + db_cve = await session.execute( + select(models.ErrataCVE).where( + models.ErrataCVE.id == ref.cve.id + ) + ) + db_cve = db_cve.scalars().first() + if db_cve is None: + db_cve = models.ErrataCVE( + id=ref.cve.id, + cvss3=ref.cve.cvss3, + cwe=ref.cve.cwe, + impact=ref.cve.impact, + public=ref.cve.public, + ) + items_to_insert.append(db_cve) + ref_title = "" + if ref.ref_type in ( + ErrataReferenceType.cve.value, + ErrataReferenceType.rhsa.value, + ): + ref_title = ref.ref_id + db_reference = models.NewErrataReference( + href=ref.href, + ref_id=ref.ref_id, + ref_type=ref.ref_type, + title=ref_title, + cve=db_cve, + ) + if ref.ref_type == ErrataReferenceType.self_ref.value: + self_ref_exists = True + db_errata.references.append(db_reference) + items_to_insert.append(db_reference) + if not self_ref_exists: + html_id = db_errata.id.replace(":", "-") + self_ref = models.NewErrataReference( + href=( + "https://errata.almalinux.org/" + f"{platform.distr_version}/{html_id}.html" + ), + ref_id=db_errata.id, + ref_type=ErrataReferenceType.self_ref, + title=db_errata.id, + ) + db_errata.references.append(self_ref) + items_to_insert.append(self_ref) - # Errata Packages - search_params = prepare_search_params(errata) + # Errata Packages + search_params = prepare_search_params(errata) - prod_repos_cache = await load_platform_packages( - platform, - search_params, - False, - db_errata.module, - ) - pkg_types = [] - for package in errata.packages: - db_package = models.NewErrataPackage( - name=package.name, - version=package.version, - release=package.release, - epoch=package.epoch, - arch=package.arch, - source_srpm=None, - reboot_suggested=False, - ) - db_errata.packages.append(db_package) - items_to_insert.append(db_package) - # Create ErrataToAlbsPackages - matching_packages, pkg_type = await get_matching_albs_packages( - db, db_package, prod_repos_cache, db_errata.module + prod_repos_cache = await load_platform_packages( + platform, + search_params, + False, + db_errata.module, ) - pkg_types.append(pkg_type) - items_to_insert.extend(matching_packages) + pkg_types = [] + for package in errata.packages: + db_package = models.NewErrataPackage( + name=package.name, + version=package.version, + release=package.release, + epoch=package.epoch, + arch=package.arch, + source_srpm=None, + reboot_suggested=False, + ) + db_errata.packages.append(db_package) + items_to_insert.append(db_package) + # Create ErrataToAlbsPackages + matching_packages, pkg_type = await get_matching_albs_packages( + session, db_package, prod_repos_cache, db_errata.module + ) + pkg_types.append(pkg_type) + items_to_insert.extend(matching_packages) - db.add_all(items_to_insert) - await db.flush() - await db.refresh(db_errata) - if not settings.github_integration_enabled: - return db_errata + session.add_all(items_to_insert) + await session.flush() + await session.refresh(db_errata) + if not settings.github_integration_enabled: + return db_errata - try: - github_client = await get_github_client() - await create_github_issue( - client=github_client, - title=errata.title, - description=errata.description, - advisory_id=alma_errata_id, - original_id=original_id, - platform_name=platform.name, - severity=errata.severity, - packages=errata.packages, - platform_id=errata.platform_id, - find_packages_types=pkg_types, - ) - except Exception as err: - logging.exception( - "Cannot create GitHub issue: %s", - err, - ) - return db_errata + try: + github_client = await get_github_client() + await create_github_issue( + client=github_client, + title=errata.title, + description=errata.description, + advisory_id=alma_errata_id, + original_id=original_id, + platform_name=platform.name, + severity=errata.severity, + packages=errata.packages, + platform_id=errata.platform_id, + find_packages_types=pkg_types, + ) + except Exception as err: + logging.exception( + "Cannot create GitHub issue: %s", + err, + ) + return db_errata async def get_errata_record( diff --git a/alws/dramatiq/__init__.py b/alws/dramatiq/__init__.py index 3f8da78a..1e473a6a 100644 --- a/alws/dramatiq/__init__.py +++ b/alws/dramatiq/__init__.py @@ -35,6 +35,7 @@ bulk_errata_release, bulk_new_errata_release, create_new_errata, + create_errata, release_errata, release_new_errata, reset_records_threshold, diff --git a/alws/dramatiq/errata.py b/alws/dramatiq/errata.py index 96e98428..df5fa7c6 100644 --- a/alws/dramatiq/errata.py +++ b/alws/dramatiq/errata.py @@ -7,6 +7,7 @@ bulk_errata_records_release, bulk_new_errata_records_release, create_new_errata_record, + create_errata_record, release_errata_record, release_new_errata_record, reset_matched_erratas_packages_threshold, @@ -17,8 +18,12 @@ __all__ = ["release_errata"] -async def _create_new_errata(errata): - await create_new_errata_record(errata) +async def _create_new_errata_record(errata): + await create_new_errata_record(errata) + + +async def _create_errata_record(errata): + await create_errata_record(errata) async def _release_new_errata_record( @@ -64,7 +69,20 @@ async def _reset_matched_erratas_packages_threshold(issued_date: str): def create_new_errata(errata): event_loop.run_until_complete(setup_all()) event_loop.run_until_complete( - _create_new_errata(errata) + _create_new_errata_record(errata) + ) + + +@dramatiq.actor( + max_retries=0, + priority=0, + queue_name="errata", + time_limit=DRAMATIQ_TASK_TIMEOUT, +) +def create_errata(errata): + event_loop.run_until_complete(setup_all()) + event_loop.run_until_complete( + _create_errata_record(errata) ) diff --git a/alws/routers/errata.py b/alws/routers/errata.py index 78042a03..bd4c925e 100644 --- a/alws/routers/errata.py +++ b/alws/routers/errata.py @@ -17,6 +17,7 @@ bulk_errata_release, bulk_new_errata_release, create_new_errata, + create_errata, release_errata, release_new_errata, reset_records_threshold, @@ -39,22 +40,35 @@ async def create_new_errata_record( errata: errata_schema.BaseErrataRecord ): - create_new_errata.send(errata) + try: + errata_data = errata.model_dump() + + # Convert date fields to strings + errata_data["issued_date"] = errata_data["issued_date"].isoformat() + errata_data["updated_date"] = errata_data["updated_date"].isoformat() - message = f"Record {errata.id} is scheduled for creation" - return {"ok": message} + create_new_errata.send(errata_data) + + return {"ok": True} + except ValueError as e: + return {"ok": False, "error": str(e)} @router.post("/", response_model=errata_schema.CreateErrataResponse) async def create_errata_record( - errata: errata_schema.BaseErrataRecord, - db: AsyncSession = Depends(AsyncSessionDependency(key=get_async_db_key())), + errata: errata_schema.BaseErrataRecord ): - record = await errata_crud.create_errata_record( - db, - errata, - ) - return {"ok": bool(record)} + try: + errata_data = errata.model_dump() + + errata_data["issued_date"] = errata_data["issued_date"].isoformat() + errata_data["updated_date"] = errata_data["updated_date"].isoformat() + + create_errata.send(errata_data) + + return {"ok": True} + except ValueError as e: + return {"ok": False, "error": str(e)} @public_router.get("/", response_model=errata_schema.ErrataRecord) diff --git a/alws/schemas/errata_schema.py b/alws/schemas/errata_schema.py index 76cdaef8..8ff1351f 100644 --- a/alws/schemas/errata_schema.py +++ b/alws/schemas/errata_schema.py @@ -3,8 +3,7 @@ from pydantic import BaseModel, field_validator -from alws.constants import ErrataReleaseStatus -from alws.models import ErrataPackageStatus +from alws.constants import ErrataReleaseStatus, ErrataPackageStatus class BaseErrataCVE(BaseModel): @@ -133,6 +132,7 @@ class CompactErrataRecord(BaseModel): class CreateErrataResponse(BaseModel): ok: bool + error: Optional[str] = None class ChangeErrataPackageStatusResponse(BaseModel): diff --git a/tests/fixtures/errata.py b/tests/fixtures/errata.py index 1895426d..9d5703fa 100644 --- a/tests/fixtures/errata.py +++ b/tests/fixtures/errata.py @@ -4,7 +4,8 @@ import pytest from sqlalchemy.ext.asyncio import AsyncSession -from alws.crud.errata import create_errata_record +from alws.crud.errata import create_errata_record, create_new_errata_record +from alws.dramatiq.errata import create_new_errata from alws.schemas.errata_schema import BaseErrataRecord @@ -107,15 +108,26 @@ async def create_errata( async_session: AsyncSession, errata_create_payload: typing.Dict[str, typing.Any], ): - await create_errata_record( - async_session, - BaseErrataRecord(**errata_create_payload), - ) + await create_errata_record(errata_create_payload) await async_session.commit() yield await async_session.rollback() +@pytest.fixture +async def create_errata_dramatiq( + async_session: AsyncSession, + errata_create_payload: typing.Dict[str, typing.Any], + monkeypatch, +): + async def func(*args, **kwargs): + await create_new_errata_record(errata_create_payload) + + monkeypatch.setattr(create_new_errata, "send", func) + await create_new_errata.send(errata_create_payload) + await async_session.commit() + yield + @pytest.fixture def pulp_updateinfos(): return [ diff --git a/tests/test_api/test_errata.py b/tests/test_api/test_errata.py index 9797f4ab..009c66ef 100644 --- a/tests/test_api/test_errata.py +++ b/tests/test_api/test_errata.py @@ -1,7 +1,4 @@ -from datetime import datetime - import pytest - from tests.mock_classes import BaseAsyncTestCase @@ -35,7 +32,9 @@ async def test_get_updateinfo_xml( async def test_list_errata_all_records( self, errata_create_payload, + create_errata_dramatiq ): + response = await self.make_request("get", "/api/v1/errata/all/") errata = response.json() assert (