Move issues to the correct statuses (#728)

* Notify about testing status

Resolves: AlmaLinux/build-system#70

* Move create issue functionality in to web server

Resolves: AlmaLinux/build-system#193

* Move issue to the "Building" section

Resolves: AlmaLinux/build-system#185

* Move issue to the "Released" section

Resolves: AlmaLinux/build-system#191

* Add second type of issues from git-updater

alws/config.py

@@ -68,6 +68,12 @@ class Settings(BaseSettings):
     sentry_dsn: Optional[str] = None
     sentry_traces_sample_rate: float = 0.2
 
+    github_integration_enabled: bool = False
+    github_token: Optional[str] = None
+    github_organization_name: str = 'AlmaLinux'
+    github_project_number: Optional[int] = None
+    github_default_repository_name: str = 'updates'
+
     @property
     def codenotary_enabled(self) -> bool:
         return bool(self.immudb_username) and bool(self.immudb_password)

alws/constants.py

@@ -187,6 +187,14 @@ class SignStatusEnum(enum.IntEnum):
     WRONG_SIGNATURE = 4
 
 
+class GitHubIssueStatus(enum.Enum):
+    TODO = "Todo"
+    DEVELOPMENT = "In Development"
+    BUILDING = "Building"
+    TESTING = "Testing"
+    RELEASED = "Released"
+
+
 class BeholderMatchMethod(enum.Enum):
     EXACT = "exact"
     CLOSEST = "closest"

alws/crud/build_node.py

@@ -13,7 +13,11 @@
 
 from alws import models
 from alws.config import settings
-from alws.constants import BuildTaskStatus, ErrataPackageStatus
+from alws.constants import (
+    BuildTaskStatus,
+    ErrataPackageStatus,
+    GitHubIssueStatus,
+)
 from alws.errors import (
     ArtifactChecksumError,
     ArtifactConversionError,
@@ -23,6 +27,12 @@
 )
 from alws.schemas import build_node_schema
 from alws.schemas.build_node_schema import BuildDoneArtifact
+from alws.utils.github_integration_helper import (
+    find_issues_by_build_id,
+    find_issues_by_record_id,
+    get_github_client,
+    move_issues,
+)
 from alws.utils.modularity import IndexWrapper, RpmArtifact
 from alws.utils.multilib import MultilibProcessor
 from alws.utils.noarch import save_noarch_packages
@@ -323,6 +333,7 @@ async def get_srpm_artifact_by_build_task_id(
 async def __process_rpms(
     db: AsyncSession,
     pulp_client: PulpClient,
+    build_id: int,
     task_id: int,
     task_arch: str,
     task_artifacts: list,
@@ -410,6 +421,7 @@ def append_errata_package(_, errata_package, artifact, rpm_info):
         for href, _, artifact in processed_packages
     ]
     rpms_info = get_rpm_packages_info(rpms)
+    errata_record_ids = set()
     for build_task_artifact in rpms:
         rpm_info = rpms_info[build_task_artifact.href]
         if rpm_info["arch"] != "src":
@@ -454,6 +466,31 @@ def append_errata_package(_, errata_package, artifact, rpm_info):
                 build_task_artifact,
                 rpm_info,
             )
+            errata_record_ids.add(errata_package.errata_record_id)
+    if settings.github_integration_enabled:
+        try:
+            github_client = await get_github_client()
+            issues = await find_issues_by_record_id(
+                github_client=github_client,
+                record_ids=list(errata_record_ids),
+            )
+            issues.extend(
+                await find_issues_by_build_id(
+                    github_client=github_client,
+                    build_ids=[build_id],
+                )
+            )
+            if issues:
+                await move_issues(
+                    github_client=github_client,
+                    issues=issues,
+                    status=GitHubIssueStatus.TESTING,
+                )
+        except Exception as err:
+            logging.exception(
+                "Cannot move issue to the Testing section: %s",
+                err,
+            )
 
     # we need to put source RPM in module as well, but it can be skipped
     # because it's built before
@@ -654,6 +691,7 @@ def _get_srpm_name(
     rpm_entries = await __process_rpms(
         db,
         pulp_client,
+        build_task.build_id,
         build_task.id,
         build_task.arch,
         rpm_artifacts,
@@ -780,10 +818,10 @@ async def __update_built_srpm_url(
     ):
         # Set the error field to describe the reason why they are fast failing
         fast_fail_msg = (
-            f"Fast failed: SRPM build failed in the initial "
+            "Fast failed: SRPM build failed in the initial "
             f"architecture ({build_task.arch}). "
-            f"Please refer to the initial architecture build "
-            f"logs for more information about the failure."
+            "Please refer to the initial architecture build "
+            "logs for more information about the failure."
         )
         update_query = (
             update(models.BuildTask)
@@ -860,9 +898,9 @@ async def fast_fail_other_tasks_by_ref(
     if len(build_tasks) != len(uncompleted_tasks):
         return
     fast_fail_msg = (
-        f"Fast failed: build processing failed in the initial "
+        "Fast failed: build processing failed in the initial "
         f"architecture ({current_task.arch}). Please refer to the initial "
-        f"architecture build logs for more information about the failure."
+        "architecture build logs for more information about the failure."
     )
     uncompleted_tasks_ids = [task.id for task in uncompleted_tasks]
     await db.execute(

alws/crud/errata.py

@@ -29,6 +29,7 @@
     ErrataPackageStatus,
     ErrataReferenceType,
     ErrataReleaseStatus,
+    GitHubIssueStatus,
 )
 from alws.dependencies import get_db, get_pulp_db
 from alws.pulp_models import (
@@ -50,6 +51,12 @@
     get_oval_title,
     get_verbose_errata_title,
 )
+from alws.utils.github_integration_helper import (
+    create_github_issue,
+    find_issues_by_record_id,
+    get_github_client,
+    move_issues,
+)
 from alws.utils.parsing import clean_release, parse_rpm_nevra
 from alws.utils.pulp_client import PulpClient
 from alws.utils.pulp_utils import (
@@ -505,7 +512,7 @@ async def get_matching_albs_packages(
     errata_package: models.NewErrataPackage,
     prod_repos_cache,
     module,
-) -> List[models.NewErrataToALBSPackage]:
+) -> List[models.ErrataToALBSPackage]:
     items_to_insert = []
     # We're going to check packages that match name-version-clean_release
     # Note that clean_release doesn't include the .module... str, we match:
@@ -537,6 +544,24 @@ async def get_matching_albs_packages(
         errata_package.source_srpm = src_nevra.name
         items_to_insert.append(mapping)
         errata_package.albs_packages.append(mapping)
+        if settings.github_integration_enabled:
+            try:
+                github_client = await get_github_client()
+                issues = await find_issues_by_record_id(
+                    github_client,
+                    [errata_package.errata_record_id],
+                )
+                if issues:
+                    await move_issues(
+                        github_client=github_client,
+                        issues=issues,
+                        status=GitHubIssueStatus.RELEASED,
+                    )
+            except Exception as err:
+                logging.exception(
+                    "Cannot move issue to the Released section: %s",
+                    err,
+                )
         return items_to_insert
 
     # If we couldn't find any pkg in production repos
@@ -617,6 +642,7 @@ async def create_errata_record(db: AsyncSession, errata: BaseErrataRecord):
     )
     platform = platform.scalars().first()
     items_to_insert = []
+    original_id = errata.id
 
     # Rebranding RHEL -> AlmaLinux
     for key in ("description", "title"):
@@ -674,6 +700,25 @@ async def create_errata_record(db: AsyncSession, errata: BaseErrataRecord):
     )
     items_to_insert.append(db_errata)
 
+    if settings.github_integration_enabled:
+        try:
+            github_client = await get_github_client()
+            await create_github_issue(
+                client=github_client,
+                title=errata.title,
+                description=errata.description,
+                advisory_id=alma_errata_id,
+                original_id=original_id,
+                platform_name=platform.name,
+                severity=errata.severity,
+                packages=errata.packages,
+            )
+        except Exception as err:
+            logging.exception(
+                "Cannot create GitHub issue: %s",
+                err,
+            )
+
     # References
     self_ref_exists = False
     for ref in errata.references:
@@ -1479,6 +1524,24 @@ async def bulk_errata_records_release(records_ids: List[str]):
                 missing_pkg_names=missing_pkg_names,
                 force_flag=False,
             )
+            if settings.github_integration_enabled:
+                try:
+                    github_client = await get_github_client()
+                    issues = await find_issues_by_record_id(
+                        github_client,
+                        [db_record.id],
+                    )
+                    if issues:
+                        await move_issues(
+                            github_client=github_client,
+                            issues=issues,
+                            status=GitHubIssueStatus.RELEASED,
+                        )
+                except Exception as err:
+                    logging.exception(
+                        "Cannot move issue to the Released section: %s",
+                        err,
+                    )
             if not tasks:
                 continue
             repos_to_publish.extend(repo_mapping.keys())

alws/dramatiq/build.py

@@ -10,7 +10,12 @@
 
 from alws import models
 from alws.build_planner import BuildPlanner
-from alws.constants import DRAMATIQ_TASK_TIMEOUT, BuildTaskStatus
+from alws.config import settings
+from alws.constants import (
+    DRAMATIQ_TASK_TIMEOUT,
+    BuildTaskStatus,
+    GitHubIssueStatus,
+)
 from alws.crud import build_node as build_node_crud
 from alws.crud import test
 from alws.database import SyncSession
@@ -25,6 +30,12 @@
     SrpmProvisionError,
 )
 from alws.schemas import build_node_schema, build_schema
+from alws.utils.github_integration_helper import (
+    find_issues_by_repo_name,
+    get_github_client,
+    move_issues,
+    set_build_id_to_issues,
+)
 
 __all__ = ['start_build', 'build_done']
 
@@ -100,6 +111,37 @@ async def _start_build(build_id: int, build_request: build_schema.BuildCreate):
             db.commit()
         db.close()
 
+    if settings.github_integration_enabled:
+        try:
+            github_client = await get_github_client()
+            repos = set()
+            for task in build_request.tasks:
+                if isinstance(task, build_schema.BuildTaskModuleRef):
+                    repos.add(f"module {task.module_name}")
+                    continue
+
+                repos.add(f"{task.url} {task.git_ref}")
+            issues = await find_issues_by_repo_name(
+                github_client=github_client,
+                repo_names=list(repos),
+            )
+            if issues:
+                await set_build_id_to_issues(
+                    github_client=github_client,
+                    issues=issues,
+                    build_id=build_id,
+                )
+                await move_issues(
+                    github_client=github_client,
+                    issues=issues,
+                    status=GitHubIssueStatus.BUILDING,
+                )
+        except Exception as err:
+            logging.exception(
+                "Cannot move issue to the Building section: %s",
+                err,
+            )
+
 
 async def _build_done(request: build_node_schema.BuildDone):
     async for db in get_db():

alws/release_planner.py

@@ -22,6 +22,7 @@
     BeholderMatchMethod,
     ErrataPackageStatus,
     ErrataReleaseStatus,
+    GitHubIssueStatus,
     PackageNevra,
     ReleasePackageTrustness,
     ReleaseStatus,
@@ -44,6 +45,11 @@
 from alws.schemas import release_schema
 from alws.utils.beholder_client import BeholderClient
 from alws.utils.debuginfo import clean_debug_name, is_debuginfo_rpm
+from alws.utils.github_integration_helper import (
+    find_issues_by_build_id,
+    get_github_client,
+    move_issues,
+)
 from alws.utils.measurements import class_measure_work_time_async
 from alws.utils.modularity import IndexWrapper, ModuleWrapper
 from alws.utils.parsing import get_clean_distr_name
@@ -576,6 +582,24 @@ async def commit_release(
                 message += "\n".join(release_messages)
             release.status = ReleaseStatus.COMPLETED
             builds_released = True
+            if settings.github_integration_enabled:
+                try:
+                    github_client = await get_github_client()
+                    issues = await find_issues_by_build_id(
+                        github_client=github_client,
+                        build_ids=release.build_ids,
+                    )
+                    if issues:
+                        await move_issues(
+                            github_client=github_client,
+                            issues=issues,
+                            status=GitHubIssueStatus.RELEASED,
+                        )
+                except Exception as err:
+                    logging.exception(
+                        "Cannot move issue to the Released section: %s",
+                        err,
+                    )
 
         await self.db.execute(
             update(models.Build)

alws/routers/errata.py

@@ -5,6 +5,7 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from alws.auth import get_current_user
+from alws.config import settings
 from alws.constants import ErrataReleaseStatus
 from alws.crud import errata as errata_crud
 from alws.dependencies import get_db