diff --git a/data/allocations.csv b/data/allocations.csv index ded976f4..c7ace387 100644 --- a/data/allocations.csv +++ b/data/allocations.csv @@ -1,45 +1,55 @@ -start, end, cluster_name, resource_name, group_name, cpu_year, gpu_year, vcpu_year, vgpu_year, project_size, project_inodes, nearline_size +start, end, cluster_name, resource_name, group_name, cpu_year, gpu_year, rgu_year, vcpu_year, vgpu_year, project_size, project_inodes, nearline_size 2017-04-01, 2018-04-01, cedar, cedar-gpu, rrg-bengioy-ad, , 100, -2017-04-01, 2018-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , 50TB, 5e6, 15TB, +2017-04-01, 2018-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , , 50TB, 5e6, 15TB, 2018-04-01, 2019-04-01, cedar, cedar-gpu, rrg-bengioy-ad, , 100, -2018-04-01, 2019-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , 70TB, , , +2018-04-01, 2019-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , , 70TB, , , 2019-04-01, 2020-04-01, beluga, beluga-gpu, rrg-bengioy-ad, , 190, -2019-04-01, 2020-04-01, beluga, beluga-storage, rrg-bengioy-ad, , , , , 90TB, 5e6, 90TB, +2019-04-01, 2020-04-01, beluga, beluga-storage, rrg-bengioy-ad, , , , , , 90TB, 5e6, 90TB, 2020-04-01, 2021-04-01, cedar, cedar-gpu, rrg-bengioy-ad, , 130, -2020-04-01, 2021-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , 30TB, 2e6, , +2020-04-01, 2021-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , , 30TB, 2e6, , 2020-04-01, 2021-04-01, beluga, beluga-compute, rrg-bengioy-ad, 219, 2020-04-01, 2021-04-01, beluga, beluga-gpu, rrg-bengioy-ad, , 200, -2020-04-01, 2021-04-01, beluga, beluga-storage, rrg-bengioy-ad, , , , , 70TB, 5e6, 80TB, +2020-04-01, 2021-04-01, beluga, beluga-storage, rrg-bengioy-ad, , , , , , 70TB, 5e6, 80TB, 2021-04-01, 2022-04-01, graham, graham-gpu, rrg-bengioy-ad, , 60, 2021-04-01, 2022-04-01, cedar, cedar-gpu, rrg-bengioy-ad, , 40, -2021-04-01, 2022-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , 30TB, 2e6, , +2021-04-01, 2022-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , , 30TB, 2e6, , 2021-04-01, 2022-04-01, beluga, beluga-compute, rrg-bengioy-ad, 185, 2021-04-01, 2022-04-01, beluga, beluga-gpu, rrg-bengioy-ad, , 150, -2021-04-01, 2022-04-01, beluga, beluga-storage, rrg-bengioy-ad, , , , , 70TB, 5e6, 80TB, +2021-04-01, 2022-04-01, beluga, beluga-storage, rrg-bengioy-ad, , , , , , 70TB, 5e6, 80TB, 2021-04-01, 2022-04-01, narval, narval-compute, rrg-bengioy-ad, 185, 2021-04-01, 2022-04-01, narval, narval-gpu, rrg-bengioy-ad, , 75, 2022-04-01, 2023-04-01, graham, graham-compute, rrg-bengioy-ad, 34, 2022-04-01, 2023-04-01, cedar, cedar-compute, rrg-bengioy-ad, 34, 2022-04-01, 2023-04-01, cedar, cedar-gpu, rrg-bengioy-ad, , 138, -2022-04-01, 2023-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , 50TB, 5e6, , +2022-04-01, 2023-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , , 50TB, 5e6, , 2022-04-01, 2023-04-01, beluga, beluga-compute, rrg-bengioy-ad, 238, 2022-04-01, 2023-04-01, beluga, beluga-gpu, rrg-bengioy-ad, , 77, -2022-04-01, 2023-04-01, beluga, beluga-storage, rrg-bengioy-ad, , , , , 50TB, 5e6, 25TB, +2022-04-01, 2023-04-01, beluga, beluga-storage, rrg-bengioy-ad, , , , , , 50TB, 5e6, 25TB, 2022-04-01, 2023-04-01, narval, narval-compute, rrg-bengioy-ad, 34, 2022-04-01, 2023-04-01, narval, narval-gpu, rrg-bengioy-ad, , 185, -2022-04-01, 2023-04-01, narval, narval-storage, rrg-bengioy-ad, , , , , 75TB, 5e6, 25TB, +2022-04-01, 2023-04-01, narval, narval-storage, rrg-bengioy-ad, , , , , , 75TB, 5e6, 25TB, -2023-04-01, 2024-04-01, cedar, cedar-compute, rrg-bengioy-ad, 197, -2023-04-01, 2024-04-01, cedar, cedar-gpu, rrg-bengioy-ad, , 127, -2023-04-01, 2024-04-01, cedar, cedar-storage, rrg-bengioy-ad, , , , , 75TB, 5e6, 25TB, -2023-04-01, 2024-04-01, beluga, beluga-compute, rrg-bengioy-ad, 197, -2023-04-01, 2024-04-01, beluga, beluga-gpu, rrg-bengioy-ad, , 127, -2023-04-01, 2024-04-01, beluga, beluga-storage, rrg-bengioy-ad, , , , , 75TB, 5e6, 125TB, -2023-04-01, 2024-04-01, narval, narval-compute, rrg-bengioy-ad, 917, -2023-04-01, 2024-04-01, narval, narval-gpu, rrg-bengioy-ad, , 154, -2023-04-01, 2024-04-01, narval, narval-storage, rrg-bengioy-ad, , , , , 150TB, 5e6, 75TB, +2023-04-01, 2024-04-03, cedar, cedar-compute, rrg-bengioy-ad, 197, +2023-04-01, 2024-04-03, cedar, cedar-gpu, rrg-bengioy-ad, , 127, +2023-04-01, 2024-04-03, cedar, cedar-storage, rrg-bengioy-ad, , , , , , 75TB, 5e6, 25TB, +2023-04-01, 2024-04-03, beluga, beluga-compute, rrg-bengioy-ad, 197, +2023-04-01, 2024-04-03, beluga, beluga-gpu, rrg-bengioy-ad, , 127, +2023-04-01, 2024-04-03, beluga, beluga-storage, rrg-bengioy-ad, , , , , , 200TB, 5e6, 125TB, +2023-04-01, 2024-04-03, narval, narval-compute, rrg-bengioy-ad, 917, +2023-04-01, 2024-04-03, narval, narval-gpu, rrg-bengioy-ad, , 154, +2023-04-01, 2024-04-03, narval, narval-storage, rrg-bengioy-ad, , , , , , 150TB, 5e6, 75TB, + +2024-04-03, 2025-04-03, cedar, cedar-compute, rrg-bengioy-ad, 125, +2024-04-03, 2025-04-03, cedar, cedar-gpu, rrg-bengioy-ad, , 118, 305, +2024-04-03, 2025-04-03, cedar, cedar-storage, rrg-bengioy-ad, , , , , , 75TB, 5e6, 25TB, +2024-04-03, 2025-04-03, beluga, beluga-compute, rrg-bengioy-ad, 125, +2024-04-03, 2025-04-03, beluga, beluga-gpu, rrg-bengioy-ad, , 111, 250, +2024-04-03, 2025-04-03, beluga, beluga-storage, rrg-bengioy-ad, , , , , , 300TB, 5e6, 125TB, +2024-04-03, 2025-04-03, narval, narval-compute, rrg-bengioy-ad, 580, +2024-04-03, 2025-04-03, narval, narval-gpu, rrg-bengioy-ad, , 110, 440, +2024-04-03, 2025-04-03, narval, narval-storage, rrg-bengioy-ad, , , , , , 150TB, 5e6, 75TB, diff --git a/sarc/allocations/allocations.py b/sarc/allocations/allocations.py index 0ef9fe1b..61fc437a 100644 --- a/sarc/allocations/allocations.py +++ b/sarc/allocations/allocations.py @@ -8,12 +8,14 @@ from pydantic import ByteSize, validator from pydantic_mongo import AbstractRepository, ObjectIdField +from sarc.traces import trace_decorator from sarc.config import BaseModel, config, validate_date class AllocationCompute(BaseModel): gpu_year: Optional[int] = 0 cpu_year: Optional[int] = 0 + rgu_year: Optional[int] = 0 vcpu_year: Optional[int] = 0 vgpu_year: Optional[int] = 0 @@ -61,6 +63,7 @@ class AllocationsRepository(AbstractRepository[Allocation]): class Meta: collection_name = "allocations" + @trace_decorator() def add(self, allocation: Allocation): document = self.to_document(allocation) query_attrs = ["cluster_name", "resource_name", "group_name", "start", "end"] @@ -120,7 +123,7 @@ def allocation_key(allocation: Allocation): for allocation in allocations: key = allocation_key(allocation) if key in summaries: - for field in ["cpu_year", "gpu_year", "vcpu_year", "vgpu_year"]: + for field in ["cpu_year", "gpu_year", "rgu_year", "vcpu_year", "vgpu_year"]: setattr( summaries[key].resources.compute, field, diff --git a/sarc/cli/acquire/allocations.py b/sarc/cli/acquire/allocations.py index 9004d4bb..b6d7899a 100644 --- a/sarc/cli/acquire/allocations.py +++ b/sarc/cli/acquire/allocations.py @@ -1,10 +1,13 @@ from __future__ import annotations +import logging import csv from dataclasses import dataclass from datetime import datetime from pathlib import Path +from sarc.traces import trace_decorator + from sarc.allocations.allocations import ( Allocation, AllocationCompute, @@ -13,7 +16,10 @@ get_allocations_collection, ) +logger = logging.getLogger(__name__) + +@trace_decorator() def convert_csv_row_to_allocation( cluster_name: str, resource_name: str, @@ -22,6 +28,7 @@ def convert_csv_row_to_allocation( end: datetime, cpu_year: None | int = None, gpu_year: None | int = None, + rgu_year: None | int = None, vcpu_year: None | int = None, vgpu_year: None | int = None, project_size: None | str = None, @@ -38,6 +45,7 @@ def convert_csv_row_to_allocation( resources=AllocationRessources( compute=AllocationCompute( gpu_year=gpu_year, + rgu_year=rgu_year, cpu_year=cpu_year, vcpu_year=vcpu_year, vgpu_year=vgpu_year, @@ -58,6 +66,8 @@ def convert_csv_row_to_allocation( @dataclass class AcquireAllocations: file: Path + # Do not actually insert the data into the database + dry: bool def execute(self) -> int: collection = get_allocations_collection() @@ -76,10 +86,11 @@ def execute(self) -> int: try: allocation = convert_csv_row_to_allocation(**row) except Exception as e: # pylint: disable=broad-exception-caught - print(f"Skipping row: {row}") - print(e) + logger.warning(f"Skipping row: {row}") continue - collection.add(allocation) + logger.info(f"Adding allocation: {allocation}") + if not self.dry: + collection.add(allocation) return 0 diff --git a/tests/functional/allocations/allocations.csv b/tests/functional/allocations/allocations.csv index 3af14819..1647e1db 100644 --- a/tests/functional/allocations/allocations.csv +++ b/tests/functional/allocations/allocations.csv @@ -1,15 +1,15 @@ -start, end, cluster_name, resource_name, group_name, cpu_year, gpu_year, vcpu_year, vgpu_year, project_size, project_inodes, nearline_size -2017-04-01, 2018-04-01, fromage, fromage-gpu, rrg-bonhomme-ad, , 100, -2017-04-01, 2018-04-01, fromage, fromage-storage,rrg-bonhomme-ad, , , , , 50TB, 5e6, 15TB, +start, end, cluster_name, resource_name, group_name, cpu_year, gpu_year, rgu_year, vcpu_year, vgpu_year, project_size, project_inodes, nearline_size +2017-04-01, 2018-04-01, fromage, fromage-gpu, rrg-bonhomme-ad, , 100, 300, +2017-04-01, 2018-04-01, fromage, fromage-storage,rrg-bonhomme-ad, , , , , , 50TB, 5e6, 15TB, -2018-04-01, 2019-04-01, fromage, fromage-gpu, rrg-bonhomme-ad, , 100, -2018-04-01, 2019-04-01, fromage, fromage-storage,rrg-bonhomme-ad, , , , , 70TB, , , +2018-04-01, 2019-04-01, fromage, fromage-gpu, rrg-bonhomme-ad, , 100, 300, +2018-04-01, 2019-04-01, fromage, fromage-storage,rrg-bonhomme-ad, , , , , , 70TB, , , -2019-04-01, 2020-04-01, patate, patate-gpu, rrg-bonhomme-ad, , 190, -2019-04-01, 2020-04-01, patate, patate-storage, rrg-bonhomme-ad, , , , , 90TB, 5e6, 90TB, +2019-04-01, 2020-04-01, patate, patate-gpu, rrg-bonhomme-ad, , 190, 400, +2019-04-01, 2020-04-01, patate, patate-storage, rrg-bonhomme-ad, , , , , , 90TB, 5e6, 90TB, -2020-04-01, 2021-04-01, fromage, fromage-gpu, rrg-bonhomme-ad, , 130, -2020-04-01, 2021-04-01, fromage, fromage-storage,rrg-bonhomme-ad, , , , , 30TB, 5e6, , +2020-04-01, 2021-04-01, fromage, fromage-gpu, rrg-bonhomme-ad, , 130, 350, +2020-04-01, 2021-04-01, fromage, fromage-storage,rrg-bonhomme-ad, , , , , , 30TB, 5e6, , 2020-04-01, 2021-04-01, patate, patate-compute, rrg-bonhomme-ad, 219, -2020-04-01, 2021-04-01, patate, patate-gpu, rrg-bonhomme-ad, , 200, -2020-04-01, 2021-04-01, patate, patate-storage, rrg-bonhomme-ad, , , , , 70TB, 5e6, 80TB, \ No newline at end of file +2020-04-01, 2021-04-01, patate, patate-gpu, rrg-bonhomme-ad, , 200, 450, +2020-04-01, 2021-04-01, patate, patate-storage, rrg-bonhomme-ad, , , , , , 70TB, 5e6, 80TB, \ No newline at end of file diff --git a/tests/functional/allocations/factory.py b/tests/functional/allocations/factory.py index de136fa5..61f4b59c 100644 --- a/tests/functional/allocations/factory.py +++ b/tests/functional/allocations/factory.py @@ -16,6 +16,7 @@ def create_allocations(): "compute": { "cpu_year": None, "gpu_year": 100, + "rgu_year": 400, "vcpu_year": None, "vgpu_year": None, }, @@ -37,6 +38,7 @@ def create_allocations(): "compute": { "cpu_year": None, "gpu_year": None, + "rgu_year": None, "vcpu_year": None, "vgpu_year": None, }, @@ -58,6 +60,7 @@ def create_allocations(): "compute": { "cpu_year": None, "gpu_year": 100, + "rpu_year": 400, "vcpu_year": None, "vgpu_year": None, }, @@ -79,6 +82,7 @@ def create_allocations(): "compute": { "cpu_year": None, "gpu_year": None, + "rgu_year": None, "vcpu_year": None, "vgpu_year": None, }, @@ -100,6 +104,7 @@ def create_allocations(): "compute": { "cpu_year": None, "gpu_year": 190, + "rgu_year": 190, "vcpu_year": None, "vgpu_year": None, }, @@ -121,6 +126,7 @@ def create_allocations(): "compute": { "cpu_year": None, "gpu_year": None, + "rgu_year": None, "vcpu_year": None, "vgpu_year": None, }, @@ -142,6 +148,7 @@ def create_allocations(): "compute": { "cpu_year": None, "gpu_year": 130, + "rgu_year": 450, "vcpu_year": None, "vgpu_year": None, }, @@ -163,6 +170,7 @@ def create_allocations(): "compute": { "cpu_year": None, "gpu_year": None, + "rgu_year": None, "vcpu_year": None, "vgpu_year": None, }, @@ -184,6 +192,7 @@ def create_allocations(): "compute": { "cpu_year": 219, "gpu_year": None, + "rgu_year": None, "vcpu_year": None, "vgpu_year": None, }, @@ -205,6 +214,7 @@ def create_allocations(): "compute": { "cpu_year": None, "gpu_year": 200, + "rgu_year": 500, "vcpu_year": None, "vgpu_year": None, }, @@ -226,6 +236,7 @@ def create_allocations(): "compute": { "cpu_year": None, "gpu_year": None, + "rgu_year": None, "vcpu_year": None, "vgpu_year": None, },