Merge pull request #252 from centerforaisafety/250-billing-system

250 billing system

bastion.tf

@@ -295,7 +295,11 @@ resource "null_resource" "cluster" {
       region = var.region,
       tenancy_ocid = var.tenancy_ocid,
       api_fingerprint = var.api_fingerprint,
-      api_user_ocid = var.api_user_ocid
+      api_user_ocid = var.api_user_ocid,
+      billing = var.billing,
+      billing_mysql_db_admin_username = var.billing_mysql_db_admin_username,
+      billing_mysql_db_admin_password = var.billing_mysql_db_admin_password,
+      billing_mysql_ip = var.billing ? oci_mysql_mysql_db_system.billing_mysql_db_system[0].ip_address : ""
       })
 
     destination   = "/opt/oci-hpc/playbooks/inventory"
@@ -433,7 +437,11 @@ resource "null_resource" "cluster" {
       compute_username = var.compute_username,
       pam = var.pam,
       sacct_limits = var.sacct_limits, 
-      use_compute_agent = var.use_compute_agent
+      use_compute_agent = var.use_compute_agent,
+      billing = var.billing,
+      billing_mysql_db_admin_username = var.billing_mysql_db_admin_username,
+      billing_mysql_db_admin_password = var.billing_mysql_db_admin_password,
+      billing_mysql_ip = var.billing ? oci_mysql_mysql_db_system.billing_mysql_db_system[0].ip_address : ""
       })
 
     destination   = "/opt/oci-hpc/conf/variables.tf"

conf/variables.tpl

@@ -123,7 +123,6 @@ variable "ldap" { default = ${ldap} }
 variable "monitoring" { default = ${monitoring} }
 variable "autoscaling_monitoring" { default = ${autoscaling_monitoring} }
 
-
 variable "tags" { default = "##TAGS##" }
 variable "private_deployment" { default = ${private_deployment} }
 variable "use_multiple_ads" { default = ${use_multiple_ads} }
@@ -135,3 +134,8 @@ variable "log_vol" { default = "${log_vol}" }
 variable "redundancy" { default = "${redundancy}" }
 
 variable "instance_pool_ocpus_denseIO_flex" { default = "##OCPU##"}
+
+variable "billing" { default = "${billing}" }
+variable "billing_mysql_db_admin_username" { default = "${billing_mysql_db_admin_username}" }
+variable "billing_mysql_db_admin_password" { default = "${billing_mysql_db_admin_password}" }
+variable "billing_mysql_ip" { default = "$billing_mysql_ip" }

inventory.tpl

@@ -76,4 +76,8 @@ inst_prin = ${inst_prin}
 api_fingerprint = ${api_fingerprint}
 api_user_ocid = ${api_user_ocid}
 sacct_limits=${sacct_limits}
-use_compute_agent=${use_compute_agent}
+use_compute_agent=${use_compute_agent}
+billing=${billing}
+billing_mysql_db_admin_username=${billing_mysql_db_admin_username}
+billing_mysql_db_admin_password=${billing_mysql_db_admin_password}
+billing_mysql_ip=${billing_mysql_ip}

mysql.tf

@@ -13,4 +13,45 @@ resource "oci_mysql_mysql_db_system" "monitoring_mysql_db_system" {
     backup_policy {
         is_enabled = false
     }
+}
+
+resource "oci_mysql_mysql_db_system" "billing_mysql_db_system" {
+    count = var.billing ? 1 : 0
+
+    # Required
+    availability_domain = var.bastion_ad
+    compartment_id = var.targetCompartment
+    shape_name = var.billing_shape_name
+    subnet_id = local.subnet_id
+
+    # Optional
+    admin_password = var.billing_mysql_db_admin_password
+    admin_username = var.billing_mysql_db_admin_username
+    backup_policy {
+        is_enabled = true
+        # Point-In-Time Recovery
+        pitr_policy {
+            is_enabled = true
+        }
+        retention_in_days = "7"
+    }
+    description = "MySQL DB System for billing"
+    display_name = "billing"
+    mysql_version = "8.0.35"
+    port = "3306"
+    port_x = "33060"
+
+    is_highly_available = "true"
+    crash_recovery = "ENABLED"
+    data_storage_size_in_gb = "50"
+    deletion_policy {
+        automatic_backup_retention = "RETAIN"
+        final_backup = "REQUIRE_FINAL_BACKUP"
+        is_delete_protected = "true"
+    }
+
+    freeform_tags = {
+        "Template" = "Production",
+        "CreatedTime" = timestamp()
+    }
 }

playbooks/README.md

@@ -18,6 +18,10 @@ For the webhooks go to [slack bot app](https://app.slack.com/app-settings/T04PVJ
 
 To uninstall notifications run notifications_uninstall.yml. Note that this does not remove the MailProg entry from the slurm.conf. This won't cause problems but will be silently erroring in the slurmctld.log. So if you want to be clean you can remove that from the config.
 
+# Billing
+
+If the billing system has been deployed a final step needs to be performed before usage data will be collected. The cluster resource usage collection scripts in `/etc/oci-hpc/billing` need to be scheduled in crontab. Each collection script has certain requirements that need to be met before they can be used. More details can be found by reviewing the scripts or by running the scripts with the `-h` or `--help` options. Once the requirements have been satisfied, uncomment the entries in `crontab -e` to begin collecting usage data. 
+
 ## Passwordless SSH for Root User
 
 We provide playbooks to enable and disable passwordless SSH for the root user. This feature is typically required for automated administrative tasks such as software upgrades, for example with Weka. Caution: Enabling passwordless SSH for the root user poses significant security risks. Be sure to disable it as soon as it is no longer necessary.

playbooks/roles/billing/files/billing.sql

@@ -0,0 +1,90 @@
+CREATE DATABASE billing;
+
+CREATE TABLE IF NOT EXISTS billing.accounts (
+    account_id INT AUTO_INCREMENT PRIMARY KEY,
+    account_name VARCHAR(255) NOT NULL,
+    billing_details TEXT,
+    email VARCHAR(255),
+    billing_address TEXT,
+    created_date DATE NOT NULL DEFAULT (CURRENT_DATE),
+    archived BOOLEAN NOT NULL DEFAULT FALSE
+);
+
+CREATE TABLE IF NOT EXISTS billing.users (
+    user_id INT AUTO_INCREMENT PRIMARY KEY,
+    account_id INT NOT NULL,
+    user_name VARCHAR(255) NOT NULL,
+    email VARCHAR(255),
+    FOREIGN KEY (account_id) REFERENCES billing.accounts(account_id),
+    created_date DATE NOT NULL DEFAULT (CURRENT_DATE),
+    archived BOOLEAN NOT NULL DEFAULT FALSE
+);
+
+CREATE TABLE IF NOT EXISTS billing.measurement_units (
+    measurement_unit_id INT AUTO_INCREMENT PRIMARY KEY,
+    measurement_unit_name VARCHAR(50) NOT NULL,
+    measurement_unit_description TEXT
+);
+
+CREATE TABLE IF NOT EXISTS billing.resource_types (
+    resource_type_id INT AUTO_INCREMENT PRIMARY KEY,
+    resource_name VARCHAR(255) NOT NULL,
+    resource_description TEXT,
+    measurement_unit_id INT,
+    FOREIGN KEY (measurement_unit_id) REFERENCES billing.measurement_units(measurement_unit_id)
+);
+
+CREATE TABLE IF NOT EXISTS billing.resource_specifications (
+    resource_spec_id INT AUTO_INCREMENT PRIMARY KEY,
+    resource_type_id INT NOT NULL,
+    specification_name VARCHAR(255),
+    FOREIGN KEY (resource_type_id) REFERENCES billing.resource_types(resource_type_id)
+);
+
+CREATE TABLE IF NOT EXISTS billing.usage_records (
+    usage_id INT AUTO_INCREMENT PRIMARY KEY,
+    user_id INT NOT NULL,
+    resource_spec_id INT NOT NULL,
+    usage_start_time DATETIME NOT NULL,
+    usage_end_time DATETIME NOT NULL,
+    usage_amount BIGINT NOT NULL,
+    FOREIGN KEY (user_id) REFERENCES billing.users(user_id),
+    FOREIGN KEY (resource_spec_id) REFERENCES billing.resource_specifications(resource_spec_id)
+);
+
+CREATE TABLE IF NOT EXISTS billing.pricing (
+    pricing_id INT AUTO_INCREMENT PRIMARY KEY,
+    account_id INT NOT NULL,
+    resource_spec_id INT NOT NULL,
+    price_per_unit DECIMAL(10, 2) NOT NULL,
+    price_effective_date DATE NOT NULL,
+    price_end_date DATE,
+    FOREIGN KEY (account_id) REFERENCES billing.accounts(account_id),
+    FOREIGN KEY (resource_spec_id) REFERENCES billing.resource_specifications(resource_spec_id)
+);
+
+-- Insert units of measure
+INSERT INTO billing.measurement_units (measurement_unit_name, measurement_unit_description)
+VALUES 
+    ('bytes', 'Used for Filesystem, Network usage.'),
+    ('bytes per second', 'Used for RAM usage.'),
+    ('seconds', 'Used for GPU, CPU usage.');
+
+
+-- Insert resource types
+INSERT INTO billing.resource_types (resource_name, measurement_unit_id)
+VALUES
+	('GPU', 3),
+	('CPU', 3),
+	('RAM', 2),
+	('Filesystem', 1),
+    ('Network', 1);
+
+-- Insert resource specifications
+INSERT INTO billing.resource_specifications (resource_type_id, specification_name)
+VALUES
+	(1, 'A100'),
+	(5, 'Compute Node Egress'),
+	(4, 'Weka'),
+	(1, 'H100'),
+    (5, 'Login Node Egress');