Skip to content

Commit

Permalink
use prom::ToText for other prometheus usage
Browse files Browse the repository at this point in the history
  • Loading branch information
@wookietreiber
wookietreiber committed Jun 6, 2024
1 parent cb6cefa commit e83352e
Show file tree
Hide file tree
Showing 5 changed files with 171 additions and 188 deletions.
19 changes: 5 additions & 14 deletions src/bin/mmoxi/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ use std::path::PathBuf;

use anyhow::{anyhow, Context, Result};
use clap::ArgMatches;

use mmoxi::prom::ToText;

mod cli;
Expand Down Expand Up @@ -173,7 +174,7 @@ fn run_prom_pool_user_distribution(args: &ArgMatches) -> Result<()> {

let scope = args.get_one::<String>("scope");

let mut user_sizes = mmoxi::policy::pool_user_distribution::run(
let data = mmoxi::policy::pool_user_distribution::run(
device_or_dir,
pool,
fileset,
Expand All @@ -183,18 +184,7 @@ fn run_prom_pool_user_distribution(args: &ArgMatches) -> Result<()> {
scope,
)?;

let mut named_user_sizes = HashMap::with_capacity(user_sizes.len());

for (user, data) in user_sizes.drain() {
let user = mmoxi::user::by_uid(&user).unwrap_or(user);
named_user_sizes.insert(user, data);
}

mmoxi::prom::write_pool_user_distribution(
pool,
&named_user_sizes,
&mut output,
)?;
data.to_prom(&mut output)?;

Ok(())
}
Expand Down Expand Up @@ -283,7 +273,8 @@ fn run_prom_pool_usage(args: &ArgMatches) -> Result<()> {

let filesystems = mmoxi::pool::run_all(&names)?;

mmoxi::pool::to_prom(&filesystems, &mut output)
filesystems
.to_prom(&mut output)
.context("converting internal data to prometheus")?;

Ok(())
Expand Down
104 changes: 90 additions & 14 deletions src/policy/pool_user_distribution.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
use std::collections::HashMap;
use std::ffi::OsStr;
use std::fs::File;
use std::io::{self, BufRead, BufReader};
use std::io::{self, BufRead, BufReader, Write};
use std::ops::AddAssign;
use std::path::Path;
use std::process::{Command, Stdio};
Expand All @@ -24,14 +24,14 @@ use tempfile::{tempdir, tempdir_in};
/// - parsing `mmapplypolicy` output
pub fn run(
device_or_dir: impl AsRef<OsStr>,
pool: impl AsRef<str>,
pool: impl Into<String>,
fileset: Option<impl AsRef<str>>,
nodes: Option<impl AsRef<OsStr>>,
local_work_dir: Option<impl AsRef<Path>>,
global_work_dir: Option<impl AsRef<Path>>,
scope: Option<impl AsRef<str>>,
) -> Result<HashMap<String, Data>> {
let pool = pool.as_ref();
) -> Result<Data> {
let pool = pool.into();

let tmp = if let Some(ref local_work_dir) = local_work_dir {
tempdir_in(local_work_dir)?
Expand All @@ -43,7 +43,7 @@ pub fn run(
let prefix = tmp.path().join("pool-scanner");

let mut file = File::create(&policy)?;
write_policy(&mut file, pool, fileset)?;
write_policy(&mut file, &pool, fileset)?;
file.sync_all()?;

let mut command = Command::new("mmapplypolicy");
Expand Down Expand Up @@ -91,16 +91,92 @@ pub fn run(
})?;
let list = BufReader::new(list);

let user_sizes = sum(list)?;
let mut raw = sum(list)?;
let mut named_user_sizes = HashMap::with_capacity(raw.len());

Ok(user_sizes)
for (user, data) in raw.drain() {
let user = crate::user::by_uid(&user).unwrap_or(user);
named_user_sizes.insert(user, data);
}

let data = Data {
pool,
raw: named_user_sizes,
};

Ok(data)
}

/// The data structure returned by this module.
pub struct Data {
pool: String,
raw: HashMap<String, Summary>,
}

impl crate::prom::ToText for Data {
fn to_prom(&self, output: &mut impl Write) -> Result<()> {
let pool = &self.pool;

writeln!(
output,
"# HELP gpfs_pool_user_distribution_files GPFS pool files per user"
)?;

writeln!(output, "# TYPE gpfs_pool_user_distribution_files gauge")?;

for (user, data) in &self.raw {
writeln!(
output,
"gpfs_pool_user_distribution_files{{pool=\"{}\",user=\"{}\"}} {}",
pool, user, data.files,
)?;
}

writeln!(
output,
"# HELP gpfs_pool_user_distribution_file_size GPFS pool file size per user in bytes"
)?;

writeln!(
output,
"# TYPE gpfs_pool_user_distribution_file_size gauge"
)?;

for (user, data) in &self.raw {
writeln!(
output,
"gpfs_pool_user_distribution_file_size{{pool=\"{}\",user=\"{}\"}} {}",
pool, user, data.file_size,
)?;
}

writeln!(
output,
"# HELP gpfs_pool_user_distribution_allocated GPFS pool allocated storage per user in kilobytes"
)?;

writeln!(
output,
"# TYPE gpfs_pool_user_distribution_allocated gauge"
)?;

for (user, data) in &self.raw {
writeln!(
output,
"gpfs_pool_user_distribution_allocated{{pool=\"{}\",user=\"{}\"}} {}",
pool, user, data.kb_allocated,
)?;
}

Ok(())
}
}

/// Data collected via policy.
#[derive(
Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Default,
)]
pub struct Data {
pub struct Summary {
/// Returns the number of files.
pub files: u64,

Expand All @@ -111,19 +187,19 @@ pub struct Data {
pub kb_allocated: u64,
}

impl AddAssign for Data {
impl AddAssign for Summary {
fn add_assign(&mut self, rhs: Self) {
self.files += rhs.files;
self.file_size += rhs.file_size;
self.kb_allocated += rhs.kb_allocated;
}
}

fn sum<I>(input: I) -> Result<HashMap<String, Data>>
fn sum<I>(input: I) -> Result<HashMap<String, Summary>>
where
I: BufRead,
{
let mut user_sizes: HashMap<String, Data> = HashMap::default();
let mut user_sizes: HashMap<String, Summary> = HashMap::default();

for line in input.byte_lines() {
let line = line?;
Expand Down Expand Up @@ -157,7 +233,7 @@ where
.parse()
.with_context(|| format!("not a number: {kb_allocated}"))?;

*user_sizes.entry(user.into()).or_default() += Data {
*user_sizes.entry(user.into()).or_default() += Summary {
files: 1,
file_size,
kb_allocated,
Expand Down Expand Up @@ -224,7 +300,7 @@ mod tests {
user_sizes.next(),
Some((
"1000".into(),
Data {
Summary {
files: 6,
file_size: 322_255,
kb_allocated: 640,
Expand All @@ -236,7 +312,7 @@ mod tests {
user_sizes.next(),
Some((
"1001".into(),
Data {
Summary {
files: 6,
file_size: 455_067,
kb_allocated: 960,
Expand Down
72 changes: 33 additions & 39 deletions src/pool.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,82 +224,76 @@ fn parse_mmlspool_output(s: &str) -> Result<Vec<Pool>> {
Ok(pools)
}

/// Converts filesystems to prometheus metric format.
///
/// # Errors
///
/// This function uses [`writeln`] to write to `output`. It can only fail if
/// any of these [`writeln`] fails.
pub fn to_prom<Output: Write>(
filesystems: &[Filesystem],
output: &mut Output,
) -> Result<()> {
writeln!(
output,
"# HELP gpfs_fs_pool_total_kbytes GPFS pool size in kilobytes."
)?;
writeln!(output, "# TYPE gpfs_fs_pool_total_kbytes gauge")?;

for fs in filesystems {
for pool in &fs.pools {
if let Some(size) = &pool.data {
writeln!(
impl crate::prom::ToText for Vec<Filesystem> {
fn to_prom(&self, output: &mut impl Write) -> Result<()> {
writeln!(
output,
"# HELP gpfs_fs_pool_total_kbytes GPFS pool size in kilobytes."
)?;
writeln!(output, "# TYPE gpfs_fs_pool_total_kbytes gauge")?;

for fs in self {
for pool in &fs.pools {
if let Some(size) = &pool.data {
writeln!(
output,
"gpfs_fs_pool_total_kbytes{{fs=\"{}\",pool=\"{}\",type=\"data\"}} {}",
fs.name,
pool.name,
size.total_kb
)?;
}
}

if let Some(size) = &pool.meta {
writeln!(
if let Some(size) = &pool.meta {
writeln!(
output,
"gpfs_fs_pool_total_kbytes{{fs=\"{}\",pool=\"{}\",type=\"meta\"}} {}",
fs.name,
pool.name,
size.total_kb
)?;
}
}
}
}

writeln!(
output,
"# HELP gpfs_fs_pool_free_kbytes GPFS pool free kilobytes."
)?;
writeln!(output, "# TYPE gpfs_fs_pool_free_kbytes gauge")?;
writeln!(
output,
"# HELP gpfs_fs_pool_free_kbytes GPFS pool free kilobytes."
)?;
writeln!(output, "# TYPE gpfs_fs_pool_free_kbytes gauge")?;

for fs in filesystems {
for pool in &fs.pools {
if let Some(size) = &pool.data {
writeln!(
for fs in self {
for pool in &fs.pools {
if let Some(size) = &pool.data {
writeln!(
output,
"gpfs_fs_pool_free_kbytes{{fs=\"{}\",pool=\"{}\",type=\"data\"}} {}",
fs.name,
pool.name,
size.free_kb
)?;
}
}

if let Some(size) = &pool.meta {
writeln!(
if let Some(size) = &pool.meta {
writeln!(
output,
"gpfs_fs_pool_free_kbytes{{fs=\"{}\",pool=\"{}\",type=\"meta\"}} {}",
fs.name,
pool.name,
size.free_kb
)?;
}
}
}
}

Ok(())
Ok(())
}
}

#[cfg(test)]
mod tests {
use super::*;
use crate::prom::ToText;

#[test]
fn parse() {
Expand Down Expand Up @@ -395,7 +389,7 @@ mod tests {
};

let mut output = vec![];
to_prom(&[fs], &mut output).unwrap();
vec![fs].to_prom(&mut output).unwrap();

let metrics = std::str::from_utf8(output.as_slice()).unwrap();

Expand Down
Loading

0 comments on commit e83352e

Please sign in to comment.