diff --git a/log-api/index.php b/log-api/index.php deleted file mode 100644 index 5de10097..00000000 --- a/log-api/index.php +++ /dev/null @@ -1,73 +0,0 @@ - []); - -$root = "/var/log/ofborg/"; - -function abrt($msg) { - echo $msg; - exit; -} - -if (!is_dir($root)) { - abrt("root missing"); -} - -if (!isset($_SERVER['REQUEST_URI']) || empty($_SERVER['REQUEST_URI'])) { - abrt("uri missing"); -} - -$reqd = substr($_SERVER['REQUEST_URI'], strlen("/logs/")); -$req = realpath("$root/$reqd"); -$serve_root = "https://logs.ofborg.org/logfile/$reqd"; - -if ($req === false) { - abrt("absent"); -} - -if (strpos($req, $root) !== 0) { - abrt("bad path"); -} - -if (!is_dir($req)) { - abrt("non dir"); -} - -if ($handle = opendir($req)) { - while (false !== ($entry = readdir($handle))) { - if ($entry != "." && $entry != "..") { - if (is_dir($req . '/' . $entry)) { - abrt("dir found"); - } - - if (is_file($req . '/' . $entry)) { - if (substr($entry, -strlen(".metadata.json"),strlen(".metadata.json")) == ".metadata.json") { - $metadata = json_decode(file_get_contents($req . '/' . $entry), JSON_OBJECT_AS_ARRAY); - $attempt = $metadata['attempt_id']; - if (!isset($d['attempts'][$attempt])) { - $d['attempts'][$attempt] = []; - } - $d['attempts'][$attempt]['metadata'] = $metadata; - } elseif (substr($entry, -strlen(".result.json"),strlen(".result.json")) == ".result.json") { - $metadata = json_decode(file_get_contents($req . '/' . $entry), JSON_OBJECT_AS_ARRAY); - $attempt = $metadata['attempt_id']; - if (!isset($d['attempts'][$attempt])) { - $d['attempts'][$attempt] = []; - } - $d['attempts'][$attempt]['result'] = $metadata; - - } else { - if (!isset($d['attempts'][$entry])) { - $d['attempts'][$entry] = []; - } - $d['attempts'][$entry]['log_url'] = "$serve_root/$entry"; - } - } - } - } - closedir($handle); -} - - -echo json_encode($d); \ No newline at end of file diff --git a/ofborg/src/bin/logapi.rs b/ofborg/src/bin/logapi.rs new file mode 100644 index 00000000..680d961c --- /dev/null +++ b/ofborg/src/bin/logapi.rs @@ -0,0 +1,132 @@ +use std::{collections::HashMap, error::Error, path::PathBuf}; + +use hyper::{ + header::ContentType, + mime, + server::{Request, Response, Server}, + status::StatusCode, +}; +use ofborg::config; +use tracing::{error, info, warn}; + +#[derive(serde::Serialize, Default)] +struct Attempt { + metadata: Option, + result: Option, + log_url: Option, +} + +#[derive(serde::Serialize)] +struct LogResponse { + attempts: HashMap, +} + +fn main() -> Result<(), Box> { + ofborg::setup_log(); + + let arg = std::env::args() + .nth(1) + .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); + let Some(cfg) = config::load(arg.as_ref()).log_api_config else { + error!("No LogApi configuration found!"); + panic!(); + }; + + let threads = std::thread::available_parallelism() + .map(|x| x.get()) + .unwrap_or(1); + info!("Will listen on {} with {threads} threads", cfg.listen); + Server::http(cfg.listen)?.handle_threads( + move |req: Request, mut res: Response| { + if req.method != hyper::Get { + *res.status_mut() = StatusCode::MethodNotAllowed; + return; + } + + let uri = req.uri.to_string(); + let Some(reqd) = uri.strip_prefix("/logs/").map(ToOwned::to_owned) else { + *res.status_mut() = StatusCode::NotFound; + let _ = res.send(b"invalid uri"); + return; + }; + let path: PathBuf = [&cfg.logs_path, &reqd].iter().collect(); + let Ok(path) = std::fs::canonicalize(&path) else { + *res.status_mut() = StatusCode::NotFound; + let _ = res.send(b"absent"); + return; + }; + let Ok(iter) = std::fs::read_dir(path) else { + *res.status_mut() = StatusCode::NotFound; + let _ = res.send(b"non dir"); + return; + }; + + let mut attempts = HashMap::::new(); + for e in iter { + let Ok(e) = e else { continue }; + let e_metadata = e.metadata(); + if e_metadata.as_ref().map(|v| v.is_dir()).unwrap_or(true) { + *res.status_mut() = StatusCode::InternalServerError; + let _ = res.send(b"dir found"); + return; + } + + if e_metadata.as_ref().map(|v| v.is_file()).unwrap_or_default() { + let Ok(file_name) = e.file_name().into_string() else { + warn!("entry filename is not a utf-8 string: {:?}", e.file_name()); + continue; + }; + + if file_name.ends_with(".metadata.json") || file_name.ends_with(".result.json") + { + let Ok(file) = std::fs::File::open(e.path()) else { + warn!("could not open file: {file_name}"); + continue; + }; + let Ok(json) = serde_json::from_reader::<_, serde_json::Value>(file) else { + warn!("file is not a valid json file: {file_name}"); + continue; + }; + let Some(attempt_id) = json + .get("attempt_id") + .and_then(|v| v.as_str()) + .map(ToOwned::to_owned) + else { + warn!("attempt_id not found in file: {file_name}"); + continue; + }; + let attempt_obj = attempts + .entry(attempt_id) + .or_insert_with(Attempt::default); + if file_name.ends_with(".metadata.json") { + attempt_obj.metadata = Some(json); + } else { + attempt_obj.result = Some(json); + } + } else { + let attempt_obj = attempts + .entry(file_name.clone()) + .or_insert_with(Attempt::default); + attempt_obj.log_url = + Some(format!("{}/{reqd}/{file_name}", &cfg.serve_root)); + } + } + } + + *res.status_mut() = StatusCode::Ok; + res.headers_mut() + .set::(hyper::header::ContentType(mime::Mime( + mime::TopLevel::Application, + mime::SubLevel::Json, + Vec::new(), + ))); + let _ = res.send( + serde_json::to_string(&LogResponse { attempts }) + .unwrap_or_default() + .as_bytes(), + ); + }, + threads, + )?; + Ok(()) +} diff --git a/ofborg/src/config.rs b/ofborg/src/config.rs index 3a578058..02f32fa8 100644 --- a/ofborg/src/config.rs +++ b/ofborg/src/config.rs @@ -17,6 +17,8 @@ use tracing::{debug, error, info, warn}; pub struct Config { /// Configuration for the webhook receiver pub github_webhook_receiver: Option, + /// Configuration for the logapi receiver + pub log_api_config: Option, /// Configuration for the evaluation filter pub evaluation_filter: Option, /// Configuration for the GitHub comment filter @@ -44,6 +46,26 @@ pub struct GithubWebhookConfig { pub rabbitmq: RabbitMqConfig, } +fn default_logs_path() -> String { + "/var/log/ofborg".into() +} + +fn default_serve_root() -> String { + "https://logs.ofborg.org/logfile".into() +} + +/// Configuration for logapi +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct LogApiConfig { + /// Listen host/port + pub listen: String, + #[serde(default = "default_logs_path")] + pub logs_path: String, + #[serde(default = "default_serve_root")] + pub serve_root: String, +} + /// Configuration for the evaluation filter #[derive(Serialize, Deserialize, Debug)] #[serde(deny_unknown_fields)] @@ -168,12 +190,23 @@ impl Config { } pub fn github(&self) -> Github { - let token = std::fs::read_to_string(self.github_app.clone().expect("No GitHub app configured").oauth_client_secret_file) - .expect("Couldn't read from GitHub app token"); + let token = std::fs::read_to_string( + self.github_app + .clone() + .expect("No GitHub app configured") + .oauth_client_secret_file, + ) + .expect("Couldn't read from GitHub app token"); let token = token.trim(); Github::new( "github.com/NixOS/ofborg", - Credentials::Client(self.github_app.clone().expect("No GitHub app configured").oauth_client_id, token.to_owned()), + Credentials::Client( + self.github_app + .clone() + .expect("No GitHub app configured") + .oauth_client_id, + token.to_owned(), + ), ) .expect("Unable to create a github client instance") }