From cf8fe6a22404d997bcc2af63668d4d38ecaea0db Mon Sep 17 00:00:00 2001 From: "James A. Overton" Date: Wed, 28 Jun 2023 20:00:38 +0000 Subject: [PATCH 1/4] First pass at actions and git support --- Cargo.toml | 6 +- Makefile | 50 ++++++++++ src/config.rs | 22 +++++ src/get.rs | 115 ++++++++++++++++++++- src/resources/action.html | 101 +++++++++++++++++++ src/resources/page.html | 51 +++++++++- src/serve.rs | 203 +++++++++++++++++++++++++++++++++----- 7 files changed, 520 insertions(+), 28 deletions(-) create mode 100644 src/resources/action.html diff --git a/Cargo.toml b/Cargo.toml index 2b80453..aa5b130 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ sqlx = { version = "0.6", features = [ "runtime-async-std-rustls", "any", "postg tabwriter = { version = "1.2.1" } tokio = { version = "1.22.0", features = ["full"] } tokio-test = "0.4.2" -toml = { version = "0.5.9" } +toml = "0.7.5" tracing = "0.1.37" tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } urlencoding = "2.1.2" @@ -32,6 +32,10 @@ async-recursion = "1.0.2" reqwest = { version = "0.11.14", features = ["blocking"] } itertools = "0.10.5" thiserror = "1.0" +indexmap = { version = "2.0.0", features = ["serde"] } +git2 = "0.17.2" +chrono = "0.4.26" +ansi-to-html = "0.1.3" [dependencies.ontodev_hiccup] git = "https://github.com/ontodev/hiccup.rs" diff --git a/Makefile b/Makefile index 2094fa9..f5586c4 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,9 @@ format: build: cargo build --release +build/: + mkdir -p $@ + target/release/nanobot: src/ cargo build --release @@ -63,3 +66,50 @@ penguins: target/release/nanobot examples/penguins/ && python3 generate.py \ && ../../$< init \ && ../../$< serve + +build/synthea.zip: | build + curl -L -o build/synthea.zip "https://synthetichealth.github.io/synthea-sample-data/downloads/synthea_sample_data_csv_apr2020.zip" + +build/synthea/: build/synthea.zip examples/synthea/ + mkdir -p build/synthea/src/data + cp -r examples/synthea/* build/synthea/ + unzip $< -d build/synthea/ + sed 's/,/ /g' build/synthea/csv/patients.csv > build/synthea/src/data/patients.tsv + sed 's/,/ /g' build/synthea/csv/observations.csv > build/synthea/src/data/observations.tsv + +# && ~/valve.rs/target/release/ontodev_valve src/schema/table.tsv .nanobot.db +.PHONY: synthea +synthea: target/release/nanobot + rm -rf build/synthea/ + make build/synthea/ + cd build/synthea/ \ + && time ../../$< init \ + && ../../$< serve + +TODAY := $(shell date +%Y-%m-%d) +ARCH := x86_64-unknown-linux-musl +TARGET := build/nanobot-$(ARCH) + +target/$(ARCH)/release/nanobot: src + docker pull clux/muslrust:stable + docker run \ + -v cargo-cache:/root/.cargo/registry \ + -v $$PWD:/volume \ + --rm -t clux/muslrust:stable \ + cargo build --release + +.PHONY: musl +musl: target/$(ARCH)/release/nanobot src/ | build/ + +.PHONY: upload +upload: target/$(ARCH)/release/nanobot | build/ + cp $< $(TARGET) + gh release upload --clobber v$(TODAY) $(TARGET) + +.PHONY: release +release: target/$(ARCH)/release/nanobot | build/ + cp $< $(TARGET) + gh release create --draft --prerelease \ + --title "$(TODAY) Alpha Release" \ + --generate-notes \ + v$(TODAY) $(TARGET) diff --git a/src/config.rs b/src/config.rs index 667cfc5..53917cb 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,3 +1,4 @@ +use indexmap::map::IndexMap; use ontodev_valve::{ get_compiled_datatype_conditions, get_compiled_rule_conditions, get_parsed_structure_conditions, valve, valve_grammar::StartParser, ColumnRule, @@ -22,6 +23,7 @@ pub struct Config { pub valve_path: String, pub valve: Option, pub template_path: Option, + pub actions: IndexMap, } #[derive(Clone, Debug, Serialize, Deserialize, PartialOrd, Ord, PartialEq, Eq)] @@ -53,6 +55,7 @@ pub struct TomlConfig { pub database: Option, pub valve: Option, pub templates: Option, + pub actions: Option>, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -106,6 +109,23 @@ pub struct TemplatesConfig { pub path: Option, } +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +pub struct ActionConfig { + pub label: String, + pub inputs: Option>, + pub commands: Vec>, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +pub struct InputConfig { + pub name: String, + pub label: String, + pub value: Option, + pub default: Option, + pub placeholder: Option, + pub test: Option, +} + pub type SerdeMap = serde_json::Map; pub const DEFAULT_TOML: &str = "[nanobot] @@ -155,6 +175,7 @@ impl Config { None => None, } }, + actions: user.actions.unwrap_or_default(), }; Ok(config) @@ -267,5 +288,6 @@ pub fn to_toml(config: &Config) -> TomlConfig { templates: Some(TemplatesConfig { path: config.template_path.clone(), }), + actions: Some(config.actions.clone()), } } diff --git a/src/get.rs b/src/get.rs index 105810c..9bbada8 100644 --- a/src/get.rs +++ b/src/get.rs @@ -7,7 +7,9 @@ use crate::sql::{ get_count_from_pool, get_message_counts_from_pool, get_table_from_pool, get_total_from_pool, LIMIT_DEFAULT, LIMIT_MAX, }; +use chrono::prelude::{DateTime, Utc}; use enquote::unquote; +use git2::Repository; use minijinja::{Environment, Source}; use ontodev_sqlrest::{Direction, OrderByColumn, Select}; use ontodev_valve::get_sql_type_from_global_config; @@ -16,11 +18,14 @@ use serde_json::{json, to_string_pretty, Map, Value}; use std::collections::HashMap; use std::error::Error; use std::fmt; +use std::fs; use std::io::Write; use std::path::Path; use tabwriter::TabWriter; use urlencoding::decode; +pub type SerdeMap = serde_json::Map; + #[derive(Debug)] pub struct GetError { details: String, @@ -44,15 +49,33 @@ impl Error for GetError { } } +impl From for GetError { + fn from(error: String) -> GetError { + GetError::new(error) + } +} + +impl From for GetError { + fn from(error: std::io::Error) -> GetError { + GetError::new(format!("{:?}", error)) + } +} + impl From for GetError { fn from(error: sqlx::Error) -> GetError { GetError::new(format!("{:?}", error)) } } -impl From for GetError { - fn from(error: String) -> GetError { - GetError::new(error) +impl From for GetError { + fn from(error: git2::Error) -> GetError { + GetError::new(format!("{:?}", error)) + } +} + +impl From for GetError { + fn from(error: std::time::SystemTimeError) -> GetError { + GetError::new(format!("{:?}", error)) } } @@ -857,6 +880,8 @@ async fn get_page( "select": select, "select_params": select2.to_params().unwrap_or_default(), "elapsed": elapsed, + "actions": get_action_map(&config).unwrap_or_default(), + "repo": get_repo_details().unwrap_or_default(), }, "table": this_table, "column": column_map, @@ -866,6 +891,84 @@ async fn get_page( Ok(result) } +pub fn get_action_map(config: &Config) -> Result { + let action_map: SerdeMap = config + .actions + .iter() + .map(|(k, v)| (k.into(), v.clone().label.into())) + .collect(); + Ok(action_map) +} + +pub fn get_repo_details() -> Result { + let mut result = SerdeMap::new(); + + let repo = Repository::open_from_env().expect("Couldn't open repository"); + let head = match repo.head() { + Ok(head) => Some(head), + Err(e) => return Err(GetError::new(e.to_string())), + }; + let head = head + .as_ref() + .and_then(|h| h.shorthand()) + .unwrap_or_default(); + let local = repo.find_branch(&head, git2::BranchType::Local)?; + tracing::debug!("GIT got local: {head}, {:?}", local.name()?); + result.insert("head".into(), head.into()); + result.insert("local".into(), local.name()?.into()); + + let upstream = local.upstream(); + if let Ok(upstream) = upstream { + let (ahead, behind) = repo.graph_ahead_behind( + local.get().target().unwrap(), + upstream.get().target().unwrap(), + )?; + let remote = repo.find_remote("origin")?; + let remote_url = format!( + "{}/tree/{}", + remote + .url() + .ok_or("No URL?") + .unwrap_or_default() + .trim_end_matches(".git"), + upstream + .name()? + .unwrap_or_default() + .trim_start_matches("origin/") + ); + tracing::debug!( + "GIT got remote: {ahead} ahead {behind} behind {:?}, {remote_url}", + upstream.name()? + ); + result.insert("upstream".into(), upstream.name()?.into()); + result.insert("remote_url".into(), remote_url.into()); + result.insert("ahead".into(), ahead.into()); + result.insert("behind".into(), behind.into()); + } else { + tracing::debug!("GIT no upstream branch"); + } + + // https://github.com/ontodev/nanobot.rs/tree/refine-ui + let mut opts = git2::StatusOptions::new(); + opts.include_ignored(false); + opts.include_untracked(false); + opts.exclude_submodules(true); + if let Ok(statuses) = repo.statuses(Some(&mut opts)) { + let uncommitted = statuses.len() > 0; + tracing::debug!("GIT got status: {uncommitted}"); + result.insert("uncommitted".into(), uncommitted.into()); + } + let path = repo.path().join("FETCH_HEAD"); + tracing::debug!("GIT repo path: {path:?} {}", path.is_file()); + if path.is_file() { + let dt: DateTime = fs::metadata(path)?.modified()?.clone().into(); + let fetched = format!("{}", dt.to_rfc3339()); + result.insert("fetched".into(), fetched.into()); + } + + Ok(result) +} + fn value_rows_to_text(rows: &Vec>) -> Result { // This would be nicer with map, but I got weird borrowing errors. let mut lines: Vec = vec![]; @@ -951,6 +1054,7 @@ pub fn page_to_html(config: &Config, template: &str, page: &Value) -> Result Result +
+ +
+ +{% endfor %} +{% endif %} + +{% if "error" in messages %} +{% for msg in messages["error"] %} +
+
+ +
+
+{% endfor %} +{% endif %} + +{% if "warn" in messages %} +{% for msg in messages["warn"] %} +
+
+ +
+
+{% endfor %} +{% endif %} + +{% if "info" in messages %} +{% for msg in messages["info"] %} +
+
+ +
+
+{% endfor %} +{% endif %} +{% endif %} + +{% if inputs %} +
+

Input required for '{{ action.label }}'

+ {% for input in inputs %} +
+
+ +
+
+ +
+
+ {% endfor %} +
+
+ +
+
+
+{% endif %} + +{% if results %} +

Results for for '{{ action.label }}'

+{% for result in results %} +
+
+ 
> {{ result.command }}
+
+
+ {% if result.stdout %}
{{ result.stdout|safe }}
{% endif %} + {% if result.status != 0 %} +
Exit status: {{ result.status }}
+ {% endif %} + {% if result.stderr %} +
Errors:
+ 
{{ result.stderr|safe }}
+ {% endif %} +
+
+{% endfor %} +{% endif %} + +{% endblock %} \ No newline at end of file diff --git a/src/resources/page.html b/src/resources/page.html index 0c86409..92c7d78 100644 --- a/src/resources/page.html +++ b/src/resources/page.html @@ -107,7 +107,11 @@
{% if page.elapsed %}

{{ page.elapsed }}ms

{% endif %} + {% if page.repo %} +

+ On branch '{{ page.repo.local }}': + {{ page.repo.ahead }} ahead, + {{ page.repo.behind }} behind + '{{ page.repo.upstream }}' + {% if page.repo.fetched %} + (fetched {{ page.repo.fetched }}) + {% else %} + (never fetched) + {% endif %} + {% if page.repo.uncommitted %} + with uncommitted changes + {% endif %} +

+ {% endif %} {% block content %}{% endblock %}
+ @@ -139,6 +174,20 @@ src="https://cdnjs.cloudflare.com/ajax/libs/typeahead.js/0.11.1/typeahead.bundle.min.js">