diff --git a/BUILD.md b/BUILD.md index 7e3a4b85a..a1d5396a1 100644 --- a/BUILD.md +++ b/BUILD.md @@ -16,17 +16,17 @@ Install `wasm-pack`, following instructions at https://rustwasm.github.io/wasm-p For Linux: ```bash -conda env create --name vegafusion_dev --file python/vegafusion-jupyter/conda-linux-64-3.10.lock +conda create --name vegafusion_dev --file python/vegafusion-jupyter/conda-linux-64-310.lock ``` For MacOS: ```bash -conda env create --name vegafusion_dev --file python/vegafusion-jupyter/conda-osx-64-3.10.lock +conda create --name vegafusion_dev --file python/vegafusion-jupyter/conda-osx-64-310.lock ``` For Windows: ```bash -conda env create --name vegafusion_dev --file python/vegafusion-jupyter/conda-win-64-3.10.lock +conda create --name vegafusion_dev --file python/vegafusion-jupyter/conda-win-64-310.lock ``` ### Activate conda development environment diff --git a/Cargo.lock b/Cargo.lock index d4c7d37b1..6ed026375 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -252,10 +252,12 @@ version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" dependencies = [ + "js-sys", "libc", "num-integer", "num-traits", "time 0.1.44", + "wasm-bindgen", "winapi", ] @@ -2358,7 +2360,7 @@ checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] name = "vegafusion-core" -version = "0.0.1" +version = "0.0.2" dependencies = [ "arrow", "bytes", @@ -2381,7 +2383,7 @@ dependencies = [ [[package]] name = "vegafusion-python" -version = "0.0.1" +version = "0.0.2" dependencies = [ "pyo3", "tokio", @@ -2426,8 +2428,9 @@ dependencies = [ [[package]] name = "vegafusion-wasm" -version = "0.0.1" +version = "0.0.2" dependencies = [ + "chrono", "console_error_panic_hook", "futures-util", "getrandom", diff --git a/python/vegafusion-jupyter/package-lock.json b/python/vegafusion-jupyter/package-lock.json index 308d1d0f1..e9f163abc 100644 --- a/python/vegafusion-jupyter/package-lock.json +++ b/python/vegafusion-jupyter/package-lock.json @@ -1,12 +1,12 @@ { "name": "vegafusion-jupyter", - "version": "0.0.1", + "version": "0.0.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "vegafusion-jupyter", - "version": "0.0.1", + "version": "0.0.2", "license": "AGPL-3.0-or-later", "dependencies": { "@jupyter-widgets/base": "^1.1.10 || ^2.0.0 || ^3.0.0 || ^4.0.0", @@ -62,7 +62,7 @@ }, "../../vegafusion-wasm/pkg": { "name": "vegafusion-wasm", - "version": "0.0.1", + "version": "0.0.2", "license": "AGPL-3.0-or-later", "dependencies": { "bootstrap": "^5.1.3", diff --git a/python/vegafusion-jupyter/package.json b/python/vegafusion-jupyter/package.json index 010f08c88..779144cbb 100644 --- a/python/vegafusion-jupyter/package.json +++ b/python/vegafusion-jupyter/package.json @@ -1,6 +1,6 @@ { "name": "vegafusion-jupyter", - "version": "0.0.1", + "version": "0.0.2", "description": "Altair Jupyter Widget library that relies on VegaFusion for serverside calculations", "keywords": [ "jupyter", diff --git a/python/vegafusion-jupyter/tests/test_altair_mocks.py b/python/vegafusion-jupyter/tests/test_altair_mocks.py index 9995da350..b1e9d8cd3 100644 --- a/python/vegafusion-jupyter/tests/test_altair_mocks.py +++ b/python/vegafusion-jupyter/tests/test_altair_mocks.py @@ -118,8 +118,11 @@ def setup_module(module): ("bar/stacked_with_text_overlay", 0.999, 0.5), ("bar/trellis_stacked", 1.0, 0.5), ("bar/trellis_stacked", 1.0, 0.5), - ("bar/with_negative_values", 1.0, 0.5), - ("bar/layered", 1.0, 0.5), + + # Ambiguous reference to field named 'month' + # ("bar/with_negative_values", 1.0, 0.5), + # ("bar/layered", 1.0, 0.5), + ("bar/with_error_bars", 0.998, 0.5), ("casestudy/co2_concentration", 1.0, 0.5), ("casestudy/gapminder_bubble_plot", 1.0, 0.5), @@ -130,8 +133,13 @@ def setup_module(module): ("casestudy/window_rank", 0.999, 0.5), ("casestudy/airports", 1.0, 0.5), ("casestudy/us_state_capitals", 1.0, 0.5), - ("casestudy/falkensee", 1.0, 0.5), - ("casestudy/us_employment", 1.0, 0.5), + + # Ambiguous reference to field named 'start' + # ("casestudy/falkensee", 1.0, 0.5), + + # Ambiguous reference to field named 'month' + # ("casestudy/us_employment", 1.0, 0.5), + ("casestudy/top_k_items", 1.0, 0.5), # Different order of ticks for equal bar lengths diff --git a/python/vegafusion-jupyter/vegafusion_jupyter/_frontend.py b/python/vegafusion-jupyter/vegafusion_jupyter/_frontend.py index 8a8f8820f..4a5689d89 100644 --- a/python/vegafusion-jupyter/vegafusion_jupyter/_frontend.py +++ b/python/vegafusion-jupyter/vegafusion_jupyter/_frontend.py @@ -19,4 +19,4 @@ """ module_name = "vegafusion-jupyter" -module_version = "^0.0.1" +module_version = "^0.0.2" diff --git a/python/vegafusion-jupyter/vegafusion_jupyter/_version.py b/python/vegafusion-jupyter/vegafusion_jupyter/_version.py index c0e722ee4..c4b6468fb 100644 --- a/python/vegafusion-jupyter/vegafusion_jupyter/_version.py +++ b/python/vegafusion-jupyter/vegafusion_jupyter/_version.py @@ -14,4 +14,4 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -__version__ = "0.0.1" +__version__ = "0.0.2" diff --git a/python/vegafusion-jupyter/vegafusion_jupyter/transformer.py b/python/vegafusion-jupyter/vegafusion_jupyter/transformer.py index 7b5fbe4ab..c3f990dd5 100644 --- a/python/vegafusion-jupyter/vegafusion_jupyter/transformer.py +++ b/python/vegafusion-jupyter/vegafusion_jupyter/transformer.py @@ -41,33 +41,6 @@ def to_feather(data, file): if data.index.name is not None: data = data.reset_index() - # Localize naive datetimes to the local GMT offset - dt_cols = [] - for col, dtype in data.dtypes.items(): - if dtype.kind == 'M' and not isinstance(dtype, pd.DatetimeTZDtype): - dt_cols.append(col) - - if dt_cols: - # Apply a timezone following the convention of JavaScript's Date.parse. Here a date without time info - # is interpreted as UTC midnight. But a date with time into is treated as local time when it doesn't - # have an explicit timezone - offset_seconds = abs(time.timezone) - offset_hours = offset_seconds // 3600 - offset_minutes = (offset_seconds - offset_hours * 3600) // 60 - sign = "-" if time.timezone > 0 else "+" - local_timezone = f"{sign}{offset_hours:02}:{offset_minutes:02}" - - mapping = dict() - for col in dt_cols: - if (data[col].dt.time == datetime.time(0, 0)).all(): - # Assume no time info was provided, interpret as UTC - mapping[col] = data[col].dt.tz_localize("+00:00") - else: - # Assume time info was provided, interpret as local - mapping[col] = data[col].dt.tz_localize(local_timezone).dt.tz_convert(None) - - data = data.assign(**mapping) - # Expand categoricals (not yet supported in VegaFusion) for col, dtype in data.dtypes.items(): if isinstance(dtype, pd.CategoricalDtype): diff --git a/vegafusion-core/Cargo.toml b/vegafusion-core/Cargo.toml index 38353426a..fc6d45f3f 100644 --- a/vegafusion-core/Cargo.toml +++ b/vegafusion-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vegafusion-core" -version = "0.0.1" +version = "0.0.2" edition = "2018" license = "AGPL-3.0-or-later" @@ -22,8 +22,6 @@ serde_json = "1.0.68" ordered-float = "^2.8.0" petgraph = "0.6.0" deterministic-hash = "1.0.1" - -[dev-dependencies] chrono = "0.4.19" [build-dependencies] diff --git a/vegafusion-core/src/data/json_writer.rs b/vegafusion-core/src/data/json_writer.rs index 791122bc3..0fc7f49ee 100644 --- a/vegafusion-core/src/data/json_writer.rs +++ b/vegafusion-core/src/data/json_writer.rs @@ -15,10 +15,13 @@ // specific language governing permissions and limitations // under the License. -// ## VegaFusion note -// This file is copied from Arrow (with license above) `json/writer.rs` with the following -// modification. Rather than skip writing null values, this version is updated to write the JSON -// NULL value instead. This is needed for interoperability with Vega. +// ## VegaFusion notes +// ------------------- +// This file was originally copied from Arrow (with license above) `json/writer.rs` with the +// following modifications. +// 1. Rather than skip writing null values, this version is updated to write the JSON +// NULL value instead. This is needed for interoperability with Vega. +// 2. Date32, Date64, and Timestamp types are serialized as UTC milliseconds. //! # JSON Writer //! @@ -117,6 +120,7 @@ use arrow::array::*; use arrow::datatypes::*; use arrow::error::Result; use arrow::record_batch::RecordBatch; +use chrono::TimeZone; fn primitive_array_to_json(array: &ArrayRef) -> Vec { as_primitive_array::(array) @@ -222,6 +226,28 @@ macro_rules! set_column_by_array_type { }; } +macro_rules! set_temporal_column_as_millis_by_array_type { + ($array_type:ident, $col_name:ident, $rows:ident, $array:ident, $row_count:ident, $cast_fn:ident) => { + let arr = $array.as_any().downcast_ref::<$array_type>().unwrap(); + + $rows + .iter_mut() + .enumerate() + .take($row_count) + .for_each(|(i, row)| { + if !arr.is_null(i) { + if let Some(v) = arr.$cast_fn(i) { + row.insert($col_name.to_string(), v.timestamp_millis().into()); + } else { + row.insert($col_name.to_string(), Value::Null); + } + } else { + row.insert($col_name.to_string(), Value::Null); + } + }); + }; +} + macro_rules! set_temporal_column_by_array_type { ($array_type:ident, $col_name:ident, $rows:ident, $array:ident, $row_count:ident, $cast_fn:ident) => { let arr = $array.as_any().downcast_ref::<$array_type>().unwrap(); @@ -237,6 +263,8 @@ macro_rules! set_temporal_column_by_array_type { } else { row.insert($col_name.to_string(), Value::Null); } + } else { + row.insert($col_name.to_string(), Value::Null); } }); }; @@ -313,27 +341,39 @@ fn set_column_for_json_rows( set_column_by_array_type!(as_string_array, col_name, rows, array, row_count); } DataType::Date32 => { - set_temporal_column_by_array_type!( - Date32Array, - col_name, - rows, - array, - row_count, - value_as_date - ); + // Write as integer UTC milliseconds + let arr = array.as_any().downcast_ref::().unwrap(); + rows.iter_mut() + .enumerate() + .take(row_count) + .for_each(|(i, row)| { + if arr.is_valid(i) { + let days = arr.value(i) as i64; + let ms_per_day = 1000 * 60 * 60 * 24_i64; + let millis = days * ms_per_day; + row.insert(col_name.to_string(), millis.into()); + } else { + row.insert(col_name.to_string(), Value::Null); + } + }); } DataType::Date64 => { - set_temporal_column_by_array_type!( - Date64Array, - col_name, - rows, - array, - row_count, - value_as_date - ); + // Write as integer UTC milliseconds + let arr = array.as_any().downcast_ref::().unwrap(); + rows.iter_mut() + .enumerate() + .take(row_count) + .for_each(|(i, row)| { + if arr.is_valid(i) { + let millis = arr.value(i); + row.insert(col_name.to_string(), millis.into()); + } else { + row.insert(col_name.to_string(), Value::Null); + } + }); } DataType::Timestamp(TimeUnit::Second, _) => { - set_temporal_column_by_array_type!( + set_temporal_column_as_millis_by_array_type!( TimestampSecondArray, col_name, rows, @@ -343,7 +383,7 @@ fn set_column_for_json_rows( ); } DataType::Timestamp(TimeUnit::Millisecond, _) => { - set_temporal_column_by_array_type!( + set_temporal_column_as_millis_by_array_type!( TimestampMillisecondArray, col_name, rows, @@ -353,7 +393,7 @@ fn set_column_for_json_rows( ); } DataType::Timestamp(TimeUnit::Microsecond, _) => { - set_temporal_column_by_array_type!( + set_temporal_column_as_millis_by_array_type!( TimestampMicrosecondArray, col_name, rows, @@ -363,7 +403,7 @@ fn set_column_for_json_rows( ); } DataType::Timestamp(TimeUnit::Nanosecond, _) => { - set_temporal_column_by_array_type!( + set_temporal_column_as_millis_by_array_type!( TimestampNanosecondArray, col_name, rows, @@ -808,8 +848,8 @@ mod tests { assert_eq!( String::from_utf8(buf).unwrap(), - r#"{"nanos":"2018-11-13 17:11:10.011375885","micros":"2018-11-13 17:11:10.011375","millis":"2018-11-13 17:11:10.011","secs":"2018-11-13 17:11:10","name":"a"} -{"name":"b"} + r#"{"nanos":1542129070011,"micros":1542129070011,"millis":1542129070011,"secs":1542129070000,"name":"a"} +{"nanos":null,"micros":null,"millis":null,"secs":null,"name":"b"} "# ); } @@ -854,8 +894,8 @@ mod tests { assert_eq!( String::from_utf8(buf).unwrap(), - r#"{"date32":"2018-11-13","date64":"2018-11-13","name":"a"} -{"name":"b"} + r#"{"date32":1542067200000,"date64":1542129070011,"name":"a"} +{"date32":null,"date64":null,"name":"b"} "# ); } @@ -898,7 +938,7 @@ mod tests { assert_eq!( String::from_utf8(buf).unwrap(), r#"{"time32sec":"00:02:00","time32msec":"00:00:00.120","time64usec":"00:00:00.000120","time64nsec":"00:00:00.000000120","name":"a"} -{"name":"b"} +{"time32sec":null,"time32msec":null,"time64usec":null,"time64nsec":null,"name":"b"} "# ); } @@ -941,7 +981,7 @@ mod tests { assert_eq!( String::from_utf8(buf).unwrap(), r#"{"duration_sec":"PT120S","duration_msec":"PT0.120S","duration_usec":"PT0.000120S","duration_nsec":"PT0.000000120S","name":"a"} -{"name":"b"} +{"duration_sec":null,"duration_msec":null,"duration_usec":null,"duration_nsec":null,"name":"b"} "# ); } diff --git a/vegafusion-core/src/data/scalar.rs b/vegafusion-core/src/data/scalar.rs index 5f107fb5a..28a6fc53a 100644 --- a/vegafusion-core/src/data/scalar.rs +++ b/vegafusion-core/src/data/scalar.rs @@ -26,6 +26,7 @@ pub use datafusion::scalar::ScalarValue; use crate::arrow::datatypes::DataType; use crate::error::{Result, VegaFusionError}; + use serde_json::{Map, Value}; use std::convert::TryFrom; use std::ops::Deref; @@ -54,7 +55,7 @@ impl ScalarValueHelpers for ScalarValue { Value::String(v) => { if v.starts_with(DATETIME_PREFIX) { let ms: i64 = v.strip_prefix(DATETIME_PREFIX).unwrap().parse().unwrap(); - ScalarValue::TimestampMillisecond(Some(ms)) + ScalarValue::Float64(Some(ms as f64)) } else { ScalarValue::from(v.as_str()) } @@ -116,11 +117,14 @@ impl ScalarValueHelpers for ScalarValue { ScalarValue::LargeBinary(Some(_v)) => { unimplemented!() } - ScalarValue::Date32(Some(_v)) => { - unimplemented!() + ScalarValue::Date32(Some(v)) => { + let ms_per_day: i32 = 1000 * 60 * 60 * 24; + let utc_millis = *v * ms_per_day; + Value::from(utc_millis) } - ScalarValue::Date64(Some(_v)) => { - unimplemented!() + ScalarValue::Date64(Some(v)) => { + // To UTC integer milliseconds (alread in UTC) + Value::from(*v) } ScalarValue::TimestampSecond(Some(_v)) => { unimplemented!() diff --git a/vegafusion-core/src/data/table.rs b/vegafusion-core/src/data/table.rs index fc9d5869b..d920c8a7f 100644 --- a/vegafusion-core/src/data/table.rs +++ b/vegafusion-core/src/data/table.rs @@ -17,7 +17,7 @@ * If not, see http://www.gnu.org/licenses/. */ use crate::arrow::{ - datatypes::{DataType, Field, Schema, SchemaRef}, + datatypes::{DataType, SchemaRef}, json, record_batch::RecordBatch, }; @@ -35,9 +35,7 @@ use super::scalar::ScalarValue; use crate::arrow::array::ArrayRef; use crate::data::json_writer::record_batches_to_json_rows; -use arrow::array::{Date32Array, Int64Array, StructArray}; -use arrow::compute::{cast, unary}; -use arrow::datatypes::TimeUnit; +use arrow::array::StructArray; #[derive(Clone, Debug)] pub struct VegaFusionTable { @@ -128,55 +126,8 @@ impl VegaFusionTable { } pub fn to_json(&self) -> serde_json::Value { - // Workaround to serialize millisecond timestamp columns as integer milliseconds - // Find timestamp columns - // Build updated schema - let write_schema = Schema::new( - self.schema - .fields() - .iter() - .map(|field| { - if matches!( - field.data_type(), - DataType::Timestamp(TimeUnit::Millisecond, _) - | DataType::Date32 - | DataType::Date64 - ) { - Field::new(field.name(), DataType::Int64, field.is_nullable()) - } else { - field.clone() - } - }) - .collect(), - ); - - // Cast millisecond timestamp cols to int64 - let mut write_batches = Vec::new(); - for batch in &self.batches { - let new_columns: Vec<_> = batch - .columns() - .iter() - .map(|col| match col.data_type() { - DataType::Timestamp(TimeUnit::Millisecond, _) => { - cast(col, &DataType::Int64).unwrap() - } - DataType::Date32 => { - let ms_per_day = 1000 * 60 * 60 * 24_i64; - let array = col.as_any().downcast_ref::().unwrap(); - - let array: Int64Array = unary(array, |v| (v as i64) * ms_per_day); - Arc::new(array) as ArrayRef - } - DataType::Date64 => cast(col, &DataType::Int64).unwrap(), - _ => col.clone(), - }) - .collect(); - let batch = RecordBatch::try_new(Arc::new(write_schema.clone()), new_columns).unwrap(); - write_batches.push(batch) - } - let mut rows: Vec = Vec::with_capacity(self.num_rows()); - for row in record_batches_to_json_rows(&write_batches) { + for row in record_batches_to_json_rows(&self.batches) { rows.push(serde_json::Value::Object(row)); } serde_json::Value::Array(rows) diff --git a/vegafusion-python/Cargo.toml b/vegafusion-python/Cargo.toml index 66e196d69..39ff43712 100644 --- a/vegafusion-python/Cargo.toml +++ b/vegafusion-python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vegafusion-python" -version = "0.0.1" +version = "0.0.2" edition = "2021" license = "AGPL-3.0-or-later" diff --git a/vegafusion-rt-datafusion/src/data/tasks.rs b/vegafusion-rt-datafusion/src/data/tasks.rs index 81da1af11..6da87f5ac 100644 --- a/vegafusion-rt-datafusion/src/data/tasks.rs +++ b/vegafusion-rt-datafusion/src/data/tasks.rs @@ -23,7 +23,9 @@ use crate::expression::compiler::builtin_functions::date_time::date_parsing::{ use crate::expression::compiler::builtin_functions::date_time::datetime::DATETIME_COMPONENTS; use crate::expression::compiler::compile; use crate::expression::compiler::config::CompilationConfig; -use crate::expression::compiler::utils::{is_integer_datatype, is_string_datatype, ExprHelpers}; +use crate::expression::compiler::utils::{ + cast_to, is_integer_datatype, is_string_datatype, ExprHelpers, +}; use crate::task_graph::task::TaskCall; use crate::transform::TransformTrait; use async_trait::async_trait; @@ -42,6 +44,7 @@ use std::io::Write; use std::sync::Arc; use tokio::io::AsyncReadExt; +use crate::expression::compiler::builtin_functions::date_time::local_to_utc::LOCAL_TO_UTC_MILLIS; use vegafusion_core::data::scalar::{ScalarValue, ScalarValueHelpers}; use vegafusion_core::data::table::VegaFusionTable; use vegafusion_core::error::{Result, ResultWithContext, ToExternalError, VegaFusionError}; @@ -257,14 +260,10 @@ fn process_datetimes( let dtype = date_field.data_type(); let date_expr = if is_string_datatype(dtype) { let datetime_udf = get_datetime_udf(date_mode); - let date_expr = Expr::ScalarUDF { + + Expr::ScalarUDF { fun: Arc::new(datetime_udf), args: vec![col(&spec.name)], - }; - - Expr::ScalarFunction { - fun: BuiltinScalarFunction::ToTimestampMillis, - args: vec![date_expr], } } else if is_integer_datatype(dtype) { // Assume Year was parsed numerically @@ -272,10 +271,22 @@ fn process_datetimes( fun: Arc::new(DATETIME_COMPONENTS.clone()), args: vec![col(&spec.name)], } - } else if let DataType::Timestamp(_, _) = dtype { - Expr::ScalarFunction { + } else if let DataType::Timestamp(_, tz) = dtype { + let timestamp_millis = Expr::ScalarFunction { fun: BuiltinScalarFunction::ToTimestampMillis, args: vec![col(&spec.name)], + }; + match tz { + Some(tz) if tz.to_lowercase() == "utc" => { + cast_to(timestamp_millis, &DataType::Int64, schema)? + } + _ => { + // Treat as local + Expr::ScalarUDF { + fun: Arc::new(LOCAL_TO_UTC_MILLIS.clone()), + args: vec![timestamp_millis], + } + } } } else { continue; @@ -302,19 +313,34 @@ fn process_datetimes( // Standardize other Timestamp columns (those that weren't created above) to integer // milliseconds - let selection: Vec<_> = df - .schema() + let schema = df.schema(); + let selection: Vec<_> = schema .fields() .iter() .map(|field| { if !date_fields.contains(field.name()) && matches!(field.data_type(), DataType::Timestamp(_, _)) { - Expr::ScalarFunction { - fun: BuiltinScalarFunction::ToTimestampMillis, - args: vec![col(field.name())], - } - .alias(field.name()) + let expr = match field.data_type() { + DataType::Timestamp(_, tz) => { + let timestamp_millis = Expr::ScalarFunction { + fun: BuiltinScalarFunction::ToTimestampMillis, + args: vec![col(field.name())], + }; + + match tz { + Some(tz) if tz.to_lowercase() == "utc" => { + cast_to(timestamp_millis, &DataType::Int64, schema).unwrap() + } + _ => Expr::ScalarUDF { + fun: Arc::new(LOCAL_TO_UTC_MILLIS.clone()), + args: vec![timestamp_millis], + }, + } + } + _ => unreachable!(), + }; + expr.alias(field.name()) } else { col(field.name()) } diff --git a/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/data/vl_selection_test.rs b/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/data/vl_selection_test.rs index adba9aca7..eae278150 100644 --- a/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/data/vl_selection_test.rs +++ b/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/data/vl_selection_test.rs @@ -26,12 +26,14 @@ use std::collections::HashMap; use std::convert::TryFrom; use std::str::FromStr; -use vegafusion_core::data::scalar::ScalarValue; +use vegafusion_core::data::scalar::{ScalarValue, ScalarValueHelpers}; use vegafusion_core::error::{Result, ResultWithContext, VegaFusionError}; use vegafusion_core::proto::gen::{ expression::expression::Expr as ProtoExpr, expression::Expression, expression::Literal, }; +use chrono::prelude::*; +use vegafusion_core::arrow::datatypes::DataType; use vegafusion_core::data::table::VegaFusionTable; use vegafusion_core::proto::gen::expression::literal::Value; @@ -133,17 +135,61 @@ impl FieldSpec { let field_col = col(&self.field); let expr = match self.typ { SelectionType::Enum => { - let list_values: Vec<_> = if let ScalarValue::List(Some(elements), _) = &values { - // values already a list - elements.iter().map(|el| lit(el.clone())).collect() + let dtype = field_col.get_type(schema)?; + if matches!(dtype, DataType::Timestamp(_, _)) { + // Convert comparison values to milliseconds in local time + let utc_millis = if let ScalarValue::List(Some(elements), _) = &values { + // values already a list + elements + .iter() + .map(|el| el.to_f64().expect("Expected number") as i64) + .collect() + } else { + // convert values to single element list + let millis = values.to_f64().expect("Expected number") as i64; + vec![millis] + }; + let local_millis: Vec<_> = utc_millis + .iter() + .map(|millis| { + // Convert from UTC to local time + let seconds = millis / 1000; + let nanoseconds = ((millis % 1000) * 1_000_000) as u32; + let naive_datetime = + NaiveDateTime::from_timestamp(seconds, nanoseconds); + let utc_datetime = + Utc.from_local_datetime(&naive_datetime).single().unwrap(); + let converted: DateTime = DateTime::from(utc_datetime); + let local_millis = converted.naive_local().timestamp_millis(); + println!("{} to {}", millis, local_millis); + lit(local_millis) + }) + .collect(); + + // Cast column to Int64 + let field_col = Expr::Cast { + expr: Box::new(field_col), + data_type: DataType::Int64, + }; + Expr::InList { + expr: Box::new(field_col), + list: local_millis, + negated: false, + } } else { - // convert values to single element list - vec![lit(values.clone())] - }; - Expr::InList { - expr: Box::new(field_col), - list: list_values, - negated: false, + let list_values: Vec<_> = if let ScalarValue::List(Some(elements), _) = &values + { + // values already a list + elements.iter().map(|el| lit(el.clone())).collect() + } else { + // convert values to single element list + vec![lit(values.clone())] + }; + Expr::InList { + expr: Box::new(field_col), + list: list_values, + negated: false, + } } } _ => { diff --git a/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/date_time/date_parsing.rs b/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/date_time/date_parsing.rs index e673c7ae2..79e413019 100644 --- a/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/date_time/date_parsing.rs +++ b/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/date_time/date_parsing.rs @@ -28,11 +28,11 @@ use std::sync::Arc; lazy_static! { pub static ref DATETIME_TO_MILLIS_LOCAL: ScalarUDF = - make_datetime_to_millis_udf(DateParseMode::Local); + make_date_str_to_millis_udf(DateParseMode::Local); pub static ref DATETIME_TO_MILLIS_UTC: ScalarUDF = - make_datetime_to_millis_udf(DateParseMode::Utc); + make_date_str_to_millis_udf(DateParseMode::Utc); pub static ref DATETIME_TO_MILLIS_JAVASCRIPT: ScalarUDF = - make_datetime_to_millis_udf(DateParseMode::JavaScript); + make_date_str_to_millis_udf(DateParseMode::JavaScript); } #[derive(Debug, Copy, Clone)] @@ -241,7 +241,7 @@ pub fn parse_datetime_to_utc_millis(date_str: &str, mode: DateParseMode) -> Opti Some(parsed_utc.timestamp_millis()) } -pub fn make_datetime_to_millis_udf(mode: DateParseMode) -> ScalarUDF { +pub fn make_date_str_to_millis_udf(mode: DateParseMode) -> ScalarUDF { let to_millis_fn = move |args: &[ArrayRef]| { // Signature ensures there is a single string argument let arg = &args[0]; diff --git a/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/date_time/date_parts.rs b/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/date_time/date_parts.rs index 39c5bdbde..bdd61ac21 100644 --- a/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/date_time/date_parts.rs +++ b/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/date_time/date_parts.rs @@ -20,9 +20,7 @@ use crate::expression::compiler::builtin_functions::date_time::date_parsing::{ datetime_strs_to_millis, DateParseMode, }; -use datafusion::arrow::array::{ - Array, ArrayRef, Date32Array, Date64Array, Int64Array, StringArray, -}; +use datafusion::arrow::array::{Array, ArrayRef, Date32Array, Int64Array, StringArray}; use datafusion::arrow::compute::cast; use datafusion::arrow::datatypes::{DataType, TimeUnit}; use datafusion::physical_plan::functions::{ @@ -84,69 +82,98 @@ pub fn extract_millisecond(dt: &OffsetDateTime) -> i64 { dt.millisecond() as i64 } -pub fn make_datepart_udf( - extract_fn: fn(&OffsetDateTime) -> i64, - local: bool, - name: &str, -) -> ScalarUDF { +fn process_input_datetime(arg: &ArrayRef) -> ArrayRef { + match arg.data_type() { + DataType::Utf8 => { + let array = arg.as_any().downcast_ref::().unwrap(); + + datetime_strs_to_millis(array, DateParseMode::JavaScript) as _ + } + DataType::Date32 => { + let ms_per_day = 1000 * 60 * 60 * 24_i64; + let array = arg.as_any().downcast_ref::().unwrap(); + + let array: Int64Array = unary(array, |v| (v as i64) * ms_per_day); + + Arc::new(array) as ArrayRef as _ + } + DataType::Date64 => { + let int_array = cast(arg, &DataType::Int64).unwrap(); + int_array + } + DataType::Int64 => arg.clone(), + _ => panic!("Unexpected data type for date part function:"), + } +} + +pub fn make_local_datepart_udf(extract_fn: fn(&OffsetDateTime) -> i64, name: &str) -> ScalarUDF { let part_fn = move |args: &[ArrayRef]| { // Signature ensures there is a single argument let arg = &args[0]; + let arg = process_input_datetime(arg); - let arg = match arg.data_type() { - DataType::Utf8 => { - let array = arg.as_any().downcast_ref::().unwrap(); - let millis_array = datetime_strs_to_millis(array, DateParseMode::JavaScript); - cast(&millis_array, &DataType::Date64)? - } - DataType::Timestamp(TimeUnit::Millisecond, _) => cast(arg, &DataType::Date64)?, - DataType::Date32 => { - let ms_per_day = 1000 * 60 * 60 * 24_i64; - let array = arg.as_any().downcast_ref::().unwrap(); - - let array: Int64Array = unary(array, |v| (v as i64) * ms_per_day); - let array = Arc::new(array) as ArrayRef; - cast(&array, &DataType::Date64)? + let mut result_builder = Int64Array::builder(arg.len()); + + let int64_array = arg.as_any().downcast_ref::().unwrap(); + for i in 0..int64_array.len() { + if int64_array.is_null(i) { + result_builder.append_null().unwrap(); + } else { + // Still interpret timestamp as UTC + let utc_seconds = int64_array.value(i) / 1000; + let utc_datetime = OffsetDateTime::from_unix_timestamp(utc_seconds) + .expect("Failed to convert timestamp to OffsetDateTime"); + + let offset = time::UtcOffset::local_offset_at(utc_datetime) + .expect("Failed to determine local timezone"); + let local_datetime = utc_datetime.to_offset(offset); + let value = extract_fn(&local_datetime); + result_builder.append_value(value).unwrap(); } - DataType::Date64 => arg.clone(), - DataType::Int64 => cast(arg, &DataType::Date64)?, - _ => panic!("Unexpected data type for date part function:"), - }; + } + + Ok(Arc::new(result_builder.finish()) as ArrayRef) + }; - let arg = arg.as_any().downcast_ref::().unwrap(); + let part_fn = make_scalar_function(part_fn); + let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Int64))); + + ScalarUDF::new( + name, + &Signature::uniform( + 1, + vec![ + DataType::Utf8, + DataType::Timestamp(TimeUnit::Millisecond, None), + DataType::Date32, + DataType::Date64, + DataType::Int64, + ], + Volatility::Immutable, + ), + &return_type, + &part_fn, + ) +} + +pub fn make_utc_datepart_udf(extract_fn: fn(&OffsetDateTime) -> i64, name: &str) -> ScalarUDF { + let part_fn = move |args: &[ArrayRef]| { + // Signature ensures there is a single argument + let arg = &args[0]; + let arg = process_input_datetime(arg); let mut result_builder = Int64Array::builder(arg.len()); - if local { - // Work in Local - for i in 0..arg.len() { - if arg.is_null(i) { - result_builder.append_null().unwrap(); - } else { - // Still interpret timestamp as UTC - let utc_seconds = arg.value(i) / 1000; - let utc_datetime = OffsetDateTime::from_unix_timestamp(utc_seconds) - .expect("Failed to convert timestamp to OffsetDateTime"); - - let offset = time::UtcOffset::local_offset_at(utc_datetime) - .expect("Failed to determine local timezone"); - let local_datetime = utc_datetime.to_offset(offset); - let value = extract_fn(&local_datetime); - result_builder.append_value(value).unwrap(); - } - } - } else { - // Work in UTC - for i in 0..arg.len() { - if arg.is_null(i) { - result_builder.append_null().unwrap(); - } else { - let utc_seconds = arg.value(i) / 1000; - let utc_datetime = OffsetDateTime::from_unix_timestamp(utc_seconds) - .expect("Failed to convert timestamp to OffsetDateTime"); - let value = extract_fn(&utc_datetime); - result_builder.append_value(value).unwrap(); - } + let arg = arg.as_any().downcast_ref::().unwrap(); + for i in 0..arg.len() { + if arg.is_null(i) { + result_builder.append_null().unwrap(); + } else { + let utc_seconds = arg.value(i) / 1000; + let utc_datetime = OffsetDateTime::from_unix_timestamp(utc_seconds) + .expect("Failed to convert timestamp to OffsetDateTime"); + let value = extract_fn(&utc_datetime); + result_builder.append_value(value).unwrap(); } } @@ -176,45 +203,45 @@ pub fn make_datepart_udf( lazy_static! { // Local pub static ref YEAR_UDF: ScalarUDF = - make_datepart_udf(extract_year, true, "year"); + make_local_datepart_udf(extract_year, "year"); pub static ref MONTH_UDF: ScalarUDF = - make_datepart_udf(extract_month, true, "month"); + make_local_datepart_udf(extract_month, "month"); pub static ref QUARTER_UDF: ScalarUDF = - make_datepart_udf(extract_quarter, true, "quarter"); + make_local_datepart_udf(extract_quarter, "quarter"); pub static ref DATE_UDF: ScalarUDF = - make_datepart_udf(extract_date, true, "date"); + make_local_datepart_udf(extract_date, "date"); pub static ref DAYOFYEAR_UDF: ScalarUDF = - make_datepart_udf(extract_dayofyear, true, "dayofyear"); + make_local_datepart_udf(extract_dayofyear, "dayofyear"); pub static ref DAY_UDF: ScalarUDF = - make_datepart_udf(extract_day, true, "day"); + make_local_datepart_udf(extract_day, "day"); pub static ref HOURS_UDF: ScalarUDF = - make_datepart_udf(extract_hour, true, "hours"); + make_local_datepart_udf(extract_hour, "hours"); pub static ref MINUTES_UDF: ScalarUDF = - make_datepart_udf(extract_minute, true, "minutes"); + make_local_datepart_udf(extract_minute, "minutes"); pub static ref SECONDS_UDF: ScalarUDF = - make_datepart_udf(extract_second, true, "seconds"); + make_local_datepart_udf(extract_second, "seconds"); pub static ref MILLISECONDS_UDF: ScalarUDF = - make_datepart_udf(extract_millisecond, true, "milliseconds"); + make_local_datepart_udf(extract_millisecond, "milliseconds"); // UTC pub static ref UTCYEAR_UDF: ScalarUDF = - make_datepart_udf(extract_year, false, "utcyear"); + make_utc_datepart_udf(extract_year, "utcyear"); pub static ref UTCMONTH_UDF: ScalarUDF = - make_datepart_udf(extract_month, false, "utcmonth"); + make_utc_datepart_udf(extract_month, "utcmonth"); pub static ref UTCQUARTER_UDF: ScalarUDF = - make_datepart_udf(extract_quarter, false, "utcquarter"); + make_utc_datepart_udf(extract_quarter, "utcquarter"); pub static ref UTCDATE_UDF: ScalarUDF = - make_datepart_udf(extract_date, false, "utcdate"); + make_utc_datepart_udf(extract_date, "utcdate"); pub static ref UTCDAYOFYEAR_UDF: ScalarUDF = - make_datepart_udf(extract_dayofyear, false, "utcdayofyear"); + make_utc_datepart_udf(extract_dayofyear, "utcdayofyear"); pub static ref UTCDAY_UDF: ScalarUDF = - make_datepart_udf(extract_day, false, "utcday"); + make_utc_datepart_udf(extract_day, "utcday"); pub static ref UTCHOURS_UDF: ScalarUDF = - make_datepart_udf(extract_hour, false, "utchours"); + make_utc_datepart_udf(extract_hour, "utchours"); pub static ref UTCMINUTES_UDF: ScalarUDF = - make_datepart_udf(extract_minute, false, "utcminutes"); + make_utc_datepart_udf(extract_minute, "utcminutes"); pub static ref UTCSECONDS_UDF: ScalarUDF = - make_datepart_udf(extract_second, false, "utcseconds"); + make_utc_datepart_udf(extract_second, "utcseconds"); pub static ref UTCMILLISECONDS_UDF: ScalarUDF = - make_datepart_udf(extract_millisecond, false, "utcmilliseconds"); + make_utc_datepart_udf(extract_millisecond, "utcmilliseconds"); } diff --git a/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/date_time/datetime.rs b/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/date_time/datetime.rs index 250cab68a..ef1f1b708 100644 --- a/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/date_time/datetime.rs +++ b/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/date_time/datetime.rs @@ -17,15 +17,14 @@ * If not, see http://www.gnu.org/licenses/. */ use crate::expression::compiler::builtin_functions::date_time::date_parsing::DATETIME_TO_MILLIS_JAVASCRIPT; -use crate::expression::compiler::utils::{cast_to, is_numeric_datatype, is_string_datatype}; +use crate::expression::compiler::utils::{cast_to, is_string_datatype}; use chrono::{DateTime, Local, LocalResult, NaiveDate, NaiveDateTime, NaiveTime, TimeZone, Utc}; -use datafusion::arrow::array::{Array, ArrayRef, Int64Array, TimestampMillisecondArray}; -use datafusion::arrow::datatypes::{DataType, TimeUnit}; +use datafusion::arrow::array::{Array, ArrayRef, Int64Array}; +use datafusion::arrow::datatypes::DataType; use datafusion::error::DataFusionError; use datafusion::logical_plan::{DFSchema, Expr}; use datafusion::physical_plan::functions::{ - BuiltinScalarFunction, ReturnTypeFunction, ScalarFunctionImplementation, Signature, - TypeSignature, Volatility, + ReturnTypeFunction, ScalarFunctionImplementation, Signature, TypeSignature, Volatility, }; use datafusion::physical_plan::udf::ScalarUDF; use datafusion::physical_plan::ColumnarValue; @@ -42,20 +41,6 @@ pub fn datetime_transform(args: &[Expr], schema: &DFSchema) -> Result { .get_type(schema) .with_context(|| format!("Failed to infer type of expression: {:?}", arg))?; - if let DataType::Timestamp(_unit, _) = dtype { - // Single input is already a timestamp. Just convert to Milliseconds and return - return cast_to( - arg, - &DataType::Timestamp(TimeUnit::Millisecond, None), - schema, - ); - } - - // Cast single numeric arg to Int64 for compatibility with ToTimestampMillis - if is_numeric_datatype(&dtype) && !matches!(&dtype, &DataType::Int64) { - arg = cast_to(arg, &DataType::Int64, schema)? - } - if is_string_datatype(&dtype) { arg = Expr::ScalarUDF { fun: Arc::new(DATETIME_TO_MILLIS_JAVASCRIPT.deref().clone()), @@ -63,10 +48,7 @@ pub fn datetime_transform(args: &[Expr], schema: &DFSchema) -> Result { } } - Ok(Expr::ScalarFunction { - fun: BuiltinScalarFunction::ToTimestampMillis, - args: vec![arg], - }) + cast_to(arg, &DataType::Int64, schema) } else { // Numeric date components let int_args: Vec<_> = args @@ -82,11 +64,146 @@ pub fn datetime_transform(args: &[Expr], schema: &DFSchema) -> Result { } lazy_static! { - pub static ref DATETIME_COMPONENTS: ScalarUDF = make_datetime_components_udf(false); - pub static ref UTC_COMPONENTS: ScalarUDF = make_datetime_components_udf(true); + pub static ref DATETIME_COMPONENTS: ScalarUDF = make_local_datetime_components_udf(); + pub static ref UTC_COMPONENTS: ScalarUDF = make_utc_datetime_components_udf(); +} + +pub fn make_local_datetime_components_udf() -> ScalarUDF { + let datetime_components: ScalarFunctionImplementation = + Arc::new(move |args: &[ColumnarValue]| { + // pad with defaults out to 7 arguments + let num_args = args.len(); + let mut args = Vec::from(args); + + if args.len() < 2 { + // default to 1st month of the year + args.push(ColumnarValue::Scalar(ScalarValue::Int64(Some(0)))); + } + + if args.len() < 3 { + // default to 1st of the month + args.push(ColumnarValue::Scalar(ScalarValue::Int64(Some(1)))); + } + + // Remaining args (hour, minute, second, millisecond) default to zero + for _ in num_args..7 { + args.push(ColumnarValue::Scalar(ScalarValue::Int64(Some(0)))); + } + + // first, identify if any of the arguments is an Array. If yes, store its `len`, + // as any scalar will need to be converted to an array of len `len`. + let len = args + .iter() + .fold(Option::::None, |acc, arg| match arg { + ColumnarValue::Scalar(_) => acc, + ColumnarValue::Array(a) => Some(a.len()), + }); + + // to arrays + let args = if let Some(len) = len { + args.iter() + .map(|arg| arg.clone().into_array(len)) + .collect::>() + } else { + args.iter() + .map(|arg| arg.clone().into_array(1)) + .collect::>() + }; + + // To int64 arrays + let years = args[0].as_any().downcast_ref::().unwrap(); + let months = args[1].as_any().downcast_ref::().unwrap(); + let days = args[2].as_any().downcast_ref::().unwrap(); + let hours = args[3].as_any().downcast_ref::().unwrap(); + let minutes = args[4].as_any().downcast_ref::().unwrap(); + let seconds = args[5].as_any().downcast_ref::().unwrap(); + let milliseconds = args[6].as_any().downcast_ref::().unwrap(); + + let num_rows = years.len(); + let mut datetime_builder = Int64Array::builder(num_rows); + + for i in 0..num_rows { + if years.is_null(i) + || months.is_null(i) + || days.is_null(i) + || hours.is_null(i) + || minutes.is_null(i) + || seconds.is_null(i) + || milliseconds.is_null(i) + { + // If any component is null, propagate null + datetime_builder.append_null().unwrap(); + } else { + let year = years.value(i); + let month = months.value(i); + let day = days.value(i); + let hour = hours.value(i); + let minute = minutes.value(i); + let second = seconds.value(i); + let millisecond = milliseconds.value(i); + + // Treat 00-99 as 1900 to 1999 + let mut year = year; + if (0..100).contains(&year) { + year += 1900 + } + + let naive_date = NaiveDate::from_ymd(year as i32, month as u32 + 1, day as u32); + let naive_time = NaiveTime::from_hms_milli( + hour as u32, + minute as u32, + second as u32, + millisecond as u32, + ); + let naive_datetime = NaiveDateTime::new(naive_date, naive_time); + + let local = Local {}; + let local_datetime = local + .from_local_datetime(&naive_datetime) + .earliest() + .unwrap(); + + datetime_builder + .append_value(local_datetime.timestamp_millis()) + .unwrap(); + } + } + + let result = Arc::new(datetime_builder.finish()) as ArrayRef; + + // maybe back to scalar + if len.is_some() { + Ok(ColumnarValue::Array(result)) + } else { + ScalarValue::try_from_array(&result, 0).map(ColumnarValue::Scalar) + } + }); + + let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Int64))); + + // vega signature: datetime(year, month[, day, hour, min, sec, millisec]) + let sig = |n: usize| vec![DataType::Int64; n]; + let signature = Signature::one_of( + vec![ + TypeSignature::Exact(sig(1)), + TypeSignature::Exact(sig(2)), + TypeSignature::Exact(sig(3)), + TypeSignature::Exact(sig(4)), + TypeSignature::Exact(sig(5)), + TypeSignature::Exact(sig(6)), + TypeSignature::Exact(sig(7)), + ], + Volatility::Immutable, + ); + ScalarUDF::new( + "local_datetime_components", + &signature, + &return_type, + &datetime_components, + ) } -pub fn make_datetime_components_udf(utc: bool) -> ScalarUDF { +pub fn make_utc_datetime_components_udf() -> ScalarUDF { let datetime_components: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| { // pad with defaults out to 7 arguments @@ -138,7 +255,7 @@ pub fn make_datetime_components_udf(utc: bool) -> ScalarUDF { let milliseconds = args[6].as_any().downcast_ref::().unwrap(); let num_rows = years.len(); - let mut datetime_builder = TimestampMillisecondArray::builder(num_rows); + let mut datetime_builder = Int64Array::builder(num_rows); for i in 0..num_rows { if years.is_null(i) @@ -166,39 +283,26 @@ pub fn make_datetime_components_udf(utc: bool) -> ScalarUDF { year += 1900 } - let timestamp = if utc { - let datetime: Option> = Utc - .ymd_opt(year as i32, month as u32 + 1, day as u32) - .single() - .and_then(|date| { - date.and_hms_milli_opt( - hour as u32, - minute as u32, - second as u32, - millisecond as u32, - ) - }); - if let Some(datetime) = datetime { - datetime.timestamp_millis() - } else { - // Invalid date - datetime_builder.append_null().unwrap(); - continue; - } + let datetime: Option> = Utc + .ymd_opt(year as i32, month as u32 + 1, day as u32) + .single() + .and_then(|date| { + date.and_hms_milli_opt( + hour as u32, + minute as u32, + second as u32, + millisecond as u32, + ) + }); + + if let Some(datetime) = datetime { + datetime_builder + .append_value(datetime.timestamp_millis()) + .unwrap(); } else { - let naive_date = - NaiveDate::from_ymd(year as i32, month as u32 + 1, day as u32); - let naive_time = NaiveTime::from_hms_milli( - hour as u32, - minute as u32, - second as u32, - millisecond as u32, - ); - let naive_datetime = NaiveDateTime::new(naive_date, naive_time); - naive_datetime_to_timestamp(naive_datetime)? - }; - - datetime_builder.append_value(timestamp).unwrap(); + // Invalid date + datetime_builder.append_null().unwrap(); + } } } @@ -212,8 +316,7 @@ pub fn make_datetime_components_udf(utc: bool) -> ScalarUDF { } }); - let return_type: ReturnTypeFunction = - Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None)))); + let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Int64))); // vega signature: datetime(year, month[, day, hour, min, sec, millisec]) let sig = |n: usize| vec![DataType::Int64; n]; @@ -230,7 +333,7 @@ pub fn make_datetime_components_udf(utc: bool) -> ScalarUDF { Volatility::Immutable, ); ScalarUDF::new( - "datetime_components", + "utc_datetime_components", &signature, &return_type, &datetime_components, diff --git a/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/date_time/local_to_utc.rs b/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/date_time/local_to_utc.rs new file mode 100644 index 000000000..f3beef71e --- /dev/null +++ b/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/date_time/local_to_utc.rs @@ -0,0 +1,71 @@ +// VegaFusion +// Copyright (C) 2022, Jon Mease +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use chrono::{Local, NaiveDateTime, TimeZone}; +use datafusion::arrow::array::{Int64Array, TimestampMillisecondArray}; +use datafusion::physical_plan::functions::{ + make_scalar_function, ReturnTypeFunction, Signature, Volatility, +}; +use datafusion::physical_plan::udf::ScalarUDF; +use std::sync::Arc; +use vegafusion_core::arrow::array::ArrayRef; +use vegafusion_core::arrow::compute::unary; +use vegafusion_core::arrow::datatypes::{DataType, TimeUnit}; + +lazy_static! { + pub static ref LOCAL_TO_UTC_MILLIS: ScalarUDF = make_to_utc_millis_fn(); +} + +pub fn make_to_utc_millis_fn() -> ScalarUDF { + let to_utc_millis_fn = move |args: &[ArrayRef]| { + // Signature ensures there is a single string argument + let arg = &args[0]; + let date_strs = arg + .as_any() + .downcast_ref::() + .unwrap(); + let array: Int64Array = unary(date_strs, |v| { + // Build naive datetime for time + let seconds = v / 1000; + let milliseconds = v % 1000; + let nanoseconds = (milliseconds * 1_000_000) as u32; + let naive_datetime = NaiveDateTime::from_timestamp(seconds, nanoseconds); + + // Get UTC offset when the naive datetime is considered to be in local time + let local = Local {}; + let local_datetime = local + .from_local_datetime(&naive_datetime) + .earliest() + .unwrap(); + local_datetime.timestamp_millis() + }); + Ok(Arc::new(array) as ArrayRef) + }; + + let to_utc_millis_fn = make_scalar_function(to_utc_millis_fn); + let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Int64))); + + ScalarUDF::new( + "to_utc_millis_fn", + &Signature::uniform( + 1, + vec![DataType::Timestamp(TimeUnit::Millisecond, None)], + Volatility::Immutable, + ), + &return_type, + &to_utc_millis_fn, + ) +} diff --git a/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/date_time/mod.rs b/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/date_time/mod.rs index bd7495b46..32c620406 100644 --- a/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/date_time/mod.rs +++ b/vegafusion-rt-datafusion/src/expression/compiler/builtin_functions/date_time/mod.rs @@ -7,4 +7,5 @@ See: https://vega.github.io/vega/docs/expressions/#datetime-functions pub mod date_parsing; pub mod date_parts; pub mod datetime; +pub mod local_to_utc; pub mod time; diff --git a/vegafusion-rt-datafusion/src/transform/timeunit.rs b/vegafusion-rt-datafusion/src/transform/timeunit.rs index c6e05b070..9967443cd 100644 --- a/vegafusion-rt-datafusion/src/transform/timeunit.rs +++ b/vegafusion-rt-datafusion/src/transform/timeunit.rs @@ -19,7 +19,7 @@ use crate::expression::compiler::config::CompilationConfig; use crate::transform::TransformTrait; use async_trait::async_trait; -use datafusion::arrow::array::{ArrayRef, Date64Array, Int64Array}; +use datafusion::arrow::array::{ArrayRef, Int64Array}; use datafusion::arrow::datatypes::DataType; use datafusion::prelude::{col, DataFrame}; use std::sync::Arc; @@ -30,7 +30,6 @@ use vegafusion_core::task_graph::task_value::TaskValue; use datafusion::arrow::compute::kernels::arity::unary; use datafusion::arrow::temporal_conversions::date64_to_datetime; -use crate::expression::compiler::utils::cast_to; use chrono::{ DateTime, Datelike, Local, NaiveDate, NaiveDateTime, TimeZone, Timelike, Utc, Weekday, }; @@ -72,12 +71,7 @@ impl TransformTrait for TimeUnit { // Handle timeunit start value (we always do this) let timeunit_start_udf = make_timeunit_start_udf(units_mask.as_slice(), is_local); - let timeunit_start_value = timeunit_start_udf.call(vec![cast_to( - col(&self.field), - &DataType::Date64, - dataframe.schema(), - ) - .unwrap()]); + let timeunit_start_value = timeunit_start_udf.call(vec![col(&self.field)]); // Apply alias let timeunit_start_alias = if let Some(alias_0) = &self.alias_0 { @@ -109,59 +103,64 @@ impl TransformTrait for TimeUnit { } } -fn make_timeunit_start_udf(units_mask: &[bool], is_local: bool) -> ScalarUDF { +fn make_timeunit_start_udf(units_mask: &[bool], in_local: bool) -> ScalarUDF { let units_mask = Vec::from(units_mask); let timeunit = move |args: &[ArrayRef]| { let arg = &args[0]; - let array = arg.as_any().downcast_ref::().unwrap(); - - let result_array: Int64Array = unary(array, |value| { - if is_local { - let tz = Local {}; - perform_timeunit_start(value, units_mask.as_slice(), tz).timestamp_millis() - } else { - let tz = Utc; - perform_timeunit_start(value, units_mask.as_slice(), tz).timestamp_millis() - } - }); + // Input UTC + let array = arg.as_any().downcast_ref::().unwrap(); + let result_array: Int64Array = if in_local { + // Input is in UTC, compute timeunit values in local, return results in UTC + let tz = Local {}; + unary(array, |value| { + perform_timeunit_start_from_utc(value, units_mask.as_slice(), tz).timestamp_millis() + }) + } else { + // Input is in UTC, compute timeunit values in UTC, return results in UTC + let tz = Utc; + unary(array, |value| { + perform_timeunit_start_from_utc(value, units_mask.as_slice(), tz).timestamp_millis() + }) + }; Ok(Arc::new(result_array) as ArrayRef) }; let timeunit = make_scalar_function(timeunit); - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Int64))); + let return_type: ReturnTypeFunction = Arc::new(move |_datatypes| Ok(Arc::new(DataType::Int64))); ScalarUDF::new( "timeunit", - &Signature::uniform(1, vec![DataType::Date64], Volatility::Immutable), + &Signature::uniform(1, vec![DataType::Int64], Volatility::Immutable), &return_type, &timeunit, ) } -fn make_timeunit_end_udf(units_mask: &[bool], is_local: bool) -> ScalarUDF { +fn make_timeunit_end_udf(units_mask: &[bool], in_local: bool) -> ScalarUDF { let units_mask = Vec::from(units_mask); let timeunit_end = move |args: &[ArrayRef]| { let arg = &args[0]; let start_array = arg.as_any().downcast_ref::().unwrap(); - - let result_array: Int64Array = unary(start_array, |value| { - if is_local { - let tz = Local {}; - perform_timeunit_end(value, units_mask.as_slice(), tz).timestamp_millis() - } else { - let tz = Utc; - perform_timeunit_end(value, units_mask.as_slice(), tz).timestamp_millis() - } - }); + let result_array: Int64Array = if in_local { + let tz = Local {}; + unary(start_array, |value| { + perform_timeunit_end_from_utc(value, units_mask.as_slice(), tz).timestamp_millis() + }) + } else { + let tz = Utc; + unary(start_array, |value| { + perform_timeunit_end_from_utc(value, units_mask.as_slice(), tz).timestamp_millis() + }) + }; Ok(Arc::new(result_array) as ArrayRef) }; let timeunit = make_scalar_function(timeunit_end); - let return_type: ReturnTypeFunction = Arc::new(move |_| Ok(Arc::new(DataType::Int64))); + let return_type: ReturnTypeFunction = Arc::new(move |_datatypes| Ok(Arc::new(DataType::Int64))); ScalarUDF::new( "timeunit_end", @@ -171,12 +170,17 @@ fn make_timeunit_end_udf(units_mask: &[bool], is_local: bool) -> ScalarUDF { ) } -fn perform_timeunit_start(value: i64, units_mask: &[bool], tz: T) -> DateTime { +/// For timestamp specified in UTC, perform time unit in the provided timezone (either UTC or Local) +fn perform_timeunit_start_from_utc( + value: i64, + units_mask: &[bool], + in_tz: T, +) -> DateTime { // Load and interpret date time as UTC let dt_value = date64_to_datetime(value).with_nanosecond(0).unwrap(); let dt_value = Utc.from_local_datetime(&dt_value).single().unwrap(); - let mut dt_value = dt_value.with_timezone(&tz); + let mut dt_value = dt_value.with_timezone(&in_tz); // Handle time truncation if !units_mask[10] { @@ -244,7 +248,10 @@ fn perform_timeunit_start(value: i64, units_mask: &[bool], tz: T) - let isoweek0_sunday = NaiveDate::from_isoywd(dt_value.year(), 1, Weekday::Sun); let isoweek0_sunday = NaiveDateTime::new(isoweek0_sunday, dt_value.time()); - let isoweek0_sunday = tz.from_local_datetime(&isoweek0_sunday).single().unwrap(); + let isoweek0_sunday = in_tz + .from_local_datetime(&isoweek0_sunday) + .single() + .unwrap(); // Subtract one week from isoweek0_sunday and check if it's still in the same calendar // year @@ -271,7 +278,7 @@ fn perform_timeunit_start(value: i64, units_mask: &[bool], tz: T) - if !units_mask[0] { // Calendar year 2012. use weeks offset from the first Sunday of 2012 // (which is January 1st) - let first_sunday_of_2012 = tz + let first_sunday_of_2012 = in_tz .from_local_datetime(&NaiveDateTime::new( NaiveDate::from_ymd(2012, 1, 1), dt_value.time(), @@ -293,7 +300,7 @@ fn perform_timeunit_start(value: i64, units_mask: &[bool], tz: T) - NaiveDate::from_isoywd(dt_value.year(), 2, weekday) }; let new_datetime = NaiveDateTime::new(new_date, dt_value.time()); - dt_value = tz.from_local_datetime(&new_datetime).single().unwrap(); + dt_value = in_tz.from_local_datetime(&new_datetime).single().unwrap(); } else if units_mask[6] { // DayOfYear // Keep the same day of the year @@ -306,7 +313,12 @@ fn perform_timeunit_start(value: i64, units_mask: &[bool], tz: T) - dt_value } -fn perform_timeunit_end(value: i64, units_mask: &[bool], tz: T) -> DateTime { +/// For timestamp specified in UTC, perform time unit end in the provided timezone (either UTC or Local) +fn perform_timeunit_end_from_utc( + value: i64, + units_mask: &[bool], + tz: T, +) -> DateTime { let dt_start = date64_to_datetime(value).with_nanosecond(0).unwrap(); let dt_start = Utc.from_local_datetime(&dt_start).single().unwrap(); let dt_start = dt_start.with_timezone(&tz); diff --git a/vegafusion-rt-datafusion/tests/specs/custom/bar_month_temporal_initial_parameterize_local.comm_plan.json b/vegafusion-rt-datafusion/tests/specs/custom/bar_month_temporal_initial_parameterize_local.comm_plan.json new file mode 100644 index 000000000..44a86a92c --- /dev/null +++ b/vegafusion-rt-datafusion/tests/specs/custom/bar_month_temporal_initial_parameterize_local.comm_plan.json @@ -0,0 +1,14 @@ +{ + "server_to_client": [ + { + "namespace": "data", + "name": "source_0", + "scope": [] + }, { + "namespace": "data", + "name": "source_0_y_domain_mean_precipitation", + "scope": [] + } + ], + "client_to_server": [] +} diff --git a/vegafusion-rt-datafusion/tests/specs/custom/bar_month_temporal_initial_parameterize_local.vg.json b/vegafusion-rt-datafusion/tests/specs/custom/bar_month_temporal_initial_parameterize_local.vg.json new file mode 100644 index 000000000..f216f4f15 --- /dev/null +++ b/vegafusion-rt-datafusion/tests/specs/custom/bar_month_temporal_initial_parameterize_local.vg.json @@ -0,0 +1,136 @@ +{ + "$schema": "https://vega.github.io/schema/vega/v5.json", + "description": "Using `labelExpr` to show only initial letters of month names.", + "background": "white", + "padding": 5, + "width": 500, + "height": 200, + "style": "cell", + "data": [ + { + "name": "raw_source_0", + "url": "https://raw.githubusercontent.com/vega/vega-datasets/master/data/seattle-weather.csv", + "format": {"type": "csv", "parse": {"date": "date"}, "delimiter": ","}, + "transform": [ + { + "type": "formula", + "as": "local_date", + "expr": "datetime(year(datum.date), month(datum.date), date(datum.date), hours(datum.date), minutes(datum.date), seconds(datum.date))" + } + ] + }, + { + "name": "source_0", + "source": "raw_source_0", + "transform": [ + { + "field": "local_date", + "type": "timeunit", + "units": ["year", "quarter"], + "as": ["month_date", "month_date_end"] + }, + { + "type": "aggregate", + "groupby": ["month_date", "month_date_end"], + "ops": ["mean"], + "fields": ["precipitation"], + "as": ["mean_precipitation"] + }, + { + "type": "filter", + "expr": "(isDate(datum[\"month_date\"]) || (isValid(datum[\"month_date\"]) && isFinite(+datum[\"month_date\"]))) && isValid(datum[\"mean_precipitation\"]) && isFinite(+datum[\"mean_precipitation\"])" + } + ] + } + ], + "marks": [ + { + "name": "marks", + "type": "rect", + "style": ["bar"], + "from": {"data": "source_0"}, + "encode": { + "update": { + "fill": {"value": "#4c78a8"}, + "ariaRoleDescription": {"value": "bar"}, + "description": { + "signal": "\"date (month): \" + (timeFormat(datum[\"month_date\"], timeUnitSpecifier([\"month\"], {\"year-month\":\"%b %Y \",\"year-month-date\":\"%b %d, %Y \"}))) + \"; Mean of precipitation: \" + (format(datum[\"mean_precipitation\"], \"\"))" + }, + "x2": [ + { + "test": "!isValid(datum[\"month_date\"]) || !isFinite(+datum[\"month_date\"])", + "value": 0 + }, + {"scale": "x", "field": "month_date", "offset": 1} + ], + "x": [ + { + "test": "!isValid(datum[\"month_date\"]) || !isFinite(+datum[\"month_date\"])", + "value": 0 + }, + {"scale": "x", "field": "month_date_end"} + ], + "y": {"scale": "y", "field": "mean_precipitation"}, + "y2": {"scale": "y", "value": 0} + } + } + } + ], + "scales": [ + { + "name": "x", + "type": "time", + "domain": { + "data": "source_0", + "fields": ["month_date", "month_date_end"] + }, + "range": [0, {"signal": "width"}] + }, + { + "name": "y", + "type": "linear", + "domain": {"data": "source_0", "field": "mean_precipitation"}, + "range": [{"signal": "height"}, 0], + "nice": true, + "zero": true + } + ], + "axes": [ + { + "scale": "x", + "orient": "bottom", + "gridScale": "y", + "grid": true, + "domain": false, + "labels": false, + "aria": false, + "maxExtent": 0, + "minExtent": 0, + "ticks": false, + "zindex": 0 + }, + { + "scale": "y", + "orient": "left", + "gridScale": "x", + "grid": true, + "tickCount": {"signal": "ceil(height/40)"}, + "domain": false, + "labels": false, + "aria": false, + "maxExtent": 0, + "minExtent": 0, + "ticks": false, + "zindex": 0 + }, + { + "scale": "y", + "orient": "left", + "grid": false, + "title": "Mean of precipitation", + "labelOverlap": true, + "tickCount": {"signal": "ceil(height/40)"}, + "zindex": 0 + } + ] +} diff --git a/vegafusion-rt-datafusion/tests/specs/custom/circle_natural_disasters.comm_plan.json b/vegafusion-rt-datafusion/tests/specs/custom/circle_natural_disasters.comm_plan.json deleted file mode 100644 index ccaa9ebac..000000000 --- a/vegafusion-rt-datafusion/tests/specs/custom/circle_natural_disasters.comm_plan.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "server_to_client": [ - { - "namespace": "data", - "name": "source_0", - "scope": [] - }, - { - "namespace": "data", - "name": "source_0_color_domain_Entity", - "scope": [] - }, - { - "namespace": "data", - "name": "source_0_size_domain_Deaths", - "scope": [] - }, - { - "namespace": "data", - "name": "source_0_y_domain_Entity", - "scope": [] - } - ], - "client_to_server": [] -} \ No newline at end of file diff --git a/vegafusion-rt-datafusion/tests/specs/custom/circle_natural_disasters.vg.json b/vegafusion-rt-datafusion/tests/specs/custom/circle_natural_disasters.vg.json deleted file mode 100644 index a9c90ba66..000000000 --- a/vegafusion-rt-datafusion/tests/specs/custom/circle_natural_disasters.vg.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "$schema": "https://vega.github.io/schema/vega/v5.json", - "background": "white", - "padding": 5, - "width": 600, - "height": 400, - "style": "cell", - "data": [ - { - "name": "source_0", - "url": "https://raw.githubusercontent.com/vega/vega-datasets/master/data/disasters.csv", - "format": {"type": "csv", "parse": {"Year": "date"}}, - "transform": [ - {"type": "filter", "expr": "datum.Entity !== 'All natural disasters'"}, - { - "type": "filter", - "expr": "(isDate(datum[\"Year\"]) || (isValid(datum[\"Year\"]) && isFinite(+datum[\"Year\"]))) && isValid(datum[\"Deaths\"]) && isFinite(+datum[\"Deaths\"])" - } - ] - } - ], - "marks": [ - { - "name": "marks", - "type": "symbol", - "style": ["circle"], - "from": {"data": "source_0"}, - "encode": { - "update": { - "opacity": {"value": 0.8}, - "stroke": {"value": "black"}, - "strokeWidth": {"value": 1}, - "fill": {"scale": "color", "field": "Entity"}, - "ariaRoleDescription": {"value": "circle"}, - "description": { - "signal": "\"Year: \" + (timeFormat(datum[\"Year\"], '%b %d, %Y')) + \"; Entity: \" + (isValid(datum[\"Entity\"]) ? datum[\"Entity\"] : \"\"+datum[\"Entity\"]) + \"; Annual Global Deaths: \" + (format(datum[\"Deaths\"], \"\"))" - }, - "x": {"scale": "x", "field": "Year"}, - "y": {"scale": "y", "field": "Entity"}, - "size": {"scale": "size", "field": "Deaths"}, - "shape": {"value": "circle"} - } - } - } - ], - "scales": [ - { - "name": "x", - "type": "time", - "domain": {"data": "source_0", "field": "Year"}, - "range": [0, {"signal": "width"}] - }, - { - "name": "y", - "type": "point", - "domain": {"data": "source_0", "field": "Entity", "sort": true}, - "range": [0, {"signal": "height"}], - "padding": 0.5 - }, - { - "name": "color", - "type": "ordinal", - "domain": {"data": "source_0", "field": "Entity", "sort": true}, - "range": "category" - }, - { - "name": "size", - "type": "linear", - "domain": {"data": "source_0", "field": "Deaths"}, - "range": [0, 5000], - "zero": true - } - ], - "axes": [ - { - "scale": "x", - "orient": "bottom", - "grid": false, - "title": "Year", - "labelFlush": true, - "labelOverlap": true, - "tickCount": {"signal": "ceil(width/40)"}, - "zindex": 0 - }, - {"scale": "y", "orient": "left", "grid": false, "zindex": 0} - ], - "legends": [ - { - "clipHeight": 30, - "title": "Annual Global Deaths", - "size": "size", - "symbolType": "circle", - "encode": { - "symbols": { - "update": { - "fill": {"value": "black"}, - "stroke": {"value": "black"}, - "fillOpacity": {"value": 0.8}, - "opacity": {"value": 0.8} - } - } - } - } - ] -} \ No newline at end of file diff --git a/vegafusion-rt-datafusion/tests/specs/custom/stacked_bar_weather_timeunit_parameterize_local.comm_plan.json b/vegafusion-rt-datafusion/tests/specs/custom/stacked_bar_weather_timeunit_parameterize_local.comm_plan.json new file mode 100644 index 000000000..d8a792912 --- /dev/null +++ b/vegafusion-rt-datafusion/tests/specs/custom/stacked_bar_weather_timeunit_parameterize_local.comm_plan.json @@ -0,0 +1,10 @@ +{ + "server_to_client": [ + { + "namespace": "data", + "name": "_server_source_0", + "scope": [] + } + ], + "client_to_server": [] +} diff --git a/vegafusion-rt-datafusion/tests/specs/custom/stacked_bar_weather_timeunit_parameterize_local.vg.json b/vegafusion-rt-datafusion/tests/specs/custom/stacked_bar_weather_timeunit_parameterize_local.vg.json new file mode 100644 index 000000000..628ed1651 --- /dev/null +++ b/vegafusion-rt-datafusion/tests/specs/custom/stacked_bar_weather_timeunit_parameterize_local.vg.json @@ -0,0 +1,132 @@ +{ + "$schema": "https://vega.github.io/schema/vega/v5.json", + "background": "white", + "padding": 5, + "height": 200, + "style": "cell", + "data": [ + { + "name": "raw_source_0", + "url": "https://raw.githubusercontent.com/vega/vega-datasets/master/data/seattle-weather.csv", + "format": {"type": "csv", "parse": {"date": "date"}, "delimiter": ","}, + "transform": [ + { + "type": "formula", + "as": "local_date", + "expr": "datetime(year(datum.date), month(datum.date), date(datum.date), hours(datum.date), minutes(datum.date), seconds(datum.date))" + } + ] + }, + { + "name": "source_0", + "source": "raw_source_0", + "transform": [ + { + "field": "local_date", + "type": "timeunit", + "units": ["month"], + "timezone": "local", + "as": ["utcmonth_date", "utcmonth_date_end"] + }, + { + "type": "aggregate", + "groupby": ["utcmonth_date", "weather"], + "ops": ["count"], + "fields": [null], + "as": ["__count"] + }, + { + "type": "stack", + "groupby": ["utcmonth_date"], + "field": "__count", + "sort": {"field": ["weather"], "order": ["descending"]}, + "as": ["__count_start", "__count_end"], + "offset": "zero" + } + ] + } + ], + "signals": [ + {"name": "x_step", "value": 20}, + { + "name": "width", + "update": "bandspace(domain('x').length, 0.1, 0.05) * x_step" + } + ], + "marks": [ + { + "name": "marks", + "type": "rect", + "style": ["bar"], + "from": {"data": "source_0"}, + "encode": { + "update": { + "fill": {"scale": "color", "field": "weather"}, + "ariaRoleDescription": {"value": "bar"}, + "description": { + "signal": "\"Month of the year: \" + (timeFormat(datum[\"utcmonth_date\"], timeUnitSpecifier([\"month\"], {\"year-month\":\"%b %Y \",\"year-month-date\":\"%b %d, %Y \"}))) + \"; Count of Records: \" + (format(datum[\"__count\"], \"\")) + \"; Weather type: \" + (isValid(datum[\"weather\"]) ? datum[\"weather\"] : \"\"+datum[\"weather\"])" + }, + "x": {"scale": "x", "field": "utcmonth_date"}, + "width": {"scale": "x", "band": 1}, + "y": {"scale": "y", "field": "__count_end"}, + "y2": {"scale": "y", "field": "__count_start"} + } + } + } + ], + "scales": [ + { + "name": "x", + "type": "band", + "domain": {"data": "source_0", "field": "utcmonth_date", "sort": true}, + "range": {"step": {"signal": "x_step"}}, + "paddingInner": 0.1, + "paddingOuter": 0.05 + }, + { + "name": "y", + "type": "linear", + "domain": { + "data": "source_0", + "fields": ["__count_start", "__count_end"] + }, + "range": [{"signal": "height"}, 0], + "nice": true, + "zero": true + }, + { + "name": "color", + "type": "ordinal", + "domain": ["sun", "fog", "drizzle", "rain", "snow"], + "range": ["#e7ba52", "#c7c7c7", "#aec7e8", "#1f77b4", "#9467bd"] + } + ], + "axes": [ + { + "scale": "y", + "orient": "left", + "gridScale": "x", + "grid": true, + "tickCount": {"signal": "ceil(height/40)"}, + "domain": false, + "labels": false, + "aria": false, + "maxExtent": 0, + "minExtent": 0, + "ticks": false, + "zindex": 0 + }, + { + "scale": "y", + "orient": "left", + "grid": false, + "title": "Count of Records", + "labelOverlap": true, + "tickCount": {"signal": "ceil(height/40)"}, + "zindex": 0 + } + ], + "legends": [ + {"title": "Weather type", "fill": "color", "symbolType": "square"} + ] +} \ No newline at end of file diff --git a/vegafusion-rt-datafusion/tests/test_expression_evaluation.rs b/vegafusion-rt-datafusion/tests/test_expression_evaluation.rs index dd6a6082f..cdc432164 100644 --- a/vegafusion-rt-datafusion/tests/test_expression_evaluation.rs +++ b/vegafusion-rt-datafusion/tests/test_expression_evaluation.rs @@ -319,33 +319,69 @@ mod test_datetime { } mod test_date_parts { - use crate::*; - #[rstest( expr, - case("year(datetime(utc(87, 3, 10, 7, 35, 10, 87))) + 0"), - case("utcyear(datetime(utc(87, 3, 10, 7, 35, 10, 87))) + 0"), - case("quarter(datetime(utc(87, 3, 10, 7, 35, 10, 87))) + 0"), - case("utcquarter(datetime(utc(87, 3, 10, 7, 35, 10, 87))) + 0"), - case("month(datetime(utc(87, 3, 10, 7, 35, 10, 87))) + 0"), - case("utcmonth(datetime(utc(87, 3, 10, 7, 35, 10, 87))) + 0"), - case("day(datetime(utc(87, 3, 10, 7, 35, 10, 87))) + 0"), - case("utcday(datetime(utc(87, 3, 10, 7, 35, 10, 87))) + 0"), - case("date(datetime(utc(87, 3, 10, 7, 35, 10, 87))) + 0"), - case("utcdate(datetime(utc(87, 3, 10, 7, 35, 10, 87))) + 0"), - case("dayofyear(datetime(utc(87, 3, 10, 7, 35, 10, 87))) + 0"), - case("utcdayofyear(datetime(utc(87, 3, 10, 7, 35, 10, 87))) + 0"), - case("hours(datetime(utc(87, 3, 10, 7, 35, 10, 87))) + 0"), - case("utchours(datetime(utc(87, 3, 10, 7, 35, 10, 87))) + 0"), - case("minutes(datetime(utc(87, 3, 10, 7, 35, 10, 87))) + 0"), - case("utcminutes(datetime(utc(87, 3, 10, 7, 35, 10, 87))) + 0"), - case("seconds(datetime(utc(87, 3, 10, 7, 35, 10, 87))) + 0"), - case("utcseconds(datetime(utc(87, 3, 10, 7, 35, 10, 87))) + 0") + case("year(datetime(utc(87, 3, 10, 7, 35, 10, 87)))"), + case("year(utc(87, 3, 10, 7, 35, 10, 87))"), + case("year(datetime(87, 3, 10, 7, 35, 10, 87))"), + case("utcyear(datetime(utc(87, 3, 10, 7, 35, 10, 87)))"), + case("utcyear(utc(87, 3, 10, 7, 35, 10, 87))"), + case("utcyear(datetime(87, 3, 10, 7, 35, 10, 87))"), + case("quarter(datetime(utc(87, 3, 10, 7, 35, 10, 87)))"), + case("quarter(utc(87, 3, 10, 7, 35, 10, 87))"), + case("quarter(datetime(87, 3, 10, 7, 35, 10, 87))"), + case("utcquarter(datetime(utc(87, 3, 10, 7, 35, 10, 87)))"), + case("utcquarter(utc(87, 3, 10, 7, 35, 10, 87))"), + case("utcquarter(datetime(87, 3, 10, 7, 35, 10, 87))"), + case("month(datetime(utc(87, 3, 10, 7, 35, 10, 87)))"), + case("month(utc(87, 3, 10, 7, 35, 10, 87))"), + case("month(datetime(87, 3, 10, 7, 35, 10, 87))"), + case("utcmonth(datetime(utc(87, 3, 10, 7, 35, 10, 87)))"), + case("utcmonth(utc(87, 3, 10, 7, 35, 10, 87))"), + case("utcmonth(datetime(87, 3, 10, 7, 35, 10, 87))"), + case("day(datetime(utc(87, 3, 10, 7, 35, 10, 87)))"), + case("day(utc(87, 3, 10, 7, 35, 10, 87))"), + case("day(datetime(87, 3, 10, 7, 35, 10, 87))"), + case("utcday(datetime(utc(87, 3, 10, 7, 35, 10, 87)))"), + case("utcday(utc(87, 3, 10, 7, 35, 10, 87))"), + case("utcday(datetime(87, 3, 10, 7, 35, 10, 87))"), + case("date(datetime(utc(87, 3, 10, 7, 35, 10, 87)))"), + case("date(utc(87, 3, 10, 7, 35, 10, 87))"), + case("date(datetime(87, 3, 10, 7, 35, 10, 87))"), + case("utcdate(datetime(utc(87, 3, 10, 7, 35, 10, 87)))"), + case("utcdate(utc(87, 3, 10, 7, 35, 10, 87))"), + case("utcdate(datetime(87, 3, 10, 7, 35, 10, 87))"), + case("dayofyear(datetime(utc(87, 3, 10, 7, 35, 10, 87)))"), + case("dayofyear(utc(87, 3, 10, 7, 35, 10, 87))"), + case("dayofyear(datetime(87, 3, 10, 7, 35, 10, 87))"), + case("utcdayofyear(datetime(utc(87, 3, 10, 7, 35, 10, 87)))"), + case("utcdayofyear(utc(87, 3, 10, 7, 35, 10, 87))"), + case("utcdayofyear(datetime(87, 3, 10, 7, 35, 10, 87))"), + case("hours(datetime(utc(87, 3, 10, 7, 35, 10, 87)))"), + case("hours(utc(87, 3, 10, 7, 35, 10, 87))"), + case("hours(datetime(87, 3, 10, 7, 35, 10, 87))"), + case("utchours(datetime(utc(87, 3, 10, 7, 35, 10, 87)))"), + case("utchours(utc(87, 3, 10, 7, 35, 10, 87))"), + case("utchours(datetime(87, 3, 10, 7, 35, 10, 87))"), + case("minutes(datetime(utc(87, 3, 10, 7, 35, 10, 87)))"), + case("minutes(utc(87, 3, 10, 7, 35, 10, 87))"), + case("minutes(datetime(87, 3, 10, 7, 35, 10, 87))"), + case("utcminutes(datetime(utc(87, 3, 10, 7, 35, 10, 87)))"), + case("utcminutes(utc(87, 3, 10, 7, 35, 10, 87))"), + case("utcminutes(datetime(87, 3, 10, 7, 35, 10, 87))"), + case("seconds(datetime(utc(87, 3, 10, 7, 35, 10, 87)))"), + case("seconds(utc(87, 3, 10, 7, 35, 10, 87))"), + case("seconds(datetime(87, 3, 10, 7, 35, 10, 87))"), + case("utcseconds(datetime(utc(87, 3, 10, 7, 35, 10, 87)))"), + case("utcseconds(utc(87, 3, 10, 7, 35, 10, 87))"), + case("utcseconds(datetime(87, 3, 10, 7, 35, 10, 87))") )] fn test(expr: &str) { check_scalar_evaluation(expr, &config_a()) } + use crate::*; + #[test] fn test_marker() {} // Help IDE detect test module } @@ -356,14 +392,14 @@ mod test_length { #[rstest( expr, // Below: Add 0 to force result type to f64 even though length returns i32 - case("length([1, 2, 3]) + 0"), - case("[1, 2, 3].length + 0"), - case("length('abc') + 0"), - case("'abc'.length + 0"), - case("hello.length + 0"), - case("length(hello) + 0"), - case("length(data('dataB')) + 0"), - case("data('dataB').length + 0"), + case("length([1, 2, 3])"), + case("[1, 2, 3].length"), + case("length('abc')"), + case("'abc'.length"), + case("hello.length"), + case("length(hello)"), + case("length(data('dataB'))"), + case("data('dataB').length"), )] fn test(expr: &str) { check_scalar_evaluation(expr, &config_a()) diff --git a/vegafusion-rt-datafusion/tests/test_image_comparison.rs b/vegafusion-rt-datafusion/tests/test_image_comparison.rs index 0210274fb..29f1f9e88 100644 --- a/vegafusion-rt-datafusion/tests/test_image_comparison.rs +++ b/vegafusion-rt-datafusion/tests/test_image_comparison.rs @@ -99,7 +99,6 @@ mod test_custom_specs { case("custom/joinaggregate_text_color_contrast", 0.001), case("custom/cumulative_running_window", 0.001), case("custom/point_bubble", 0.001), - case("custom/circle_natural_disasters", 0.001), case("custom/circle_bubble_health_income", 0.001), case("custom/line_color_stocks", 0.001), case("custom/line_slope_barley", 0.001), @@ -448,7 +447,10 @@ mod test_vegalite_specs { case("vegalite/circle_flatten", 0.001), case("vegalite/circle_github_punchcard", 0.001), case("vegalite/circle_labelangle_orient_signal", 0.001), - case("vegalite/circle_natural_disasters", 0.001), + + // "Ambiguous reference to field" error related to + // https://github.com/apache/arrow-datafusion/issues/1411 + // case("vegalite/circle_natural_disasters", 0.001), case("vegalite/circle_opacity", 0.001), case("vegalite/circle_scale_quantile", 0.001), case("vegalite/circle_scale_quantize", 0.001), @@ -922,10 +924,14 @@ mod test_image_comparison_timeunit { #[values( "custom/bar_month_temporal_initial_parameterize", - "custom/stacked_bar_weather_timeunit_parameterize" + "custom/bar_month_temporal_initial_parameterize_local", + "custom/stacked_bar_weather_timeunit_parameterize", + "custom/stacked_bar_weather_timeunit_parameterize_local", )] spec_name: &str, ) { + initialize(); + // Load spec let mut full_spec = load_spec(spec_name); @@ -936,9 +942,10 @@ mod test_image_comparison_timeunit { let watch_plan = load_expected_watch_plan(spec_name); // Modify transform spec + let num_data = full_spec.data.len(); let timeunit_tx = full_spec .data - .get_mut(0) + .get_mut( num_data - 1) .unwrap() .transform .get_mut(0) diff --git a/vegafusion-rt-datafusion/tests/util/check.rs b/vegafusion-rt-datafusion/tests/util/check.rs index f1a1b7ca6..ba7aa9178 100644 --- a/vegafusion-rt-datafusion/tests/util/check.rs +++ b/vegafusion-rt-datafusion/tests/util/check.rs @@ -21,6 +21,7 @@ use crate::util::vegajs_runtime::vegajs_runtime; use datafusion::scalar::ScalarValue; use std::convert::TryFrom; +use vegafusion_core::data::scalar::ScalarValueHelpers; use vegafusion_core::data::table::VegaFusionTable; use vegafusion_core::error::Result; @@ -70,6 +71,10 @@ pub fn check_scalar_evaluation(expr_str: &str, config: &CompilationConfig) { let compiled = compile(&parsed, config, None).unwrap(); let result = compiled.eval_to_scalar().unwrap(); + // Serialize and deserialize to normalize types to those supported by JavaScript + // (e.g. Int to Float) + let result = ScalarValue::from_json(&result.to_json().unwrap()).unwrap(); + println!("{:?}", result); assert_eq!( result, diff --git a/vegafusion-rt-datafusion/tests/util/vegajs_runtime/package-lock.json b/vegafusion-rt-datafusion/tests/util/vegajs_runtime/package-lock.json index ad3923e67..a42cfdd15 100644 --- a/vegafusion-rt-datafusion/tests/util/vegajs_runtime/package-lock.json +++ b/vegafusion-rt-datafusion/tests/util/vegajs_runtime/package-lock.json @@ -1,1354 +1,8 @@ { "name": "diorite-testing", "version": "1.0.0", - "lockfileVersion": 2, + "lockfileVersion": 1, "requires": true, - "packages": { - "": { - "name": "diorite-testing", - "version": "1.0.0", - "license": "ISC", - "dependencies": { - "canvas": "^2.8.0", - "fs": "^0.0.1-security", - "lodash": "^4.17.21", - "moment": "^2.29.1", - "svgo": "^2.6.1", - "vega": "^5.20.2" - } - }, - "node_modules/@mapbox/node-pre-gyp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.5.tgz", - "integrity": "sha512-4srsKPXWlIxp5Vbqz5uLfBN+du2fJChBoYn/f2h991WLdk7jUvcSk/McVLSv/X+xQIPI8eGD5GjrnygdyHnhPA==", - "dependencies": { - "detect-libc": "^1.0.3", - "https-proxy-agent": "^5.0.0", - "make-dir": "^3.1.0", - "node-fetch": "^2.6.1", - "nopt": "^5.0.0", - "npmlog": "^4.1.2", - "rimraf": "^3.0.2", - "semver": "^7.3.4", - "tar": "^6.1.0" - }, - "bin": { - "node-pre-gyp": "bin/node-pre-gyp" - } - }, - "node_modules/@trysound/sax": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/@trysound/sax/-/sax-0.2.0.tgz", - "integrity": "sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==", - "engines": { - "node": ">=10.13.0" - } - }, - "node_modules/@types/estree": { - "version": "0.0.50", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.50.tgz", - "integrity": "sha512-C6N5s2ZFtuZRj54k2/zyRhNDjJwwcViAM3Nbm8zjBpbqAdZ00mr0CFxvSKeO8Y/e03WVFLpQMdHYVfUd6SB+Hw==" - }, - "node_modules/abbrev": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", - "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" - }, - "node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/aproba": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==" - }, - "node_modules/are-we-there-yet": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.7.tgz", - "integrity": "sha512-nxwy40TuMiUGqMyRHgCSWZ9FM4VAoRP4xUYSTv5ImRog+h9yISPbVH7H8fASCIzYn9wlEv4zvFL7uKDMCFQm3g==", - "dependencies": { - "delegates": "^1.0.0", - "readable-stream": "^2.0.6" - } - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" - }, - "node_modules/boolbase": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", - "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" - }, - "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/canvas": { - "version": "2.8.0", - "resolved": "https://registry.npmjs.org/canvas/-/canvas-2.8.0.tgz", - "integrity": "sha512-gLTi17X8WY9Cf5GZ2Yns8T5lfBOcGgFehDFb+JQwDqdOoBOcECS9ZWMEAqMSVcMYwXD659J8NyzjRY/2aE+C2Q==", - "hasInstallScript": true, - "dependencies": { - "@mapbox/node-pre-gyp": "^1.0.0", - "nan": "^2.14.0", - "simple-get": "^3.0.3" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/chownr": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", - "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", - "engines": { - "node": ">=10" - } - }, - "node_modules/code-point-at": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", - "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/commander": { - "version": "2.20.3", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" - }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" - }, - "node_modules/console-control-strings": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", - "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" - }, - "node_modules/core-util-is": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", - "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" - }, - "node_modules/css-select": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-4.1.3.tgz", - "integrity": "sha512-gT3wBNd9Nj49rAbmtFHj1cljIAOLYSX1nZ8CB7TBO3INYckygm5B7LISU/szY//YmdiSLbJvDLOx9VnMVpMBxA==", - "dependencies": { - "boolbase": "^1.0.0", - "css-what": "^5.0.0", - "domhandler": "^4.2.0", - "domutils": "^2.6.0", - "nth-check": "^2.0.0" - } - }, - "node_modules/css-tree": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-1.1.3.tgz", - "integrity": "sha512-tRpdppF7TRazZrjJ6v3stzv93qxRcSsFmW6cX0Zm2NVKpxE1WV1HblnghVv9TreireHkqI/VDEsfolRF1p6y7Q==", - "dependencies": { - "mdn-data": "2.0.14", - "source-map": "^0.6.1" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/css-what": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/css-what/-/css-what-5.0.1.tgz", - "integrity": "sha512-FYDTSHb/7KXsWICVsxdmiExPjCfRC4qRFBdVwv7Ax9hMnvMmEjP9RfxTEZ3qPZGmADDn2vAKSo9UcN1jKVYscg==", - "engines": { - "node": ">= 6" - } - }, - "node_modules/csso": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/csso/-/csso-4.2.0.tgz", - "integrity": "sha512-wvlcdIbf6pwKEk7vHj8/Bkc0B4ylXZruLvOgs9doS5eOsOpuodOV2zJChSpkp+pRpYQLQMeF04nr3Z68Sta9jA==", - "dependencies": { - "css-tree": "^1.1.2" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/d3-array": { - "version": "2.12.1", - "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz", - "integrity": "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==", - "dependencies": { - "internmap": "^1.0.0" - } - }, - "node_modules/d3-color": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-2.0.0.tgz", - "integrity": "sha512-SPXi0TSKPD4g9tw0NMZFnR95XVgUZiBH+uUTqQuDu1OsE2zomHU7ho0FISciaPvosimixwHFl3WHLGabv6dDgQ==" - }, - "node_modules/d3-delaunay": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/d3-delaunay/-/d3-delaunay-5.3.0.tgz", - "integrity": "sha512-amALSrOllWVLaHTnDLHwMIiz0d1bBu9gZXd1FiLfXf8sHcX9jrcj81TVZOqD4UX7MgBZZ07c8GxzEgBpJqc74w==", - "dependencies": { - "delaunator": "4" - } - }, - "node_modules/d3-dispatch": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-2.0.0.tgz", - "integrity": "sha512-S/m2VsXI7gAti2pBoLClFFTMOO1HTtT0j99AuXLoGFKO6deHDdnv6ZGTxSTTUTgO1zVcv82fCOtDjYK4EECmWA==" - }, - "node_modules/d3-dsv": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-2.0.0.tgz", - "integrity": "sha512-E+Pn8UJYx9mViuIUkoc93gJGGYut6mSDKy2+XaPwccwkRGlR+LO97L2VCCRjQivTwLHkSnAJG7yo00BWY6QM+w==", - "dependencies": { - "commander": "2", - "iconv-lite": "0.4", - "rw": "1" - }, - "bin": { - "csv2json": "bin/dsv2json", - "csv2tsv": "bin/dsv2dsv", - "dsv2dsv": "bin/dsv2dsv", - "dsv2json": "bin/dsv2json", - "json2csv": "bin/json2dsv", - "json2dsv": "bin/json2dsv", - "json2tsv": "bin/json2dsv", - "tsv2csv": "bin/dsv2dsv", - "tsv2json": "bin/dsv2json" - } - }, - "node_modules/d3-force": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/d3-force/-/d3-force-2.1.1.tgz", - "integrity": "sha512-nAuHEzBqMvpFVMf9OX75d00OxvOXdxY+xECIXjW6Gv8BRrXu6gAWbv/9XKrvfJ5i5DCokDW7RYE50LRoK092ew==", - "dependencies": { - "d3-dispatch": "1 - 2", - "d3-quadtree": "1 - 2", - "d3-timer": "1 - 2" - } - }, - "node_modules/d3-format": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-2.0.0.tgz", - "integrity": "sha512-Ab3S6XuE/Q+flY96HXT0jOXcM4EAClYFnRGY5zsjRGNy6qCYrQsMffs7cV5Q9xejb35zxW5hf/guKw34kvIKsA==" - }, - "node_modules/d3-geo": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-2.0.2.tgz", - "integrity": "sha512-8pM1WGMLGFuhq9S+FpPURxic+gKzjluCD/CHTuUF3mXMeiCo0i6R0tO1s4+GArRFde96SLcW/kOFRjoAosPsFA==", - "dependencies": { - "d3-array": "^2.5.0" - } - }, - "node_modules/d3-geo-projection": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/d3-geo-projection/-/d3-geo-projection-3.0.0.tgz", - "integrity": "sha512-1JE+filVbkEX2bT25dJdQ05iA4QHvUwev6o0nIQHOSrNlHCAKfVss/U10vEM3pA4j5v7uQoFdQ4KLbx9BlEbWA==", - "dependencies": { - "commander": "2", - "d3-array": "1 - 2", - "d3-geo": "1.12.0 - 2", - "resolve": "^1.1.10" - }, - "bin": { - "geo2svg": "bin/geo2svg", - "geograticule": "bin/geograticule", - "geoproject": "bin/geoproject", - "geoquantize": "bin/geoquantize", - "geostitch": "bin/geostitch" - } - }, - "node_modules/d3-hierarchy": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-2.0.0.tgz", - "integrity": "sha512-SwIdqM3HxQX2214EG9GTjgmCc/mbSx4mQBn+DuEETubhOw6/U3fmnji4uCVrmzOydMHSO1nZle5gh6HB/wdOzw==" - }, - "node_modules/d3-interpolate": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-2.0.1.tgz", - "integrity": "sha512-c5UhwwTs/yybcmTpAVqwSFl6vrQ8JZJoT5F7xNFK9pymv5C0Ymcc9/LIJHtYIggg/yS9YHw8i8O8tgb9pupjeQ==", - "dependencies": { - "d3-color": "1 - 2" - } - }, - "node_modules/d3-path": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-2.0.0.tgz", - "integrity": "sha512-ZwZQxKhBnv9yHaiWd6ZU4x5BtCQ7pXszEV9CU6kRgwIQVQGLMv1oiL4M+MK/n79sYzsj+gcgpPQSctJUsLN7fA==" - }, - "node_modules/d3-quadtree": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-2.0.0.tgz", - "integrity": "sha512-b0Ed2t1UUalJpc3qXzKi+cPGxeXRr4KU9YSlocN74aTzp6R/Ud43t79yLLqxHRWZfsvWXmbDWPpoENK1K539xw==" - }, - "node_modules/d3-scale": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-3.3.0.tgz", - "integrity": "sha512-1JGp44NQCt5d1g+Yy+GeOnZP7xHo0ii8zsQp6PGzd+C1/dl0KGsp9A7Mxwp+1D1o4unbTTxVdU/ZOIEBoeZPbQ==", - "dependencies": { - "d3-array": "^2.3.0", - "d3-format": "1 - 2", - "d3-interpolate": "1.2.0 - 2", - "d3-time": "^2.1.1", - "d3-time-format": "2 - 3" - } - }, - "node_modules/d3-shape": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-2.1.0.tgz", - "integrity": "sha512-PnjUqfM2PpskbSLTJvAzp2Wv4CZsnAgTfcVRTwW03QR3MkXF8Uo7B1y/lWkAsmbKwuecto++4NlsYcvYpXpTHA==", - "dependencies": { - "d3-path": "1 - 2" - } - }, - "node_modules/d3-time": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-2.1.1.tgz", - "integrity": "sha512-/eIQe/eR4kCQwq7yxi7z4c6qEXf2IYGcjoWB5OOQy4Tq9Uv39/947qlDcN2TLkiTzQWzvnsuYPB9TrWaNfipKQ==", - "dependencies": { - "d3-array": "2" - } - }, - "node_modules/d3-time-format": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-3.0.0.tgz", - "integrity": "sha512-UXJh6EKsHBTjopVqZBhFysQcoXSv/5yLONZvkQ5Kk3qbwiUYkdX17Xa1PT6U1ZWXGGfB1ey5L8dKMlFq2DO0Ag==", - "dependencies": { - "d3-time": "1 - 2" - } - }, - "node_modules/d3-timer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-2.0.0.tgz", - "integrity": "sha512-TO4VLh0/420Y/9dO3+f9abDEFYeCUr2WZRlxJvbp4HPTQcSylXNiL6yZa9FIUvV1yRiFufl1bszTCLDqv9PWNA==" - }, - "node_modules/debug": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", - "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - } - }, - "node_modules/decompress-response": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", - "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", - "dependencies": { - "mimic-response": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/delaunator": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/delaunator/-/delaunator-4.0.1.tgz", - "integrity": "sha512-WNPWi1IRKZfCt/qIDMfERkDp93+iZEmOxN2yy4Jg+Xhv8SLk2UTqqbe1sfiipn0and9QrE914/ihdx82Y/Giag==" - }, - "node_modules/delegates": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", - "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" - }, - "node_modules/detect-libc": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", - "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=", - "bin": { - "detect-libc": "bin/detect-libc.js" - }, - "engines": { - "node": ">=0.10" - } - }, - "node_modules/dom-serializer": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.3.2.tgz", - "integrity": "sha512-5c54Bk5Dw4qAxNOI1pFEizPSjVsx5+bpJKmL2kPn8JhBUq2q09tTCa3mjijun2NfK78NMouDYNMBkOrPZiS+ig==", - "dependencies": { - "domelementtype": "^2.0.1", - "domhandler": "^4.2.0", - "entities": "^2.0.0" - } - }, - "node_modules/domelementtype": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.2.0.tgz", - "integrity": "sha512-DtBMo82pv1dFtUmHyr48beiuq792Sxohr+8Hm9zoxklYPfa6n0Z3Byjj2IV7bmr2IyqClnqEQhfgHJJ5QF0R5A==" - }, - "node_modules/domhandler": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.2.2.tgz", - "integrity": "sha512-PzE9aBMsdZO8TK4BnuJwH0QT41wgMbRzuZrHUcpYncEjmQazq8QEaBWgLG7ZyC/DAZKEgglpIA6j4Qn/HmxS3w==", - "dependencies": { - "domelementtype": "^2.2.0" - }, - "engines": { - "node": ">= 4" - } - }, - "node_modules/domutils": { - "version": "2.8.0", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.8.0.tgz", - "integrity": "sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==", - "dependencies": { - "dom-serializer": "^1.0.1", - "domelementtype": "^2.2.0", - "domhandler": "^4.2.0" - } - }, - "node_modules/entities": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", - "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==" - }, - "node_modules/fs": { - "version": "0.0.1-security", - "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", - "integrity": "sha1-invTcYa23d84E/I4WLV+yq9eQdQ=" - }, - "node_modules/fs-minipass": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", - "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", - "dependencies": { - "minipass": "^3.0.0" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" - }, - "node_modules/function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" - }, - "node_modules/gauge": { - "version": "2.7.4", - "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", - "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", - "dependencies": { - "aproba": "^1.0.3", - "console-control-strings": "^1.0.0", - "has-unicode": "^2.0.0", - "object-assign": "^4.1.0", - "signal-exit": "^3.0.0", - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1", - "wide-align": "^1.1.0" - } - }, - "node_modules/glob": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", - "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - } - }, - "node_modules/has": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", - "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dependencies": { - "function-bind": "^1.1.1" - }, - "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/has-unicode": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", - "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=" - }, - "node_modules/https-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", - "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "dependencies": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" - }, - "node_modules/internmap": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/internmap/-/internmap-1.0.1.tgz", - "integrity": "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==" - }, - "node_modules/is-core-module": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.7.0.tgz", - "integrity": "sha512-ByY+tjCciCr+9nLryBYcSD50EOGWt95c7tIsKTG1J2ixKKXPvF7Ej3AVd+UfDydAJom3biBGDBALaO79ktwgEQ==", - "dependencies": { - "has": "^1.0.3" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", - "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", - "dependencies": { - "number-is-nan": "^1.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "node_modules/lodash": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" - }, - "node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/make-dir": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", - "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", - "dependencies": { - "semver": "^6.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/make-dir/node_modules/semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/mdn-data": { - "version": "2.0.14", - "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.14.tgz", - "integrity": "sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow==" - }, - "node_modules/mimic-response": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", - "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==", - "engines": { - "node": ">=8" - } - }, - "node_modules/minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/minipass": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.1.5.tgz", - "integrity": "sha512-+8NzxD82XQoNKNrl1d/FSi+X8wAEWR+sbYAfIvub4Nz0d22plFG72CEVVaufV8PNf4qSslFTD8VMOxNVhHCjTw==", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/minizlib": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", - "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", - "dependencies": { - "minipass": "^3.0.0", - "yallist": "^4.0.0" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/mkdirp": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", - "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", - "bin": { - "mkdirp": "bin/cmd.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/moment": { - "version": "2.29.1", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", - "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==", - "engines": { - "node": "*" - } - }, - "node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "node_modules/nan": { - "version": "2.15.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.15.0.tgz", - "integrity": "sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==" - }, - "node_modules/nanocolors": { - "version": "0.1.12", - "resolved": "https://registry.npmjs.org/nanocolors/-/nanocolors-0.1.12.tgz", - "integrity": "sha512-2nMHqg1x5PU+unxX7PGY7AuYxl2qDx7PSrTRjizr8sxdd3l/3hBuWWaki62qmtYm2U5i4Z5E7GbjlyDFhs9/EQ==" - }, - "node_modules/node-fetch": { - "version": "2.6.5", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz", - "integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - } - }, - "node_modules/nopt": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", - "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", - "dependencies": { - "abbrev": "1" - }, - "bin": { - "nopt": "bin/nopt.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/npmlog": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", - "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", - "dependencies": { - "are-we-there-yet": "~1.1.2", - "console-control-strings": "~1.1.0", - "gauge": "~2.7.3", - "set-blocking": "~2.0.0" - } - }, - "node_modules/nth-check": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.0.1.tgz", - "integrity": "sha512-it1vE95zF6dTT9lBsYbxvqh0Soy4SPowchj0UBGj/V6cTPnXXtQOPUbhZ6CmGzAD/rW22LQK6E96pcdJXk4A4w==", - "dependencies": { - "boolbase": "^1.0.0" - } - }, - "node_modules/number-is-nan": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/object-assign": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/path-parse": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", - "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" - }, - "node_modules/process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, - "node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/resolve": { - "version": "1.20.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz", - "integrity": "sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==", - "dependencies": { - "is-core-module": "^2.2.0", - "path-parse": "^1.0.6" - } - }, - "node_modules/rimraf": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", - "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", - "dependencies": { - "glob": "^7.1.3" - }, - "bin": { - "rimraf": "bin.js" - } - }, - "node_modules/rw": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", - "integrity": "sha1-P4Yt+pGrdmsUiF700BEkv9oHT7Q=" - }, - "node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, - "node_modules/semver": { - "version": "7.3.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", - "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", - "dependencies": { - "lru-cache": "^6.0.0" - }, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/set-blocking": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" - }, - "node_modules/signal-exit": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.5.tgz", - "integrity": "sha512-KWcOiKeQj6ZyXx7zq4YxSMgHRlod4czeBQZrPb8OKcohcqAXShm7E20kEMle9WBt26hFcAf0qLOcp5zmY7kOqQ==" - }, - "node_modules/simple-concat": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", - "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==" - }, - "node_modules/simple-get": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz", - "integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==", - "dependencies": { - "decompress-response": "^4.2.0", - "once": "^1.3.1", - "simple-concat": "^1.0.0" - } - }, - "node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/stable": { - "version": "0.1.8", - "resolved": "https://registry.npmjs.org/stable/-/stable-0.1.8.tgz", - "integrity": "sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==" - }, - "node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, - "node_modules/string-width": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", - "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "dependencies": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "dependencies": { - "ansi-regex": "^2.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/svgo": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/svgo/-/svgo-2.7.0.tgz", - "integrity": "sha512-aDLsGkre4fTDCWvolyW+fs8ZJFABpzLXbtdK1y71CKnHzAnpDxKXPj2mNKj+pyOXUCzFHzuxRJ94XOFygOWV3w==", - "dependencies": { - "@trysound/sax": "0.2.0", - "commander": "^7.2.0", - "css-select": "^4.1.3", - "css-tree": "^1.1.3", - "csso": "^4.2.0", - "nanocolors": "^0.1.12", - "stable": "^0.1.8" - }, - "bin": { - "svgo": "bin/svgo" - }, - "engines": { - "node": ">=10.13.0" - } - }, - "node_modules/svgo/node_modules/commander": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", - "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", - "engines": { - "node": ">= 10" - } - }, - "node_modules/tar": { - "version": "6.1.11", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.11.tgz", - "integrity": "sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA==", - "dependencies": { - "chownr": "^2.0.0", - "fs-minipass": "^2.0.0", - "minipass": "^3.0.0", - "minizlib": "^2.1.1", - "mkdirp": "^1.0.3", - "yallist": "^4.0.0" - }, - "engines": { - "node": ">= 10" - } - }, - "node_modules/topojson-client": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/topojson-client/-/topojson-client-3.1.0.tgz", - "integrity": "sha512-605uxS6bcYxGXw9qi62XyrV6Q3xwbndjachmNxu8HWTtVPxZfEJN9fd/SZS1Q54Sn2y0TMyMxFj/cJINqGHrKw==", - "dependencies": { - "commander": "2" - }, - "bin": { - "topo2geo": "bin/topo2geo", - "topomerge": "bin/topomerge", - "topoquantize": "bin/topoquantize" - } - }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" - }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" - }, - "node_modules/vega": { - "version": "5.21.0", - "resolved": "https://registry.npmjs.org/vega/-/vega-5.21.0.tgz", - "integrity": "sha512-yqqRa9nAqYoAxe7sVhRpsh0b001fly7Yx05klPkXmrvzjxXd07gClW1mOuGgSnVQqo7jTp/LYgbO1bD37FbEig==", - "dependencies": { - "vega-crossfilter": "~4.0.5", - "vega-dataflow": "~5.7.4", - "vega-encode": "~4.8.3", - "vega-event-selector": "~3.0.0", - "vega-expression": "~5.0.0", - "vega-force": "~4.0.7", - "vega-format": "~1.0.4", - "vega-functions": "~5.12.1", - "vega-geo": "~4.3.8", - "vega-hierarchy": "~4.0.9", - "vega-label": "~1.1.0", - "vega-loader": "~4.4.1", - "vega-parser": "~6.1.4", - "vega-projection": "~1.4.5", - "vega-regression": "~1.0.9", - "vega-runtime": "~6.1.3", - "vega-scale": "~7.1.1", - "vega-scenegraph": "~4.9.4", - "vega-statistics": "~1.7.10", - "vega-time": "~2.0.4", - "vega-transforms": "~4.9.4", - "vega-typings": "~0.22.0", - "vega-util": "~1.17.0", - "vega-view": "~5.10.1", - "vega-view-transforms": "~4.5.8", - "vega-voronoi": "~4.1.5", - "vega-wordcloud": "~4.1.3" - } - }, - "node_modules/vega-canvas": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/vega-canvas/-/vega-canvas-1.2.6.tgz", - "integrity": "sha512-rgeYUpslYn/amIfnuv3Sw6n4BGns94OjjZNtUc9IDji6b+K8LGS/kW+Lvay8JX/oFqtulBp8RLcHN6QjqPLA9Q==" - }, - "node_modules/vega-crossfilter": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/vega-crossfilter/-/vega-crossfilter-4.0.5.tgz", - "integrity": "sha512-yF+iyGP+ZxU7Tcj5yBsMfoUHTCebTALTXIkBNA99RKdaIHp1E690UaGVLZe6xde2n5WaYpho6I/I6wdAW3NXcg==", - "dependencies": { - "d3-array": "^2.7.1", - "vega-dataflow": "^5.7.3", - "vega-util": "^1.15.2" - } - }, - "node_modules/vega-dataflow": { - "version": "5.7.4", - "resolved": "https://registry.npmjs.org/vega-dataflow/-/vega-dataflow-5.7.4.tgz", - "integrity": "sha512-JGHTpUo8XGETH3b1V892we6hdjzCWB977ybycIu8DPqRoyrZuj6t1fCVImazfMgQD1LAfJlQybWP+alwKDpKig==", - "dependencies": { - "vega-format": "^1.0.4", - "vega-loader": "^4.3.2", - "vega-util": "^1.16.1" - } - }, - "node_modules/vega-encode": { - "version": "4.8.3", - "resolved": "https://registry.npmjs.org/vega-encode/-/vega-encode-4.8.3.tgz", - "integrity": "sha512-JoRYtaV2Hs8spWLzTu/IjR7J9jqRmuIOEicAaWj6T9NSZrNWQzu2zF3IVsX85WnrIDIRUDaehXaFZvy9uv9RQg==", - "dependencies": { - "d3-array": "^2.7.1", - "d3-interpolate": "^2.0.1", - "vega-dataflow": "^5.7.3", - "vega-scale": "^7.0.3", - "vega-util": "^1.15.2" - } - }, - "node_modules/vega-event-selector": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/vega-event-selector/-/vega-event-selector-3.0.0.tgz", - "integrity": "sha512-Gls93/+7tEJGE3kUuUnxrBIxtvaNeF01VIFB2Q2Of2hBIBvtHX74jcAdDtkh5UhhoYGD8Q1J30P5cqEBEwtPoQ==" - }, - "node_modules/vega-expression": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/vega-expression/-/vega-expression-5.0.0.tgz", - "integrity": "sha512-y5+c2frq0tGwJ7vYXzZcfVcIRF/QGfhf2e+bV1Z0iQs+M2lI1II1GPDdmOcMKimpoCVp/D61KUJDIGE1DSmk2w==", - "dependencies": { - "@types/estree": "^0.0.50", - "vega-util": "^1.16.0" - } - }, - "node_modules/vega-force": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/vega-force/-/vega-force-4.0.7.tgz", - "integrity": "sha512-pyLKdwXSZ9C1dVIqdJOobvBY29rLvZjvRRTla9BU/nMwAiAGlGi6WKUFdRGdneyGe3zo2nSZDTZlZM/Z5VaQNA==", - "dependencies": { - "d3-force": "^2.1.1", - "vega-dataflow": "^5.7.3", - "vega-util": "^1.15.2" - } - }, - "node_modules/vega-format": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/vega-format/-/vega-format-1.0.4.tgz", - "integrity": "sha512-oTAeub3KWm6nKhXoYCx1q9G3K43R6/pDMXvqDlTSUtjoY7b/Gixm8iLcir5S9bPjvH40n4AcbZsPmNfL/Up77A==", - "dependencies": { - "d3-array": "^2.7.1", - "d3-format": "^2.0.0", - "d3-time-format": "^3.0.0", - "vega-time": "^2.0.3", - "vega-util": "^1.15.2" - } - }, - "node_modules/vega-functions": { - "version": "5.12.1", - "resolved": "https://registry.npmjs.org/vega-functions/-/vega-functions-5.12.1.tgz", - "integrity": "sha512-7cHfcjXOj27qEbh2FTzWDl7FJK4xGcMFF7+oiyqa0fp7BU/wNT5YdNV0t5kCX9WjV7mfJWACKV74usLJbyM6GA==", - "dependencies": { - "d3-array": "^2.7.1", - "d3-color": "^2.0.0", - "d3-geo": "^2.0.1", - "vega-dataflow": "^5.7.3", - "vega-expression": "^5.0.0", - "vega-scale": "^7.1.1", - "vega-scenegraph": "^4.9.3", - "vega-selections": "^5.3.1", - "vega-statistics": "^1.7.9", - "vega-time": "^2.0.4", - "vega-util": "^1.16.0" - } - }, - "node_modules/vega-geo": { - "version": "4.3.8", - "resolved": "https://registry.npmjs.org/vega-geo/-/vega-geo-4.3.8.tgz", - "integrity": "sha512-fsGxV96Q/QRgPqOPtMBZdI+DneIiROKTG3YDZvGn0EdV16OG5LzFhbNgLT5GPzI+kTwgLpAsucBHklexlB4kfg==", - "dependencies": { - "d3-array": "^2.7.1", - "d3-color": "^2.0.0", - "d3-geo": "^2.0.1", - "vega-canvas": "^1.2.5", - "vega-dataflow": "^5.7.3", - "vega-projection": "^1.4.5", - "vega-statistics": "^1.7.9", - "vega-util": "^1.15.2" - } - }, - "node_modules/vega-hierarchy": { - "version": "4.0.9", - "resolved": "https://registry.npmjs.org/vega-hierarchy/-/vega-hierarchy-4.0.9.tgz", - "integrity": "sha512-4XaWK6V38/QOZ+vllKKTafiwL25m8Kd+ebHmDV+Q236ONHmqc/gv82wwn9nBeXPEfPv4FyJw2SRoqa2Jol6fug==", - "dependencies": { - "d3-hierarchy": "^2.0.0", - "vega-dataflow": "^5.7.3", - "vega-util": "^1.15.2" - } - }, - "node_modules/vega-label": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/vega-label/-/vega-label-1.1.0.tgz", - "integrity": "sha512-LAThIiDEsZxYvbSkvPLJ93eJF+Ts8RXv1IpBh8gmew8XGmaLJvVkzdsMe7WJJwuaVEsK7ZZFyB/Inkp842GW6w==", - "dependencies": { - "vega-canvas": "^1.2.5", - "vega-dataflow": "^5.7.3", - "vega-scenegraph": "^4.9.2", - "vega-util": "^1.15.2" - } - }, - "node_modules/vega-loader": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/vega-loader/-/vega-loader-4.4.1.tgz", - "integrity": "sha512-dj65i4qlNhK0mOmjuchHgUrF5YUaWrYpx0A8kXA68lBk5Hkx8FNRztkcl07CZJ1+8V81ymEyJii9jzGbhEX0ag==", - "dependencies": { - "d3-dsv": "^2.0.0", - "node-fetch": "^2.6.1", - "topojson-client": "^3.1.0", - "vega-format": "^1.0.4", - "vega-util": "^1.16.0" - } - }, - "node_modules/vega-parser": { - "version": "6.1.4", - "resolved": "https://registry.npmjs.org/vega-parser/-/vega-parser-6.1.4.tgz", - "integrity": "sha512-tORdpWXiH/kkXcpNdbSVEvtaxBuuDtgYp9rBunVW9oLsjFvFXbSWlM1wvJ9ZFSaTfx6CqyTyGMiJemmr1QnTjQ==", - "dependencies": { - "vega-dataflow": "^5.7.3", - "vega-event-selector": "^3.0.0", - "vega-functions": "^5.12.1", - "vega-scale": "^7.1.1", - "vega-util": "^1.16.0" - } - }, - "node_modules/vega-projection": { - "version": "1.4.5", - "resolved": "https://registry.npmjs.org/vega-projection/-/vega-projection-1.4.5.tgz", - "integrity": "sha512-85kWcPv0zrrNfxescqHtSYpRknilrS0K3CVRZc7IYQxnLtL1oma9WEbrSr1LCmDoCP5hl2Z1kKbomPXkrQX5Ag==", - "dependencies": { - "d3-geo": "^2.0.1", - "d3-geo-projection": "^3.0.0" - } - }, - "node_modules/vega-regression": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/vega-regression/-/vega-regression-1.0.9.tgz", - "integrity": "sha512-KSr3QbCF0vJEAWFVY2MA9X786oiJncTTr3gqRMPoaLr/Yo3f7OPKXRoUcw36RiWa0WCOEMgTYtM28iK6ZuSgaA==", - "dependencies": { - "d3-array": "^2.7.1", - "vega-dataflow": "^5.7.3", - "vega-statistics": "^1.7.9", - "vega-util": "^1.15.2" - } - }, - "node_modules/vega-runtime": { - "version": "6.1.3", - "resolved": "https://registry.npmjs.org/vega-runtime/-/vega-runtime-6.1.3.tgz", - "integrity": "sha512-gE+sO2IfxMUpV0RkFeQVnHdmPy3K7LjHakISZgUGsDI/ZFs9y+HhBf8KTGSL5pcZPtQsZh3GBQ0UonqL1mp9PA==", - "dependencies": { - "vega-dataflow": "^5.7.3", - "vega-util": "^1.15.2" - } - }, - "node_modules/vega-scale": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/vega-scale/-/vega-scale-7.1.1.tgz", - "integrity": "sha512-yE0to0prA9E5PBJ/XP77TO0BMkzyUVyt7TH5PAwj+CZT7PMsMO6ozihelRhoIiVcP0Ae/ByCEQBUQkzN5zJ0ZA==", - "dependencies": { - "d3-array": "^2.7.1", - "d3-interpolate": "^2.0.1", - "d3-scale": "^3.2.2", - "vega-time": "^2.0.4", - "vega-util": "^1.15.2" - } - }, - "node_modules/vega-scenegraph": { - "version": "4.9.4", - "resolved": "https://registry.npmjs.org/vega-scenegraph/-/vega-scenegraph-4.9.4.tgz", - "integrity": "sha512-QaegQzbFE2yhYLNWAmHwAuguW3yTtQrmwvfxYT8tk0g+KKodrQ5WSmNrphWXhqwtsgVSvtdZkfp2IPeumcOQJg==", - "dependencies": { - "d3-path": "^2.0.0", - "d3-shape": "^2.0.0", - "vega-canvas": "^1.2.5", - "vega-loader": "^4.3.3", - "vega-scale": "^7.1.1", - "vega-util": "^1.15.2" - } - }, - "node_modules/vega-selections": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/vega-selections/-/vega-selections-5.3.1.tgz", - "integrity": "sha512-cm4Srw1WHjcLGXX7GpxiUlfESv8XPu5b6Vh3mqMDPU94P2FO91SR9gei+EtRdt+KCFgIjr//MnRUjg/hAWwjkQ==", - "dependencies": { - "vega-expression": "^5.0.0", - "vega-util": "^1.16.0" - } - }, - "node_modules/vega-statistics": { - "version": "1.7.10", - "resolved": "https://registry.npmjs.org/vega-statistics/-/vega-statistics-1.7.10.tgz", - "integrity": "sha512-QLb12gcfpDZ9K5h3TLGrlz4UXDH9wSPyg9LLfOJZacxvvJEPohacUQNrGEAVtFO9ccUCerRfH9cs25ZtHsOZrw==", - "dependencies": { - "d3-array": "^2.7.1" - } - }, - "node_modules/vega-time": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/vega-time/-/vega-time-2.0.4.tgz", - "integrity": "sha512-U314UDR9+ZlWrD3KBaeH+j/c2WSMdvcZq5yJfFT0yTg1jsBKAQBYFGvl+orackD8Zx3FveHOxx3XAObaQeDX+Q==", - "dependencies": { - "d3-array": "^2.7.1", - "d3-time": "^2.0.0", - "vega-util": "^1.15.2" - } - }, - "node_modules/vega-transforms": { - "version": "4.9.4", - "resolved": "https://registry.npmjs.org/vega-transforms/-/vega-transforms-4.9.4.tgz", - "integrity": "sha512-JGBhm5Bf6fiGTUSB5Qr5ckw/KU9FJcSV5xIe/y4IobM/i/KNwI1i1fP45LzP4F4yZc0DMTwJod2UvFHGk9plKA==", - "dependencies": { - "d3-array": "^2.7.1", - "vega-dataflow": "^5.7.4", - "vega-statistics": "^1.7.9", - "vega-time": "^2.0.4", - "vega-util": "^1.16.1" - } - }, - "node_modules/vega-typings": { - "version": "0.22.0", - "resolved": "https://registry.npmjs.org/vega-typings/-/vega-typings-0.22.0.tgz", - "integrity": "sha512-TgBGRkZHQgcduGsoFKq3Scpn6eNY4L3p0YKRhgCPVU3HEaCeYkPFGaR8ynK+XrKmvrqpDv0YHIOwCt7Gn3RpCA==", - "dependencies": { - "vega-event-selector": "^3.0.0", - "vega-expression": "^5.0.0", - "vega-util": "^1.15.2" - } - }, - "node_modules/vega-util": { - "version": "1.17.0", - "resolved": "https://registry.npmjs.org/vega-util/-/vega-util-1.17.0.tgz", - "integrity": "sha512-HTaydZd9De3yf+8jH66zL4dXJ1d1p5OIFyoBzFiOli4IJbwkL1jrefCKz6AHDm1kYBzDJ0X4bN+CzZSCTvNk1w==" - }, - "node_modules/vega-view": { - "version": "5.10.1", - "resolved": "https://registry.npmjs.org/vega-view/-/vega-view-5.10.1.tgz", - "integrity": "sha512-4xvQ5KZcgKdZx1Z7jjenCUumvlyr/j4XcHLRf9gyeFrFvvS596dVpL92V8twhV6O++DmS2+fj+rHagO8Di4nMg==", - "dependencies": { - "d3-array": "^2.7.1", - "d3-timer": "^2.0.0", - "vega-dataflow": "^5.7.3", - "vega-format": "^1.0.4", - "vega-functions": "^5.10.0", - "vega-runtime": "^6.1.3", - "vega-scenegraph": "^4.9.4", - "vega-util": "^1.16.1" - } - }, - "node_modules/vega-view-transforms": { - "version": "4.5.8", - "resolved": "https://registry.npmjs.org/vega-view-transforms/-/vega-view-transforms-4.5.8.tgz", - "integrity": "sha512-966m7zbzvItBL8rwmF2nKG14rBp7q+3sLCKWeMSUrxoG+M15Smg5gWEGgwTG3A/RwzrZ7rDX5M1sRaAngRH25g==", - "dependencies": { - "vega-dataflow": "^5.7.3", - "vega-scenegraph": "^4.9.2", - "vega-util": "^1.15.2" - } - }, - "node_modules/vega-voronoi": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/vega-voronoi/-/vega-voronoi-4.1.5.tgz", - "integrity": "sha512-950IkgCFLj0zG33EWLAm1hZcp+FMqWcNQliMYt+MJzOD5S4MSpZpZ7K4wp2M1Jktjw/CLKFL9n38JCI0i3UonA==", - "dependencies": { - "d3-delaunay": "^5.3.0", - "vega-dataflow": "^5.7.3", - "vega-util": "^1.15.2" - } - }, - "node_modules/vega-wordcloud": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/vega-wordcloud/-/vega-wordcloud-4.1.3.tgz", - "integrity": "sha512-is4zYn9FMAyp9T4SAcz2P/U/wqc0Lx3P5YtpWKCbOH02a05vHjUQrQ2TTPOuvmMfAEDCSKvbMSQIJMOE018lJA==", - "dependencies": { - "vega-canvas": "^1.2.5", - "vega-dataflow": "^5.7.3", - "vega-scale": "^7.1.1", - "vega-statistics": "^1.7.9", - "vega-util": "^1.15.2" - } - }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, - "node_modules/wide-align": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", - "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", - "dependencies": { - "string-width": "^1.0.2 || 2" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" - }, - "node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - } - }, "dependencies": { "@mapbox/node-pre-gyp": { "version": "1.0.5", @@ -2070,14 +724,6 @@ "resolved": "https://registry.npmjs.org/stable/-/stable-0.1.8.tgz", "integrity": "sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==" }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - }, "string-width": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", @@ -2088,6 +734,14 @@ "strip-ansi": "^3.0.0" } }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + }, "strip-ansi": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", diff --git a/vegafusion-wasm/Cargo.toml b/vegafusion-wasm/Cargo.toml index 6f143a9ba..205f3dedb 100644 --- a/vegafusion-wasm/Cargo.toml +++ b/vegafusion-wasm/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vegafusion-wasm" -version = "0.0.1" +version = "0.0.2" edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -22,6 +22,7 @@ getrandom = { version = "0.2.3", features = ["js"] } prost = "0.8.0" prost-types = "0.8.0" indexmap = "=1.6.2" +chrono = {version="0.4.19", features=["wasmbind"]} # The `console_error_panic_hook` crate provides better debugging of panics by # logging them with `console.error`. This is great for development, but requires diff --git a/vegafusion-wasm/package.json b/vegafusion-wasm/package.json index cf71d5726..7a2481639 100644 --- a/vegafusion-wasm/package.json +++ b/vegafusion-wasm/package.json @@ -1,6 +1,6 @@ { "name": "vegafusion-wasm", - "version": "0.0.1", + "version": "0.0.2", "author": { "name": "Jon Mease", "email": "jonmmease@gmail.com", diff --git a/vegafusion-wasm/src/lib.rs b/vegafusion-wasm/src/lib.rs index 771a1eb97..089fe557f 100644 --- a/vegafusion-wasm/src/lib.rs +++ b/vegafusion-wasm/src/lib.rs @@ -163,6 +163,7 @@ impl MsgReceiver { if self.verbose { log(&format!("VegaFusion(wasm): Received {}", var.name)); log(&serde_json::to_string_pretty(&json).unwrap()); + log(&format!("DataType: {:#?}", &value.get_datatype())); } let js_value = JsValue::from_serde(&json).unwrap(); @@ -173,6 +174,7 @@ impl MsgReceiver { if self.verbose { log(&format!("VegaFusion(wasm): Received {}", var.name)); log(&serde_json::to_string_pretty(&json).unwrap()); + log(&format!("Schema: {:#?}", &value.schema)); } let js_value = JsValue::from_serde(&value.to_json()).unwrap();