From 43380555ae41fb1201a3da96bf3af61e03f476ae Mon Sep 17 00:00:00 2001 From: Jon Ludlam Date: Thu, 11 Apr 2019 18:24:23 +0100 Subject: [PATCH] First cut of jupyter output Signed-off-by: Jon Ludlam --- bin/dune | 14 ++++- bin/jupyter.ml | 129 +++++++++++++++++++++++++++++++++++++++++++++++ bin/main.ml | 2 +- bin/notebook.atd | 57 +++++++++++++++++++++ dune-project | 1 + mdx.opam | 1 + 6 files changed, 202 insertions(+), 2 deletions(-) create mode 100644 bin/jupyter.ml create mode 100644 bin/notebook.atd diff --git a/bin/dune b/bin/dune index f3ce6e0c1..2f9c1f89b 100644 --- a/bin/dune +++ b/bin/dune @@ -1,7 +1,7 @@ (library (name cli) (modules cli) - (libraries cmdliner fmt.cli logs.fmt fmt.tty logs.cli mdx)) + (libraries cmdliner fmt.cli logs.fmt fmt.tty logs.cli mdx yojson atdgen)) (executable (name main) @@ -9,3 +9,15 @@ (package mdx) (modules :standard \ cli) (libraries cli mdx)) + +(rule + (targets notebook_j.ml notebook_j.mli) + (deps notebook.atd) + (action + (run atdgen -j -j-std %{deps}))) + +(rule + (targets notebook_t.ml notebook_t.mli) + (deps notebook.atd) + (action + (run atdgen -t %{deps}))) diff --git a/bin/jupyter.ml b/bin/jupyter.ml new file mode 100644 index 000000000..4a89dc6a9 --- /dev/null +++ b/bin/jupyter.ml @@ -0,0 +1,129 @@ +open Mdx.Util.Result.Infix +open Cmdliner + +let raw t = + Notebook_t. + { + cell_type = `Raw; + metadata = { collapsed = None; scrolled = None }; + source = String.concat "\n" t; + outputs = None; + execution_count = None; + } + +let txt source = + Notebook_t. + { + cell_type = `Markdown; + metadata = { collapsed = None; scrolled = None }; + source; + outputs = None; + execution_count = None; + } + +let execution_count = ref 1 + +let ocaml contents = + let cell = + Notebook_t. + { + cell_type = `Code; + metadata = { collapsed = None; scrolled = None }; + source = String.concat "\n" contents; + outputs = Some []; + execution_count = Some !execution_count; + } + in + incr execution_count; + cell + +let toplevel x = + let cell = + Notebook_t. + { + cell_type = `Code; + metadata = { collapsed = None; scrolled = None }; + source = String.concat "\n" x.Mdx.Toplevel.command; + outputs = Some []; + execution_count = Some !execution_count; + } + in + incr execution_count; + cell + +let metadata = + Notebook_t. + { + kernelspec = + { + display_name = "OCaml 4.07.1"; + language = "OCaml"; + name = "ocaml-jupyter"; + }; + language_info = + { + name = "OCaml"; + version = "4.07.1"; + codemirror_mode = Some "text/x-ocaml"; + file_extension = ".ml"; + mimetype = "text/x-ocaml"; + nbconverter_exporter = None; + pygments_lexer = "OCaml"; + }; + } + +let rec collapse_text = function + | Mdx.Text x :: Mdx.Text y :: xs -> + collapse_text (Mdx.Text (x ^ "\n" ^ y) :: xs) + | (Mdx.Section _ as s) :: Mdx.Text y :: xs -> + let s = Mdx.to_string [ s ] in + collapse_text (Mdx.Text (s ^ "\n" ^ y) :: xs) + | (Mdx.Section _ as s) :: xs -> + let s = Mdx.to_string [ s ] in + collapse_text (Mdx.Text s :: xs) + | x :: ys -> x :: collapse_text ys + | [] -> [] + +let run _setup (`Syntax syntax) (`File file) = + let cells = ref [] in + Mdx.run ?syntax file ~f:(fun _file_contents items -> + let syntax = + match syntax with + | Some s -> s + | None -> ( + match Mdx.Syntax.infer ~file with + | Some s -> s + | None -> failwith "Couldn't get syntax") + in + List.iter + (function + | Mdx.Text "" -> () + | Mdx.Text x -> cells := txt x :: !cells + | Mdx.Block { value = OCaml { env = User_defined _; _ }; _ } + | Mdx.Block { value = Toplevel { env = User_defined _; _ }; _ } -> + failwith "internal error, cannot handle user defined environments" + | Mdx.Block { value = OCaml _; contents; _ } -> + cells := ocaml contents :: !cells + | Mdx.Block { value = Toplevel _; contents; loc; _ } -> + let blocks = Mdx.Toplevel.of_lines ~syntax ~loc contents in + let newcells = List.rev_map toplevel blocks in + cells := newcells @ !cells + | Mdx.Block { value = Raw _; contents; _ } -> + cells := raw contents :: !cells + | x -> + failwith + (Printf.sprintf "internal error, cannot handle: %s" + (Mdx.to_string [ x ]))) + (collapse_text items); + "OK") + >>! fun () -> + let notebook = + Notebook_t. + { metadata; nbformat = 4; nbformat_minor = 2; cells = List.rev !cells } + in + Printf.fprintf stdout "%s" (Notebook_j.string_of_notebook notebook); + 0 + +let cmd : int Term.t * Term.info = + let doc = "Convert an mdx file to a jupyter notebook." in + (Term.(pure run $ Cli.setup $ Cli.syntax $ Cli.file), Term.info "jupyter" ~doc) diff --git a/bin/main.ml b/bin/main.ml index eeb07f0a9..229535fef 100644 --- a/bin/main.ml +++ b/bin/main.ml @@ -16,7 +16,7 @@ open Cmdliner -let cmds = [ Test.cmd; Pp.cmd; Deps.cmd; Dune_gen.cmd ] +let cmds = [ Test.cmd; Pp.cmd; Deps.cmd; Dune_gen.cmd; Jupyter.cmd ] let main (`Setup ()) = `Help (`Pager, None) diff --git a/bin/notebook.atd b/bin/notebook.atd new file mode 100644 index 000000000..7e332c5c3 --- /dev/null +++ b/bin/notebook.atd @@ -0,0 +1,57 @@ +type kernelspec = { + display_name : string; + language: string; + name: string; +} + +type language_info = { + name: string; + version: string; + codemirror_mode: string nullable; + file_extension: string; + mimetype: string; + nbconverter_exporter: string nullable; + pygments_lexer: string; +} + +type metadata = { + kernelspec: kernelspec; + language_info: language_info; +} + +type cell_metadata = { + ?collapsed: bool nullable; + ?scrolled: bool nullable; +} + +type cell_type = [ + | Code + | Markdown + | Raw +] + +type output_type = [ + Stream + | Display_data + | Execute_result + | Error +] + +type output = { + output_type : output_type; +} + +type cell = { + cell_type : cell_type; + metadata: cell_metadata; + source: string; + ?outputs: output list nullable; + ?execution_count: int option; +} + +type notebook = { + metadata: metadata; + nbformat: int; + nbformat_minor: int; + cells: cell list +} diff --git a/dune-project b/dune-project index 141726e4d..e04415a6a 100644 --- a/dune-project +++ b/dune-project @@ -27,6 +27,7 @@ (csexp (>= 1.3.2)) astring + atdgen (logs (>= 0.7.0)) (cmdliner (>= 1.0.0)) diff --git a/mdx.opam b/mdx.opam index 8e826b946..948491a5a 100644 --- a/mdx.opam +++ b/mdx.opam @@ -25,6 +25,7 @@ depends: [ "cppo" {build} "csexp" {>= "1.3.2"} "astring" + "atdgen" "logs" {>= "0.7.0"} "cmdliner" {>= "1.0.0"} "re" {>= "1.7.2"}