speed up load of namespaces by caching and skiping

viebel · Oct 15, 2016 · 26408d0 · 26408d0
2 parents 05cb539 + 956d4e1
commit 26408d0
Show file tree

Hide file tree

Showing 9 changed files with 307 additions and 44 deletions.
diff --git a/.gitignore b/.gitignore
@@ -3,7 +3,6 @@ figwheel_server.log
 klipse-access-logs/
 resources/private/
 resources/public/cards
-resources/public/dbg
 resources/public/fig
 resources/public/dev
 resources/public/plugin
@@ -12,3 +11,4 @@ target/
 .lein-repl-history
 *-init.clj
 *.swp
+.cljs_node_repl/
diff --git a/cache-src/core.cljs b/cache-src/core.cljs
@@ -0,0 +1,6 @@
+(ns core
+  (:require
+    cljs.js
+    [clojure.set :refer [union]]))
+
+(defn foo [] (union #{1 2} #{2 4}))
diff --git a/images/gadjett.gif b/images/gadjett.gif
diff --git a/project.clj b/project.clj
@@ -1,11 +1,12 @@
 (defproject klipse "3.1.7"
   :description "Cljs compiler in cljs"
   :dependencies [[org.clojure/clojure "1.8.0"]
-                 [org.clojure/clojurescript "1.9.225"]
+                 [org.clojure/clojurescript "1.9.229"]
                  [org.clojure/core.async "0.2.385"]
                  [binaryage/devtools "0.8.2"]
                  [im.chit/purnam "0.5.2"]
                  [cljsjs/markdown "0.6.0-beta1-0"]
+                 [org.clojure/core.match "0.3.0-alpha4"]
                  [cljs-http "0.1.41"]
                  [viebel/gadjett "0.2.1"]
                  [org.omcljs/om "1.0.0-alpha41"]
@@ -23,7 +24,7 @@
                                     "resources/public/plugin/js"]
   :plugins [[lein-cljsbuild "1.1.1"]
             [lein-doo "0.1.6"]]
-  :source-paths ["src"]
+  :source-paths ["src" "scripts"]
   :cljsbuild {
               :builds {
                        :test {
@@ -67,6 +68,15 @@
                                           :elide-asserts false
                                           :optimizations :advanced
                                           :verbose true}}
+                       :cache {
+                               :source-paths ["cache-src"]
+                               :compiler {;:main "klipse.run.all"
+                                          :asset-path "cache/js"
+                                          :output-to "resources/public/cache/js/klipse.cache.js"
+                                          :output-dir "resources/public/cache/js"
+                                          :verbose true
+                                          :optimizations :none
+                                          }}
                        :figwheel {
                                   :figwheel true
                                   :source-paths ["src"]
@@ -77,7 +87,6 @@
                                              :output-dir "resources/public/fig/js"
                                              ;:elide-asserts true
                                              :verbose false}}
-
                        :devcards {
                                   :figwheel { :devcards true }
                                   :source-paths ["src"]
@@ -87,6 +96,4 @@
                                              :output-dir "resources/public/cards/js"
                                              ;:elide-asserts true
                                              :verbose false}}
-
-
                        }})
diff --git a/resources/public/playground-dbg.html b/resources/public/playground-dbg.html
@@ -7,31 +7,59 @@
 <link rel="stylesheet" type="text/css" href="css/codemirror.css">
 </head>
 <body>
-<div class="clojure" data-gist-id="viebel/21fd0e02d1a1d85b2e2089568f16828c">
-            </div>
 <pre><code class="clojure">
-    "Adasdas"
-</code></pre>
+    (map inc [1 2 3])
+    </code></pre>
 
-<pre><code class="clojure">
-    (str "Adasdas" ": \" dasdas\"")
+    <pre><code class="clojure" data-external-libs="https://raw.githubusercontent.com/viebel/math.combinatorics/master/src/main/clojure">
+    (ns my.combinatorics
+      (:require [clojure.math.combinatorics :refer [permutations]]))
+
+    (permutations [1 2 3])
+    </code></pre>
+
+
+<pre><code class="clojure" data-gist-id="viebel/820f672dd176169fd831c7748e6f3cd3" data-external-libs="https://raw.githubusercontent.com/mfikes/andare/master/src/main/clojure/">
 </code></pre>
+<pre><code class="clojure" data-external-libs="https://raw.githubusercontent.com/clojure/test.check/master/src/main/clojure">
+    (ns my.test   (:require [clojure.test.check :as tc]
+                [clojure.test.check.generators :as gen]
+                            [clojure.test.check.properties :as prop :include-macros true]))
+    (def sort-idempotent-prop
+      (prop/for-all [v (gen/vector gen/int)]
+            (= (sort v) (sort (sort v)))))
+
+            (tc/quick-check 100 sort-idempotent-prop)
 
-<pre><code class="clojure">
-    ["Adasdas"]
 </code></pre>
 <pre><code class="clojure">
-    #(+ 1 2)
-</code></pre>
+    (ns my.spec
+    (:require [clojure.spec :as s]))
+
+    (s/def ::a integer?)
+    (s/conform ::a 12)
+    </code></pre>
 <pre><code class="clojure">
-    (+ 1 2)
-</code></pre>
+    (ns my.cljs
+      (:require [cljs.js :as cljs :refer [eval-str]]))
+
+      (eval-str (cljs.js/empty-state) "(+ 1 2)" 'cljs.me {:eval cljs.js/js-eval} identity)
+
+
+    </code></pre>
+
 <pre><code class="clojure">
-    (map inc [1 2 3])
-</code></pre>
-<pre><code class="clojure" data-beautify-strings="true">
-    (js/Date)
-</code></pre>
+(ns my.walk
+  (:require [clojure.walk :refer [postwalk]]))
+  (postwalk identity {:a 1})
+    </code></pre>
+
+<pre><code class="clojure">
+(ns my.ns
+  (:require [clojure.string :as string :refer [lower-case]]))
+  (lower-case "AbsCs")
+    </code></pre>
+
 <script>
     window.klipse_settings = {
     codemirror_options_in: {

diff --git a/scripts/cache/build.clj b/scripts/cache/build.clj
@@ -0,0 +1,73 @@
+(ns cache.build
+  (:require [clojure.java.io :as io]
+            [cljs.build.api :as api]
+            [cljs.analyzer :as ana]
+            [cljs.env :as env]
+            [cognitect.transit :as transit])
+  (:import [java.io ByteArrayOutputStream FileInputStream]))
+
+(defmacro my-dump-core []  (let [state @env/*compiler*]
+                             (if-not (false? (get-in state [:options :dump-core]))
+                               `(quote ~(get-in state [::ana/namespaces 'cljs.core]))
+                               `(hash-map))))
+(defn write-cache [cache out-path]
+  (let [out (ByteArrayOutputStream. 1000000)
+        writer (transit/writer out :json)]
+    (transit/write writer cache)
+    (spit (io/file out-path) (.toString out))))
+
+(defn build []
+  (with-redefs [env/*compiler* (atom {})]
+  (cljs.analyzer/with-warning-handlers
+    [(fn [warning-type env extra]
+       (print (class env))
+       (when (warning-type cljs.analyzer/*cljs-warnings*)
+         (when-let [s (cljs.analyzer/error-message warning-type extra)]
+           (binding [*out* *err*]
+             (println "WARNING:" (cljs.analyzer/message env s)))
+           #_(System/exit 1))))]
+    (api/build (api/inputs "cache-src")
+               {:output-dir         "out"
+               :output-to          "out/main.js"
+               :optimizations      :none
+               :static-fns         true
+               :optimize-constants false
+               :dump-core          false
+               :parallel-build     true})
+    env/*compiler*)))
+
+(defn copy-source
+  [filename]
+  (spit (str "out/" filename)
+    (slurp (io/resource filename))))
+
+(defn copy-sources []
+  (copy-source "cljs/test.cljc")
+  (copy-source "cljs/spec.cljc")
+  (copy-source "cljs/spec/test.cljc")
+  (copy-source "cljs/spec/test.cljs")
+  (copy-source "cljs/spec/impl/gen.cljc")
+  (copy-source "cljs/analyzer/api.cljc")
+  (copy-source "clojure/template.clj"))
+
+(defn write-caches []
+  (let [res (io/resource "cljs/core.cljs.cache.aot.edn")
+            cache (read-string (slurp res))]
+    (doseq [key (keys cache)]
+      (write-cache (key cache) (str "out/cljs/core.cljs.cache.aot." (munge key) ".json")))))
+
+(defn write-caches-macros []
+(let [res "out/cljs/core$macros.cljc.cache.json"
+      cache (transit/read (transit/reader (FileInputStream. res) :json))]
+  (doseq [key (keys cache)]
+    (write-cache (key cache) (str "out/cljs/core$macros.cljc.cache." (munge key) ".json")))))
+
+(defn do-the-job []
+  (println "Building")
+  (build)
+  (copy-sources)
+  (write-caches)
+  (write-caches-macros)
+  (println "Done building")
+  (System/exit 0))
+
diff --git a/src/klipse/compiler.cljs b/src/klipse/compiler.cljs
@@ -1,5 +1,6 @@
 (ns klipse.compiler
   (:require-macros
+    [cljs.core.match :refer [match]]
     [gadjett.core :as gadjett :refer [deftrack dbg]]
     [cljs.core.async.macros :refer [go go-loop]])
   (:require 
@@ -9,6 +10,7 @@
     [klipse.plugin :refer [register-mode]]
     [klipse.io :as io]
     [clojure.string :as s]
+    [cljs.analyzer.api :as api]
     [cljs.core.async :refer [chan put! <!]]
     [replumb.core :as replumb]
     [cljs.js :as cljs]))
@@ -26,17 +28,6 @@
 (defn load-inlined [opts cb]
   (cb {:lang :js :source ""}))
 
-(def known-src-paths 
-  {"goog-closure" "https://raw.githubusercontent.com/google/closure-library/v20160713/closure/"
-   "gist" "https://gist.githubusercontent.com"
-  "clojurescript" ["https://raw.githubusercontent.com/clojure/clojurescript/r1.9.225/src/main/clojure" "https://raw.githubusercontent.com/clojure/clojurescript/r1.9.225/src/main/cljs"]
-   })
-
-(defn repos [additional-libs]
-  (-> (vals known-src-paths)
-      flatten
-      (concat additional-libs)))
-
 (defn special-fetch [file-url src-cb]
   (-> (s/replace file-url #"gist_" "")
       (io/fetch-file! src-cb)))
@@ -51,9 +42,9 @@
 
 (defn result-as-str [{:keys [form warning error value success?]} opts]
   (let [status (if error :error :ok)
-        res (if success?
+        res (if-not error
                 (display value opts)
-              (pr-str error))]
+             (pr-str error))]
     [status res]))
 
 (defn read-result [{:keys [form warning error value success?]}]
@@ -99,7 +90,7 @@
     c))
 
 (defn build-repl-opts [{:keys [static-fns context external-libs]}]
-  (merge (replumb/options :browser (repos external-libs) special-fetch)
+  (merge (replumb/options :browser #_(repos external-libs) (partial io/load-ns external-libs))
          {:warning-as-error false
           :static-fns static-fns
           :no-pr-str-on-value true
@@ -113,7 +104,16 @@
     (set! js/COMPILED true)
     (replumb/read-eval-call opts cb s)))
 
-(deftrack eval-async-1 [s opts]
+(def create-state (memoize cljs/empty-state))
+
+(defn core-eval-clj [s {:keys [static-fns context external-libs] :or {static-fns false context nil external-libs '()}} cb]
+  (let [st (create-state)]
+    (cljs/eval-str st s  "my.klipse" {:eval cljs/js-eval
+                                      :context (keyword context)
+                                      :static-fns static-fns
+                                      :load (partial io/load-ns external-libs)} cb)))
+
+(deftrack eval-async-1 [s {:keys [print-length] :as opts}]
   (let [c (chan)]
     (core-eval s opts #(put! c (result-as-str % opts)))
     c))

diff --git a/src/klipse/core.cljs b/src/klipse/core.cljs
@@ -1,5 +1,5 @@
 (ns ^:figwheel-no-load klipse.core
-  (:require 
+  (:require
     [gadjett.core-fn :as gadjett]))
 
 (enable-console-print!)