change install from file based to xar install (#2)

version 1.0.2 -> 1.1.0

Co-authored-by: dominik kuehne <[email protected]>

README.md

@@ -22,7 +22,7 @@ Synchronizes your data collection with GitHub and GitLab.
 
 ## Current restrictions
 
-In version 1.0.2 not implemented:
+In version 1.1.0 not implemented:
 -  webhooks are not fully implemented.
 
 ## Building and Installation
@@ -96,15 +96,18 @@ tba
 ## Dashboard
 
 The dashboard can trigger a full deployment or an incremental update. 
-Full deployment clones the repository from git and overwrites the database collection.
+Full deployment clones the repository from git and install it as a xar.
 With incremental update only the changes to the database collection are applied.
 
 ### Lets start
 
-1) install xar of the data collection
+1) customize the configuration (modules/config.xql)
 2) click on 'full' to trigger a full deployment from git to existdb
 3) now you can update your collection with a click on 'incremental'
 
+**REMARK: A valid expath-pkg.xml and repo.xml must be present**
+**REMARK: Only use it with data collections**
+
 ## API
 
 The page below is reachable via [api.html](api.html) in your installed tuttle app. 

gulpfile.js

@@ -54,7 +54,7 @@ function watchTemplates () {
 exports["watch:tmpl"] = watchTemplates
 
 
-const static = 'src/**/*.{xml,html,xq,xquery,xql,xqm,xsl,xconf,json,svg,js,css,png,jpg}'
+const static = 'src/**/*.{xml,html,xq,xquery,xql,xqm,xsl,xconf,json,svg,js,css,png,jpg,map}'
 
 /**
  * copy html templates, XSL stylesheet, XMLs and XQueries to 'build'

package.json

@@ -1,6 +1,6 @@
 {
   "name": "tuttle",
-  "version": "1.0.2",
+  "version": "1.1.0",
   "description": "tuttle - a Git-integration for eXist-db",
   "scripts": {
     "test": "gulp install && mocha --exit",

src/api.json

@@ -1,7 +1,7 @@
 {
     "openapi": "3.0.0",
     "info": {
-        "version": "1.0.2",
+        "version": "1.1.0",
         "title": "Tuttle API",
         "description": "API for Tuttle Git integration"
     },
@@ -75,15 +75,6 @@
                 },
                 "operationId": "api:git-deploy",
                 "parameters": [
-                    {
-                        "name": "pre-install",
-                        "description": "Run pre install function",
-                        "in": "query",
-                        "schema":{
-                            "type": "boolean",
-                            "default": false
-                        }
-                    },
                     {
                         "name": "collection",
                         "in":  "path",
@@ -152,18 +143,7 @@
                 "x-constraints": {
                     "user": "admin"
                 },
-                "operationId": "api:git-deploy",
-                "parameters": [
-                    {
-                        "name": "pre-install",
-                        "description": "Run pre install function",
-                        "in": "query",
-                        "schema":{
-                            "type": "boolean",
-                            "default": false
-                        }
-                    }
-                ],
+                "operationId": "api:git-deploy",                
                 "responses": {
                     "200":{
                         "description": "JSON dump of request",
@@ -735,4 +715,4 @@
             "basicAuth": []
         }
     ]
-}
+}

src/controller.xql

@@ -63,6 +63,10 @@ else if (matches($exist:resource, "\.(js)$", "s")) then
     <dispatch xmlns="http://exist.sourceforge.net/NS/exist">
     </dispatch>
 
+else if (matches($exist:resource, "\.(js.map)$", "s")) then
+    <dispatch xmlns="http://exist.sourceforge.net/NS/exist">
+    </dispatch>
+
 else if (matches($exist:resource, "\.(png)$", "s")) then
     <dispatch xmlns="http://exist.sourceforge.net/NS/exist">
     </dispatch>

src/expath-pkg.xml

@@ -2,7 +2,7 @@
 <package 
   xmlns="http://expath.org/ns/pkg"
   name="http://e-editiones.org/tuttle"
-  abbrev="tuttle" version="1.0.2" spec="1.0">
+  abbrev="tuttle" version="1.1.0" spec="1.0">
   <title>Tuttle - Git for eXist-db</title>
   <dependency processor="http://exist-db.org" semver-min="5.3.0"/>
   <dependency package="http://e-editiones.org/roaster" semver-min="1.0.0"/>

src/index.html

@@ -108,7 +108,8 @@ <h1><img class="tuttle" src="resources/images/HPTuttle-1866.png"></img>
                            serialization="none">
 
                 <fx-action event="submit-done">
-                    <fx-message>Data deployed to database</fx-message>
+                    <fx-message if="instance('response')?message = 'success'">Data deployed to database</fx-message>
+                    <fx-message if="instance('response')?message = 'no expath-pkg.xml or repo.xml in repo'">No expath-pkg.xml or repo.xml in repo</fx-message>
                     <fx-send submission="config"></fx-send>
                 </fx-action>
 
@@ -203,7 +204,7 @@ <h3>Git Repositories</h3>
             </fx-case>
         </fx-switch>
     </fx-fore>
-    <script type="module" src="./js/fore-debug.js"></script>
+    <script type="module" src="./js/fore-all.js"></script>
 
 
 </div>

src/modules/api.xql

@@ -7,6 +7,7 @@ import module namespace roaster="http://e-editiones.org/roaster";
 import module namespace rutil="http://e-editiones.org/roaster/util";
 import module namespace errors="http://e-editiones.org/roaster/errors";
 import module namespace xmldb="http://exist-db.org/xquery/xmldb";
+import module namespace compression="http://exist-db.org/xquery/compression";
 
 import module namespace github="http://exist-db.org/apps/tuttle/github" at "github.xql";
 import module namespace gitlab="http://exist-db.org/apps/tuttle/gitlab" at "gitlab.xql";
@@ -33,7 +34,7 @@ declare function api:get-status($request as map(*)) {
                 let $hash-deploy := $config:prefix || "/" || $collection || "/gitsha.xml"
                 let $hash-git := if($col-config?vcs = "github") then github:get-lastcommit-sha($col-config)
                                     else gitlab:get-lastcommit-sha($col-config)
-                let $status := if ($hash-git?sha = "" or not(xmldb:collection-available($collection-path)) ) then
+                let $status := if ($hash-git?sha = "" ) then
                                     "error"
                                 else if (doc($hash-deploy)/hash/value/text() = $hash-git?sha) then
                                     "uptodate"
@@ -45,11 +46,8 @@ declare function api:get-status($request as map(*)) {
                                 github:get-url($col-config) 
                             else 
                                 gitlab:get-url($col-config)
-                let $message := if($status = "error" ) then 
-                                    if (not(xmldb:collection-available($collection-path))) then
-                                        "Destination collection not exist"
-                                    else 
-                                         $url?message
+                let $message := if($status = "error" ) then
+                                    concat($col-config?vcs, " error: ", $url?message)
                                 else
                                     ""
 
@@ -196,32 +194,43 @@ declare function api:git-deploy($request as map(*)) {
     return
         if (exists($config))  then (
             try {
-                if (not(xmldb:collection-available($collection-destination))) then (
-                    map { "message" : "Destination collection '" || $collection-destination || "' does not exist" }
-                )
-                else if (not(xmldb:collection-available($collection-staging-uri))) then (
+                if (not(xmldb:collection-available($collection-staging-uri))) then (
                     map { "message" : "Staging collection '" || $collection-staging || "' does not exist" }
                 )
                 else if (exists(doc($lockfile))) then (
                     map { "message" : doc($lockfile)/task/value/text() || " in progress" }
                 )
                 else (
-                            let $write-lock := app:lock-write($collection-destination, "deploy")
-                            let $pre-install := if ($request?parameters?pre-install) then
-                                                    config:pre-install($collection-destination,$collection-staging-uri)
-                                                else false()
-                            let $cleanup-col := app:cleanup-collection($git-collection, $config:prefix)
-                            let $cleanup-res := app:cleanup-resources($git-collection, $config:prefix)
-                            let $move-col := app:move-collections($collection-staging, $git-collection, $config:prefix)
-                            let $move-res := app:move-resources($collection-staging, $git-collection, $config:prefix)
-                            let $remove-staging := xmldb:remove($collection-staging-uri)
-                            let $set-permissions := app:set-permission($git-collection)
-                            let $remove-lock := app:lock-remove($collection-destination)
+                    let $check-lock-dst := if (xmldb:collection-available($collection-destination)) then ()
+                    else (
+                        xmldb:create-collection($config:prefix, $git-collection)
+                    ) 
+                    let $write-lock := app:lock-write($collection-destination, "deploy")
+                    let $xar-list := xmldb:get-child-resources($collection-staging-uri)
+                    let $xar-check := if (not($xar-list = "expath-pkg.xml" and $xar-list = "repo.xml")) then (
+                            map {
+                                        "message" : "no expath-pkg.xml or repo.xml in repo"
+                                }
+                        )
+                        else (
+                            let $remove-pkg := if (contains(repo:list(), $git-collection)) then (
+                                let $package := doc(concat($collection-destination, "/expath-pkg.xml"))//@name/string()
+                                let $undeploy := repo:undeploy($package)
+                                return repo:remove($package) 
+                            )
+                            else ()
+                            let $xar := xmldb:store-as-binary($collection-staging-uri, "pkg.xar", 
+                                compression:zip(xs:anyURI($collection-staging-uri),true(), $collection-staging-uri))
+                            let $install := repo:install-and-deploy-from-db(concat($collection-staging-uri, "/pkg.xar"))
                             return 
                                 map {
                                     "sha" : app:production-sha($git-collection),
                                     "message" : "success"
-                    }
+                            }                            
+                        )
+                    let $remove-staging := xmldb:remove($collection-staging-uri)
+                    let $remove-lock := app:lock-remove($collection-destination)
+                    return $xar-check
                 )
             }
             catch * {

src/modules/app.xql

@@ -56,63 +56,6 @@ declare function app:unzip-filter($path as xs:string, $data-type as xs:string, $
             true()
 };
 
-(:~
- : Move staging collection to final collection
- :)
-declare function app:move-collections($collection-source as xs:string, $collection-target as xs:string, $prefix as xs:string) {
-    let $fullpath-collection-source :=  $prefix || "/" || $collection-source
-    let $fullpath-collection-target :=  $prefix || "/" || $collection-target
-
-    return
-        for $child in xmldb:get-child-collections($fullpath-collection-source) 
-            let $fullpath-child-source := $fullpath-collection-source || "/" || $child
-            let $fullpath-child-target := $fullpath-collection-target || "/" 
-            return 
-                xmldb:move($fullpath-child-source, $fullpath-child-target)
-};
-
-(:~
- : Move staging collection to final collection
- :)
-declare function app:move-resources($collection-source as xs:string, $collection-target as xs:string, $prefix as xs:string) {
-    let $fullpath-collection-source :=  $prefix || "/" || $collection-source
-    let $fullpath-collection-target :=  $prefix || "/" || $collection-target
-
-    return
-        for $child in xmldb:get-child-resources($fullpath-collection-source) 
-            return 
-                xmldb:move($fullpath-collection-source, $fullpath-collection-target, $child)
-};
-
-(:~ 
- : Cleanup destination collection - delete collections from target collection
- :)
-declare function app:cleanup-collection($collection as xs:string, $prefix as xs:string) {
-    let $blacklist := [$config:blacklist, $config:lock]
-    let $fullpath-collection :=  $prefix || "/" || $collection 
-
-    return
-        for $child in xmldb:get-child-collections($fullpath-collection)
-            where not(contains($child, $blacklist))
-                let $fullpath-child := $fullpath-collection || "/" || $child
-                return 
-                    xmldb:remove($fullpath-child)
-};
-
-(:~
- : Cleanup destination collection - delete resources from target collection
- :)
-declare function app:cleanup-resources($collection as xs:string, $prefix as xs:string) {
-    let $blacklist := $config:blacklist
-    let $fullpath-collection :=  $prefix || "/" || $collection 
-
-    return
-        for $child in xmldb:get-child-resources($fullpath-collection)
-            where not(contains($child, $blacklist))
-            return 
-                xmldb:remove($fullpath-collection, $child)
-};
-
 (:~
  : Random apikey generator
  :)

src/modules/config.xql

@@ -28,6 +28,7 @@ declare variable $config:collections := map {
             }
         };
 
+
 (:~
  : Defile default collection
  :)
@@ -36,7 +37,7 @@ declare variable $config:default-collection := "sample-collection-github";
 (:~
  : Blacklist - these files are not checkout from git and are ignored
  :)
-declare variable $config:blacklist := ["build.xml", "expath-pkg.xml", "repo.xml", "controller.xql", ".gitignore", "collection.xconf"];
+declare variable $config:blacklist := [".existdb.json", "build.xml", "README.md", ".gitignore", "expath-pkg.xml.tmpl", "repo.xml.tmpl", "build.properties.xml"];
 
 (:~
  : Suffix of the checked out git statging collection 
@@ -67,9 +68,3 @@ declare variable $config:sm := map {
     "mode" : "rw-r--r--"
 };
 
-(:~ 
- : Custom Pre-install function - parameter destination-collection and stating-collection as full URI
-:)
-declare function config:pre-install ($collection as xs:string, $staging as xs:string) {
-    true()
-};

src/modules/github.xql

@@ -38,7 +38,7 @@ declare function github:clone($config as map(*), $collection as xs:string, $sha
 
                     let $clone := compression:unzip ($request[2], $filter, $filter-params,  $unzip-action, $data-params)
                     return  map {
-                            "message" : "Success"
+                            "message" : "success"
                     }
                 )
         }

src/modules/gitlab.xql

@@ -22,23 +22,22 @@ declare function gitlab:clone($config as map(*), $collection as xs:string, $sha
                     "message" : concat($config?vcs, " error: ", gitlab:request($url, $config?token)[1]/xs:string(@message))
                     } )
             else (
-                    let $request := gitlab:request($url, $config?token)
-                    let $filter := app:unzip-filter#3
-                    let $unzip-action := app:unzip-store#4
-                    let $filter-params := () 
-                    let $data-params := ($collection)         
-                    let $delete-collection :=
-                        if(xmldb:collection-available($collection)) then 
-                            xmldb:remove($collection)
-                        else ()
-                    let $create-collection := xmldb:create-collection("/", $collection)
-                    let $write-sha := app:write-sha($collection, gitlab:get-lastcommit-sha($config)?sha)
-                    let $clone := compression:unzip ($request[2], $filter, $filter-params,  $unzip-action, $data-params)
-                    return  map {
-                            "message" : "Success"
-                    }
-                )
-            )
+                let $request := gitlab:request($url, $config?token)
+                let $filter := app:unzip-filter#3
+                let $unzip-action := app:unzip-store#4
+                let $filter-params := () 
+                let $data-params := ($collection)         
+                let $delete-collection :=
+                    if(xmldb:collection-available($collection)) then 
+                        xmldb:remove($collection)
+                    else ()
+                let $create-collection := xmldb:create-collection("/", $collection)
+                let $write-sha := app:write-sha($collection, gitlab:get-lastcommit-sha($config)?sha)
+                let $clone := compression:unzip ($request[2], $filter, $filter-params,  $unzip-action, $data-params)
+                return  map {
+                        "message" : "success"
+                }
+            )   
         }
         catch * {
             map {