fix(api): several issues with deployment

- replace dbutil:scan (shared-resources) with collection:scan - set permissions to resources as well - resources that belong to `nobody/nogroup` are invisible to `admin/dba` the example permissions are therefore set to `admin/dba` and mode is `rw-r----` which is quite restrictive but safe. The comment in the example configuration changed to reflect that. To test that permissions are applied `guest/guest` or `SYSTEM/dba` are options.
eeditiones · Oct 5, 2023 · e1d2713 · e1d2713
1 parent 9d439dc
commit e1d2713
Show file tree

Hide file tree

Showing 5 changed files with 135 additions and 177 deletions.
diff --git a/src/data/tuttle-example-config.xml b/src/data/tuttle-example-config.xml
@@ -64,7 +64,10 @@
 
   <!-- prefix, suffix, lock and apikeys can usually be left as-is -->
   <config prefix="/db/apps/" suffix="-stage" lock="git-lock.xml" apikeys="/db/system/auth/tuttle-token.xml">
-    <!-- the permissions the deployed data gets assigned -->
-    <sm user="nobody" group="nogroup" mode="rw-r--r--"/>
+    <!-- 
+      The permissions the deployed data gets assigned, if no expath-pkg.xml is found.
+      The initial setting is safe, but this can and should be adapted to your requirements. 
+    -->
+    <sm user="admin" group="dba" mode="rw-r-----"/>
   </config>
 </tuttle>
diff --git a/src/modules/api.xql b/src/modules/api.xql
@@ -227,28 +227,25 @@ declare %private function api:pull($config as map(*), $hash as xs:string?) as ma
 declare function api:git-deploy($request as map(*)) as map(*) {
     try {
         let $config := api:get-collection-config($request?parameters?collection)
-        let $collection-destination := $config?path
-        let $collection-destination-sha := $config?path || "/gitsha.xml"
+        let $destination := $config?path
         let $lockfile := $config?path || "/" || config:lock()
-
-        let $collection-staging := $config?collection || config:suffix() 
-        let $collection-staging-uri := $config?path || config:suffix() 
+        let $staging := $config?path || config:suffix() 
 
-        let $ensure-destination-collection := collection:create($config?path)
+        let $ensure-destination-collection := collection:create($destination)
         return
-            if (not(xmldb:collection-available($collection-staging-uri)))
-            then map { "message" : "Staging collection '" || $collection-staging-uri || "' does not exist!" }
+            if (not(xmldb:collection-available($staging)))
+            then map { "message" : "Staging collection '" || $staging || "' does not exist!" }
             else if (doc-available($lockfile))
             then map { "message" : doc($lockfile)/task/value/text() || " in progress!" }
             else if (exists($ensure-destination-collection?error))
             then map { "message" : "Could not create destination collection!", "error": $ensure-destination-collection?error }
             else
-                let $write-lock := app:lock-write($config?path, "deploy")
-                let $is-expath-package := xmldb:get-child-resources($collection-staging-uri) = ("expath-pkg.xml", "repo.xml")
+                let $write-lock := app:lock-write($destination, "deploy")
+                let $is-expath-package := xmldb:get-child-resources($staging) = ("expath-pkg.xml", "repo.xml")
                 let $deploy :=
                     if ($is-expath-package)
                     then (
-                        let $package := doc(concat($config?path, "/expath-pkg.xml"))//@name/string()
+                        let $package := doc(concat($staging, "/expath-pkg.xml"))//@name/string()
                         let $remove-pkg :=
                             if ($package = repo:list())
                             then (
@@ -260,26 +257,24 @@ declare function api:git-deploy($request as map(*)) as map(*) {
 
                         let $xar :=
                             xmldb:store-as-binary(
-                                $collection-staging-uri, "pkg.xar", 
-                                compression:zip(xs:anyURI($collection-staging-uri), true(), $collection-staging-uri))
+                                $staging, "pkg.xar", 
+                                compression:zip(xs:anyURI($staging), true(), $staging))
 
                         let $install := repo:install-and-deploy-from-db($xar)
                         return "package installation"
                     )
                     else (
-                        let $cleanup-col := app:cleanup-collection($config?collection, config:prefix())
-                        let $cleanup-res := app:cleanup-resources($config?collection, config:prefix())
-                        let $move-col := app:move-collections($collection-staging, $config?collection, config:prefix())
-                        let $move-res := app:move-resources($collection-staging, $config?collection, config:prefix())
-                        let $set-permissions := app:set-permission($config?path)
+                        let $cleanup-col := app:cleanup-collection($destination)
+                        let $move-col := app:move-collections($staging, $destination)
+                        let $set-permissions := app:set-permission($destination)
                         return "data move"
                     )
 
-                let $remove-staging := collection:remove($collection-staging-uri, true())
-                let $remove-lock := app:lock-remove($collection-destination)
+                let $remove-staging := collection:remove($staging, true())
+                let $remove-lock := app:lock-remove($destination)
 
                 return map {
-                    "hash": config:deployed-sha($config?path),
+                    "hash": config:deployed-sha($destination),
                     "message": "success"
                 }
 

diff --git a/src/modules/app.xql b/src/modules/app.xql
@@ -1,112 +1,72 @@
 xquery version "3.1";
 
 module namespace app="http://e-editiones.org/tuttle/app";
-declare namespace output="http://www.w3.org/2010/xslt-xquery-serialization";
 
 import module namespace xmldb="http://exist-db.org/xquery/xmldb";
 import module namespace http="http://expath.org/ns/http-client";
 import module namespace compression="http://exist-db.org/xquery/compression";
 import module namespace repo="http://exist-db.org/xquery/repo";
 import module namespace sm="http://exist-db.org/xquery/securitymanager";
-import module namespace dbutil="http://exist-db.org/xquery/dbutil";
+
+import module namespace collection="http://existsolutions.com/modules/collection" at "collection.xqm";
 
 import module namespace config="http://e-editiones.org/tuttle/config" at "config.xql";
 
+declare function app:extract-archive($zip as xs:base64Binary, $collection as xs:string) {
+    compression:unzip($zip, 
+        app:unzip-filter#3, config:ignore(),
+        app:unzip-store#4, $collection)
+};
+
 (:~
  : Unzip helper function
  :)
-declare function app:unzip-store($path as xs:string, $data-type as xs:string, $data as item()?, $param as item()*) {
-    let $archive-root := substring-before($path, '/')
-    let $archive-root-length := string-length($archive-root)
-    let $object := substring-after($path, '/') 
-
-    return 
-        if ($data-type = 'folder') then 
-            let $mkcol := app:mkcol($param, $object)
-            return
-                <entry path="{$object}" data-type="{$data-type}"/>
-        else 
-            let $resource-path := "/" || tokenize($object, '[^/]+$')[1]
-            let $resource-collection := concat($param, $resource-path)
-            let $resource-filename := xmldb:encode(replace($object, $resource-path, ""))
-            return
-                try {
-                    let $collection-check := if (xmldb:collection-available($resource-collection)) then () else app:mkcol($resource-collection)
-                    let $store := xmldb:store($resource-collection, $resource-filename, $data)
-                    return ()
-                }
-                catch * {
-                    map {
-                        "_error": map {
-                        "code": $err:code, "description": $err:description, "value": $err:value, 
-                        "line": $err:line-number, "column": $err:column-number, "module": $err:module
-                    }
-                }
-            }
+declare function app:unzip-store($path as xs:string, $data-type as xs:string, $data as item()?, $base as xs:string) as map(*) {
+    if ($data-type = 'folder') then (
+        let $create := collection:create($base || "/" || substring-after($path, '/'))
+        return map { "path": $path }
+    ) else (
+        try {
+            let $resource := app:file-to-resource($base, substring-after($path, '/'))
+            let $collection-check := collection:create($resource?collection)
+            let $store := xmldb:store($resource?collection, $resource?name, $data)
+            return map { "path": $path }
+        }
+        catch * {
+            map { "path": $path, "error": $err:description }
+        }
+    )
 };
 
 (:~
  : Filter out ignored resources
+ : returning true() _will_ extract the file or folder
  :)
-declare function app:unzip-filter($path as xs:string, $data-type as xs:string, $param as item()*) as xs:boolean { 
-    not(contains($path, config:ignore()))
+declare function app:unzip-filter($path as xs:string, $data-type as xs:string, $ignore as xs:string*) as xs:boolean { 
+    not(substring-after($path, '/') = $ignore)
 };
 
 (:~
  : Move staging collection to final collection
  :)
-declare function app:move-collections($collection-source as xs:string, $collection-target as xs:string, $prefix as xs:string) {
-    let $fullpath-collection-source := $prefix || "/" || $collection-source
-    let $fullpath-collection-target := $prefix || "/" || $collection-target
-
-    return
-        for $child in xmldb:get-child-collections($fullpath-collection-source) 
-            let $fullpath-child-source := $fullpath-collection-source || "/" || $child
-            let $fullpath-child-target := $fullpath-collection-target || "/" 
-            return 
-                xmldb:move($fullpath-child-source, $fullpath-child-target)
-};
-
-(:~
- : Move staging collection to final collection
- :)
-declare function app:move-resources($collection-source as xs:string, $collection-target as xs:string, $prefix as xs:string) {
-    let $fullpath-collection-source := $prefix || "/" || $collection-source
-    let $fullpath-collection-target := $prefix || "/" || $collection-target
-
-    return
-        for $child in xmldb:get-child-resources($fullpath-collection-source) 
-        return 
-            xmldb:move($fullpath-collection-source, $fullpath-collection-target, $child)
+declare function app:move-collections($collection-source as xs:string, $collection-target as xs:string) {
+    xmldb:get-child-collections($collection-source) 
+        ! xmldb:move($collection-source || "/" || ., $collection-target),
+    xmldb:get-child-resources($collection-source) 
+        ! xmldb:move($collection-source, $collection-target, .)
 };
 
 (:~ 
  : Cleanup destination collection - delete collections from target collection
  :)
-declare function app:cleanup-collection($collection as xs:string, $prefix as xs:string) {
-    let $ignore := [config:ignore(), config:lock()]
-    let $fullpath-collection :=  $prefix || "/" || $collection 
-
-    return
-        for $child in xmldb:get-child-collections($fullpath-collection)
-            where not(contains($child, $ignore))
-                let $fullpath-child := $fullpath-collection || "/" || $child
-                return 
-                    xmldb:remove($fullpath-child)
-};
-
-(:~
- : Cleanup destination collection - delete resources from target collection
- :)
-declare function app:cleanup-resources($collection as xs:string, $prefix as xs:string) {
-    let $ignore := config:ignore()
-    let $fullpath-collection :=  $prefix || "/" || $collection 
-
-    return
-        for $child in xmldb:get-child-resources($fullpath-collection)
-            where not(contains($child, $ignore))
-            return 
-                xmldb:remove($fullpath-collection, $child)
+declare function app:cleanup-collection($collection as xs:string) {
+    let $ignore := (config:ignore(), config:lock())
+    return (
+        xmldb:get-child-collections($collection)[not(.= $ignore)]
+            ! xmldb:remove($collection || "/" || .),
+        xmldb:get-child-resources($collection)[not(.= $ignore)]
+            ! xmldb:remove($collection, .)
+    )
 };
 
 (:~
@@ -136,8 +96,8 @@ declare function app:write-apikey($collection as xs:string, $apikey as xs:string
     try {
         let $collection-prefix := tokenize(config:apikeys(), '[^/]+$')[1]
         let $apikey-resource := xmldb:encode(replace(config:apikeys(), $collection-prefix, ""))
-        let $collection-check := 
-            if (xmldb:collection-available($collection-prefix)) then () else app:mkcol($collection-prefix)
+        let $collection-check := collection:create($collection-prefix)
+
         return 
             if (doc(config:apikeys())//apikeys/collection[name = $collection]/key/text()) then
                 update replace doc(config:apikeys())//apikeys/collection[name = $collection]/key with <key>{$apikey}</key>
@@ -180,79 +140,46 @@ declare function app:lock-remove($collection as xs:string) {
 };
 
 (:~
-: Set recursive permissions to whole collection 
-:)
+ : Set permissions to collection recursively 
+ :)
 declare function app:set-permission($collection as xs:string) {
-    let $collection-uri := config:prefix() || "/" || $collection
+    let $permissions := app:get-permissions($collection)
+    let $callback := app:set-permission(?, ?, $permissions)
 
-    return 
-        dbutil:scan(xs:anyURI($collection-uri), function($collection-url, $resource) {
-        let $path := ($resource, $collection-url)[1]
-        return (
-            if ($resource) then
-                    app:set-permission($collection, $path, "resource")
-                else
-                    app:set-permission($collection, $path, "collection")
-            )
-        })
+    return
+        collection:scan($collection, $callback)
 };
 
-(:~
-: Set permissions for $path 
-: $type: 'collection' or 'resource'
-:)
-declare function app:set-permission($collection as xs:string, $path as xs:string, $type as xs:string) {
-    let $collection-uri := config:prefix() || "/" || $collection
-    let $repo := doc(concat($collection-uri, "/repo.xml"))/repo:meta/repo:permissions
-
-    return (
-        if (exists($repo)) then (
-            sm:chown($path, $repo/@user/string()),
-            sm:chgrp($path, $repo/@group/string()),
-            if ($type = "resource") then
-                    sm:chmod($path, $repo/@mode/string())
-                else
-                    sm:chmod($path, replace($repo/@mode/string(), "(..).(..).(..).", "$1x$2x$3x"))
-        )
-        else (
-            sm:chown($path, config:sm()?user),
-            sm:chgrp($path, config:sm()?group),
-            if ($type = "resource") then
-                    sm:chmod($path, config:sm()?mode)
-                else
-                    sm:chmod($path, replace(config:sm()?mode, "(..).(..).(..).", "$1x$2x$3x")))
-        )
+declare function app:get-permissions ($collection as xs:string) {
+    if (doc-available($collection || "/repo.xml")) then (
+        let $repo := doc($collection || "/repo.xml")//repo:permissions
+        return map {
+            "user":  $repo/@user/string(),
+            "group":  $repo/@group/string(),
+            "mode":  $repo/@mode/string()
+        }
+    ) else (
+        config:sm()
+    )
 };
 
 (:~
- : Helper function of unzip:mkcol() 
+: Set permissions for either a collection or resource
 :)
-declare function app:mkcol-recursive($collection, $components) as xs:string* {
-    if (exists($components)) then
-        let $newColl := concat($collection, "/", $components[1])
-        return (
-            xmldb:create-collection($collection, $components[1]),
-            if ($components[2]) then 
-                app:mkcol-recursive($newColl, subsequence($components, 2))
-            else ()
+declare function app:set-permission($collection as xs:string, $resource as xs:string?, $permissions as map(*)) {
+    if (exists($resource)) then (
+        xs:anyURI($resource) ! (
+            sm:chown(., $permissions?user),
+            sm:chgrp(., $permissions?group),
+            sm:chmod(., $permissions?mode)
         )
-    else ()
-};
-
-(:~
- : Helper function to recursively create a collection hierarchy
- :)
-declare function app:mkcol($collection, $path) as xs:string* {
-    app:mkcol-recursive($collection,
-        tokenize($path, "/") ! xmldb:encode(.))
-};
-
-(:~
- : Helper function to recursively create a collection hierarchy
- :)
-declare function app:mkcol($path) as xs:string* {
-    app:mkcol('/db',
-        substring-after($path, "/db/"))
+    ) else (
+        xs:anyURI($collection) ! (
+            sm:chown(., $permissions?user),
+            sm:chgrp(., $permissions?group),
+            sm:chmod(., replace($permissions?mode, "(r.)-", "$1x"))
+        )
+    )
 };
 
 (:~
@@ -296,10 +223,6 @@ declare function app:request($request as element(http:request)) {
         else $response
 };
 
-declare function app:extract-archive($zip as xs:base64Binary, $collection as xs:string) {
-    compression:unzip($zip, app:unzip-filter#3, (), app:unzip-store#4, $collection)
-};
-
 (:~
  : resolve file path against a base collection
  : $base the absolute DB path to a collection. Assumes no slash at the end
@@ -331,15 +254,15 @@ declare function app:delete-resource($config as map(*), $filepath as xs:string)
  :)
 declare function app:add-resource($config as map(*), $filepath as xs:string, $data as item()) as xs:boolean {
     let $resource := app:file-to-resource($config?path, $filepath)
-
+    let $permissions := app:get-permissions($config?path)
     let $collection-check := 
         if (xmldb:collection-available($resource?collection)) then ()
         else (
-            app:mkcol($resource?collection),
-            app:set-permission($config?collection, $resource?collection, "collection")
+            collection:create($resource?collection),
+            app:set-permission($resource?collection, (), $permissions)
         )
 
     let $store := xmldb:store($resource?collection, $resource?name, $data)
-    let $chmod := app:set-permission($config?collection, $resource?collection || $resource?name, "resource")
+    let $chmod := app:set-permission($resource?collection, $store, $permissions)
     return true()
 };