Merge pull request #51 from line-o/fix/publish-package

joewiz · web-flow · commit 78010d1481c4 · 2021-02-08T14:18:45.000-05:00
fix: timeout on publish
diff --git a/admin.html b/admin.html
@@ -30,8 +30,8 @@ <h1>Package Upload</h1>
         </div>
         <div class="col-md-6">
             <p>
-                <a href="index.html?publish=true" class="btn btn-primary">
-                    <i class="glyphicon glyphicon-share"/> Publish
+                <a href="index.html" class="btn btn-primary">
+                    <i class="glyphicon glyphicon-share"/> Return to list 
                 </a>
             </p>
             <p>Publishing will take a while to process apps.</p>
diff --git a/build.xml b/build.xml
@@ -27,7 +27,7 @@
                 <include name="modules/**"/>
                 <include name="resources/**"/>
                 <include name="templates/**"/>
-                <include name="public/*"/>
+                <include name="meta/*"/>
                 <exclude name="${build}/*"/>
                 <exclude name=".git*"/>
                 <exclude name="*.tmpl"/>
diff --git a/index.html b/index.html
@@ -19,7 +19,6 @@ <h1>Public Application Repository</h1>
                 Manager app. Also, subscribe to the <a href="feed.xml">atom feed</a> for all package updates
                 for this repository.</p>
         </div>
-        <div data-template="app:publish"/>
     </div>
     <div class="row">
         <div id="listWrapper" class="col-md-8">
diff --git a/meta/apps.xml b/meta/apps.xml
@@ -0,0 +1 @@
+<apps/>
diff --git a/meta/packages.xml b/meta/packages.xml
@@ -0,0 +1 @@
+<packages/>
diff --git a/modules/app.xql b/modules/app.xql
@@ -5,15 +5,8 @@ module namespace app="http://exist-db.org/xquery/app";
 import module namespace config="http://exist-db.org/xquery/apps/config" at "config.xqm";
 import module namespace scanrepo="http://exist-db.org/xquery/admin/scanrepo" at "scan.xql";
 
-declare function app:publish($node as node(), $model as map(*), $publish as xs:boolean?) {
-    if ($publish) then
-        scanrepo:scan()
-    else
-        ()
-};
-
 declare function app:list-packages($node as node(), $model as map(*), $mode as xs:string?) {
-    for $app in collection($config:public)//app
+    for $app in doc($config:apps-meta)//app
     let $show-details := false()
     order by lower-case($app/title)
     return
@@ -23,7 +16,7 @@ declare function app:list-packages($node as node(), $model as map(*), $mode as x
 declare function app:view-package($node as node(), $model as map(*), $mode as xs:string?) {
     let $abbrev := request:get-parameter("abbrev", ())
     let $procVersion := request:get-parameter("eXist-db-min-version", "2.2.0")
-    let $matching-abbrev := collection($config:public)//abbrev[. eq $abbrev]
+    let $matching-abbrev := doc($config:apps-meta)//abbrev[. eq $abbrev]
     let $apps := $matching-abbrev/parent::app
     return
         (
diff --git a/modules/config.xqm b/modules/config.xqm
@@ -27,6 +27,12 @@ declare variable $config:app-root :=
 
 declare variable $config:public := concat($config:app-root, "/public");
 
+declare variable $config:metadata-collection := concat($config:app-root, "/meta");
+declare variable $config:apps-doc := 'apps.xml';
+declare variable $config:packages-doc := 'packages.xml';
+declare variable $config:packages-meta := concat($config:metadata-collection, '/', $config:packages-doc);
+declare variable $config:apps-meta := concat($config:metadata-collection, '/', $config:apps-doc);
+
 (:~
  : Returns the repo.xml descriptor for the current application.
  :)
diff --git a/modules/feed.xql b/modules/feed.xql
@@ -6,7 +6,7 @@ declare option exist:serialize "method=xml media-type=application/atom+xml";
 
 declare function local:feed-entries() {
     let $repoURL := concat(substring-before(request:get-url(), "public-repo/"), "public-repo/")
-    for $app in collection($config:public)//app
+    for $app in doc($config:apps-meta)//app
     let $icon :=
         if ($app/icon) then
             if ($app/@status) then
diff --git a/modules/find.xql b/modules/find.xql
@@ -18,17 +18,17 @@ let $procVersion := request:get-parameter("processor", "2.2.0")
 let $app-root-absolute-url := request:get-parameter("app-root-absolute-url", ())
 let $app :=
     if ($name) then
-        collection($config:app-root || "/public")//app[name eq $name]
+        doc($config:apps-meta)//app[name eq $name]
     else
-        collection($config:app-root || "/public")//app[abbrev eq $abbrev]
+        doc($config:apps-meta)//app[abbrev eq $abbrev]
 let $app-versions := ($app, $app/other/version)
 let $compatible-xar := app:find-version($app-versions, $procVersion, $version, $semVer, $minVersion, $maxVersion)
 return
     if ($compatible-xar) then
         let $abs-public := $app-root-absolute-url || "/public/"
         return
             if ($info) then
-                let $app := collection($config:app-root || "/public")//(app|version)[@path eq $compatible-xar]
+                let $app := doc($config:apps-meta)//(app|version)[@path eq $compatible-xar]
                 return
                     <found>{$app/@sha256,($app/version,$app/@version)[1] ! attribute version {.},$compatible-xar}</found>
             else if ($zip) then
diff --git a/modules/list.xql b/modules/list.xql
@@ -74,11 +74,11 @@ declare function list:find-version($versions as element()*, $version as xs:strin
 
 let $version := request:get-parameter("version", $list:DEFAULT_VERSION)
 let $version := if (matches($version, "^\d+\.\d+\.\d+-?.*$")) then $version else $list:DEFAULT_VERSION
-let $apps := doc($config:public || "/apps.xml")/apps
+
 return
     <apps version="{$version}">
     {
-        for $app in $apps/app
+        for $app in doc($config:apps-meta)//app
         return
             list:get-app($app, $version)
     }
diff --git a/modules/scan.xql b/modules/scan.xql
@@ -145,47 +145,61 @@ declare %private function scanrepo:compare-versions($installed as xs:string*, $a
         $compare(head($available), head($installed))
 };
 
+declare function scanrepo:handle-icon ($path as xs:string, $data as item()?, $param as item()*) as element(icon) {
+    let $pkgName := substring-before($param, ".xar")
+
+    let $suffix := replace($path, "^.*\.([^\.]+)", "$1")
+    let $name := concat($pkgName, ".", $suffix)
+    let $stored := xmldb:store($config:public, $name, $data)
+
+    return
+        <icon>{ $name }</icon>
+};
+
+declare function scanrepo:handle-expath-package ($root as element(expath:package)) as element()* {
+    <name>{$root/@name/string()}</name>,
+    <title>{$root/expath:title/text()}</title>,
+    <abbrev>{$root/@abbrev/string()}</abbrev>,
+    <version>{$root/@version/string()}</version>,
+    if ($root/expath:dependency[starts-with(@processor, "http://exist-db.org")]) then
+        <requires>{ $root/expath:dependency[starts-with(@processor, "http://exist-db.org")]/@* }</requires>
+    else
+        ()
+};
+
+declare function scanrepo:handle-repo-meta ($root as element(repo:meta)) as element()+ {
+    for $author in $root/repo:author
+    return
+        <author>{$author/text()}</author>
+    ,
+    <description>{$root/repo:description/text()}</description>,
+    <website>{$root/repo:website/text()}</website>,
+    <license>{$root/repo:license/text()}</license>,
+    <type>{$root/repo:type/text()}</type>
+    ,
+    for $note in $root/repo:note
+    return
+        <note>{$note/text()}</note>
+    ,
+    <changelog>
+    {
+        scanrepo:copy-changelog($root/repo:changelog/repo:change)
+    }
+    </changelog>
+};
+
 declare function scanrepo:entry-data($path as xs:anyURI, $type as xs:string, $data as item()?, $param as item()*) as item()*
 {
-    if (starts-with($path, "icon")) then
-        let $pkgName := substring-before($param, ".xar")
-        let $suffix := replace($path, "^.*\.([^\.]+)", "$1")
-        let $name := concat($pkgName, ".", $suffix)
-        let $stored :=
-            xmldb:store($config:public, $name, $data)
-        return
-            <icon>{ $name }</icon>
+    if (starts-with($path, "icon")) then 
+        scanrepo:handle-icon($path, $data, $param)
     else
         let $root := $data/*
         return
             typeswitch ($root)
-                case element(expath:package) return (
-                    <name>{$root/@name/string()}</name>,
-                    <title>{$root/expath:title/text()}</title>,
-                    <abbrev>{$root/@abbrev/string()}</abbrev>,
-                    <version>{$root/@version/string()}</version>,
-                    if ($root/expath:dependency[starts-with(@processor, "http://exist-db.org")]) then
-                        <requires>{ $root/expath:dependency[starts-with(@processor, "http://exist-db.org")]/@* }</requires>
-                    else
-                        ()
-                )
-                case element(repo:meta) return (
-                    for $author in $root/repo:author
-                    return
-                        <author>{$author/text()}</author>,
-                    <description>{$root/repo:description/text()}</description>,
-                    <website>{$root/repo:website/text()}</website>,
-                    <license>{$root/repo:license/text()}</license>,
-                    <type>{$root/repo:type/text()}</type>,
-                    for $note in $root/repo:note
-                    return
-                        <note>{$note/text()}</note>,
-                    <changelog>
-                    {
-                        scanrepo:copy-changelog($root/repo:changelog/repo:change)
-                    }
-                    </changelog>
-                )
+                case element(expath:package) return
+                    scanrepo:handle-expath-package($root)
+                case element(repo:meta) return
+                    scanrepo:handle-repo-meta($root)
                 default return
                     ()
 };
@@ -203,23 +217,27 @@ declare function scanrepo:copy-changelog($nodes as node()*) {
                 $node
 };
 
-declare function scanrepo:entry-filter($path as xs:anyURI, $type as xs:string, $param as item()*) as xs:boolean
-{
-    starts-with($path, "icon.") or $path = ("repo.xml", "expath-pkg.xml")
+declare function scanrepo:entry-filter($path as xs:anyURI, $type as xs:string, $param as item()*) as xs:boolean {
+    starts-with($path, "icon.") or 
+    $path = ("repo.xml", "expath-pkg.xml")
 };
 
-declare function scanrepo:extract-metadata($resource as xs:string) {
+declare function scanrepo:extract-metadata ($resource as xs:string) as element(app) {
     let $xar := concat($config:public, "/", $resource)
-    let $hash := crypto:hash(
-        util:binary-doc($xar),
-        "sha256",
-        "hex"
-    )
+    let $data := util:binary-doc($xar)
+
+    let $hash := crypto:hash($data, "sha256", "hex")
+
     return
         <app path="{$resource}" size="{xmldb:size($config:public, $resource)}" sha256="{$hash}">
         {
-            compression:unzip(util:binary-doc($xar), util:function(xs:QName("scanrepo:entry-filter"), 3), (),
-                util:function(xs:QName("scanrepo:entry-data"), 4), $resource)
+            compression:unzip(
+                $data,
+                scanrepo:entry-filter#3, 
+                (),
+                scanrepo:entry-data#4,
+                $resource
+            )
         }
         </app>
 };
@@ -232,8 +250,32 @@ declare function scanrepo:scan-all() {
 };
 
 declare function scanrepo:scan() {
-    let $data := scanrepo:scan-all()
-    let $processed := scanrepo:process($data)
+    let $data := doc($config:packages-meta)//app
+    let $processed := <apps> { scanrepo:process($data) }</apps>
+    let $store := xmldb:store($config:metadata-collection, $config:apps-doc, $processed)
+
+    return $processed
+};
+
+declare function scanrepo:rebuild-package-meta() as xs:string {
+    xmldb:store($config:metadata-collection, $config:packages-doc,
+        <packages> { scanrepo:scan-all() }</packages>)
+};
+
+declare function scanrepo:add-package-meta($meta as element(app)) {
+    let $packages := doc($config:packages-meta)/packages
+    let $node-to-update := $packages/app[@path=$meta/@path]
+
     return
-        xmldb:store($config:public, "apps.xml", <apps> { $processed }</apps>)
+        if (exists($node-to-update))
+        then (update replace $node-to-update with $meta)
+        else (update insert $meta into $packages)
+};
+
+declare function scanrepo:publish($xar as xs:string) {
+    let $meta := $xar
+        => scanrepo:extract-metadata()
+        => scanrepo:add-package-meta()
+    
+    return scanrepo:scan()
 };
diff --git a/modules/update.xql b/modules/update.xql
@@ -2,6 +2,10 @@ xquery version "1.0";
 
 import module namespace scanrepo="http://exist-db.org/xquery/admin/scanrepo" at "scan.xql";
 
-let $scan := scanrepo:scan()
-return
-    ()
+scanrepo:rebuild-package-meta()
+=> scanrepo:scan()
+
+(: $file
+=> scanrepo:extract-metadata()
+=> scanrepo:add-package-meta()
+=> scanrepo:scan() :)
diff --git a/modules/upload.xql b/modules/upload.xql
@@ -1,30 +1,32 @@
 xquery version "3.0";
 
 import module namespace config="http://exist-db.org/xquery/apps/config" at "config.xqm";
+import module namespace scanrepo="http://exist-db.org/xquery/admin/scanrepo" at "scan.xql";
 
 declare namespace json="http://www.json.org";
 
 declare option exist:serialize "method=json media-type=application/json";
 
-declare function local:upload($collection, $path, $data) {
-    let $path := xmldb:store($collection, $path, $data)
-    let $upload :=
+declare function local:upload-and-publish($xar as xs:string, $data) {
+    let $path := xmldb:store($config:public, $xar, $data)
+    let $scan := scanrepo:publish($xar)
+
+    return
         <result>
             <files json:array="true">
                <name>{$path}</name>
                <type>{xmldb:get-mime-type($path)}</type>
-               <size>{xmldb:size($collection, $path)}</size>
+               <size>{xmldb:size($config:public, $xar)}</size>
             </files>
        </result>
-    return
-        $upload
 };
 
 let $name := request:get-uploaded-file-name("files[]")
 let $data := request:get-uploaded-file-data("files[]")
+
 return
     try {
-        local:upload($config:public, $name, $data)
+        local:upload-and-publish($name, $data)
     } catch * {
         <result>
            <name>{$name}</name>
diff --git a/post-install.xql b/post-install.xql
@@ -59,5 +59,5 @@ else
 system:as-user("repo", "repo", (
     local:copy-previous-public-from-temp-or-create(),
     xmldb:store-files-from-pattern(concat($target, "/public"), local:get-repo-dir(), "*.xar"),
-    scanrepo:scan()
+    scanrepo:rebuild-package-meta()
 ))

Original file line number	Diff line number	Diff line change
`@@ -74,11 +74,11 @@ declare function list:find-version($versions as element()*, $version as xs:strin`
`74`	`74`
`75`	`75`	`let $version := request:get-parameter("version", $list:DEFAULT_VERSION)`
`76`	`76`	`let $version := if (matches($version, "^\d+\.\d+\.\d+-?.*$")) then $version else $list:DEFAULT_VERSION`
`77`		`-let $apps := doc($config:public \|\| "/apps.xml")/apps`
	`77`	`+`
`78`	`78`	`return`
`79`	`79`	`<apps version="{$version}">`
`80`	`80`	`{`
`81`		`- for $app in $apps/app`
	`81`	`+ for $app in doc($config:apps-meta)//app`
`82`	`82`	`return`
`83`	`83`	`list:get-app($app, $version)`
`84`	`84`	`}`