Merge pull request #51 from intersystems-community/stage

Proxy Registry

Author: nsolov

Dockerfile

@@ -1,7 +1,10 @@
-FROM store/intersystems/iris-community:2020.1.0.215.0
+#FROM store/intersystems/iris-community:2020.1.0.215.0
+FROM intersystemsdc/iris-community:2020.4.0.524.0-zpm
 
 USER root
 
+COPY zpm-registry.yaml /usr/irissys/
+
 WORKDIR /opt/zpm
 RUN chown ${ISC_PACKAGE_MGRUSER}:${ISC_PACKAGE_IRISGROUP} .
 
@@ -15,7 +18,11 @@ SHELL ["/irissession.sh"]
 
 RUN \
   do $SYSTEM.OBJ.Load("Installer.cls", "ck") \
-  set sc = ##class(ZPM.Installer).setup()
+  set sc = ##class(ZPM.Installer).setup()    \
+  zn "registry"                              \
+  zpm "install yaml-utils"
 
 # bringing the standard shell back
-SHELL ["/bin/bash", "-c"]
+SHELL ["/bin/bash", "-c"]
+
+

Installer.cls

@@ -27,18 +27,31 @@ XData setup
       Recurse="1" />
 
   </Namespace>
-
   <Invoke Class="ZPM.Installer" Method="SetDispatchClass">
     <Arg Value="${CSPAPP}"/>
     <Arg Value="ZPM.Registry"/>
   </Invoke>
 
+  <Invoke Class="ZPM.Installer" Method="AddSSLConfiguration"></Invoke>
+
   <Invoke Class="Security.SQLPrivileges" Method="Import" CheckStatus="true">
     <Arg Value="${APPPATH}SQLPriv.xml"/>
   </Invoke>
+
 </Manifest>
 }
 
+ClassMethod AddSSLConfiguration()
+{
+    Set host = "zpmregistry"
+    New $NAMESPACE
+    Set $NAMESPACE = "%SYS"
+    If '##class(Security.SSLConfigs).Exists(host) {
+        Do ##class(Security.SSLConfigs).Create(host)
+    }
+    Return host
+}
+
 ClassMethod setup(ByRef pVars, pLogLevel As %Integer = 3, pInstaller As %Installer.Installer, pLogger As %Installer.AbstractLogger) As %Status [ CodeMode = objectgenerator, Internal ]
 {
   #; Let XGL document generate code for this method. 

SQLPriv.xml

@@ -16,4 +16,12 @@
     <Grantor>_SYSTEM</Grantor>
     <Grantable>0</Grantable>
   </SQLPrivileges>
+  <SQLPrivileges>
+    <Namespace>REGISTRY</Namespace>
+    <SQLObject>1,ZPM.UpLink</SQLObject>
+    <Privilege>s</Privilege>
+    <Grantee>_PUBLIC</Grantee>
+    <Grantor>_SYSTEM</Grantor>
+    <Grantable>0</Grantable>
+  </SQLPrivileges>
 </SQLPrivilegesExport>

module.xml

@@ -4,19 +4,26 @@
     <Module>
       <Name>zpm-registry</Name>
       <Description>Registry server for ZPM</Description>
-      <Version>1.0.10</Version>
+      <Version>1.1.0</Version>
       <Packaging>module</Packaging>
+      <Dependencies>
+        <ModuleReference>
+          <Name>yaml-utils</Name>
+          <Version>0.1.*</Version>
+        </ModuleReference>
+      </Dependencies>
       <SourcesRoot>src</SourcesRoot>
       <Resource Name="ZPM.PKG"/>
-
+      <Invoke Class="ZPM.Utils" Method="SQLsetup"></Invoke>
+      <Invoke Class="ZPM.Utils" Method="AddSSLConfiguration"></Invoke>
       <CSPApplication 
         Url="/registry"
         Path="/src"
         Recurse="1"
         Directory="{$cspdir}/registry" 
         MatchRoles=":{$dbrole}"
         PasswordAuthEnabled="1"
-        UnauthenticatedEnabled="0"
+        UnauthenticatedEnabled="1"
         DispatchClass="ZPM.Registry"
         ServeFiles="1"
         CookiePath="/registry"

src/cls/ZPM/Package.cls

@@ -29,11 +29,11 @@ Index Repository On repository;
 
 Property dependencies As array Of %String;
 
-Property package As %Stream.GlobalBinary(%JSONINCLUDE = "INPUTONLY") [ Required ];
+Property package As %Stream.GlobalBinary(%JSONINCLUDE = "INPUTONLY");
 
 Property size As %Integer(%JSONINCLUDE = "OUTPUTONLY");
 
-Property manifest As %Stream.GlobalCharacter(%JSONINCLUDE = "INPUTONLY") [ Required ];
+Property manifest As %Stream.GlobalCharacter(%JSONINCLUDE = "INPUTONLY");
 
 Property installer As %Stream.GlobalCharacter(%JSONINCLUDE = "INPUTONLY");
 
@@ -55,6 +55,12 @@ Property versionPrerelease As %String [ SqlComputeCode = {Set {*} = $Piece($Piec
 
 Property versionBuildmetadata As %String [ SqlComputeCode = {Set {*} = $Piece({version},"+",2)}, SqlComputed, SqlComputeOnChange = %%INSERT ];
 
+Property UpLink As ZPM.UpLink(%JSONINCLUDE = "NONE");
+
+ForeignKey UpLinkFK(UpLink) References ZPM.UpLink() [ OnDelete = cascade ];
+
+Index NameUpLink On (name, UpLink);
+
 Method nameSet(value As %String) As %Status
 {
   set i%name = $$$lcase(value)
@@ -68,8 +74,9 @@ Method sizeGet() As %Integer
 
 ClassMethod GetPackageHash(pStream As %Stream.Object) As %String
 {
-  If '$isobject(pStream) Set pStream = ..packageOpen(pStream)
-  set hash = $system.Encryption.SHA1HashStream(pStream)
+  If (pStream="") { Return "" }
+  If '$isobject(pStream) { Set pStream = ..packageOpen(pStream) }
+  Set hash = $system.Encryption.SHA1HashStream(pStream)
   Set str="" 
   For i=1:1:$length(hash) {
 		Set str=str_$translate($justify($zhex($ascii(hash, i)),2)," ",0)
@@ -79,18 +86,19 @@ ClassMethod GetPackageHash(pStream As %Stream.Object) As %String
 
 ClassMethod ServerURL() As %String
 {
-  SET host = ""
-  IF $ISOBJECT($GET(%request)) {
-    set tSecure = %request.Secure 
-    SET host = %request.GetCgiEnv("HTTP_X_FORWARDED_HOST", %request.GetCgiEnv("SERVER_NAME"))
-    set proto = %request.GetCgiEnv("HTTP_X_FORWARDED_PROTO", $SELECT(tSecure: "https", 1: "http"))
-    SET port = %request.GetCgiEnv("HTTP_X_FORWARDED_PORT", %request.GetCgiEnv("SERVER_PORT"))
-    SET host = proto _ "://" _ host
-    IF '$LISTFIND($LISTBUILD(80, 443), port) {
-      SET host = host_":"_port
+  Set host = ""
+  If $ISOBJECT($Get(%request)) {
+    Set tSecure = %request.Secure 
+    Set host = %request.GetCgiEnv("HTTP_X_FORWARDED_HOST", %request.GetCgiEnv("SERVER_NAME"))
+    Set proto = %request.GetCgiEnv("HTTP_X_FORWARDED_PROTO", $SELECT(tSecure: "https", 1: "http"))
+    Set port = %request.GetCgiEnv("HTTP_X_FORWARDED_PORT", %request.GetCgiEnv("SERVER_PORT"))
+    Set host = proto _ "://" _ host
+    If '$LISTFIND($LISTBUILD(80, 443), port) {
+      Set host = host_":"_port
     }    
   }
-  RETURN host
+  //Set host = host_$p(%request.Application,"/",1,*-1)
+  Return host
 }
 
 Method urlGet() As %String
@@ -135,6 +143,62 @@ Method versionsGet() As %ListOfDataTypes
   return tList
 }
 
+ClassMethod VersionFind(pkg As %String = "", version As %String = "") As %String
+{
+  If (version = "") || (version = "latest") || (version = "*") {
+    // package was published directly in this registry - return the last version
+    &sql(SELECT TOP 1 Version INTO :version FROM ZPM.Package WHERE Name = :pkg AND UpLink IS NULL
+          ORDER BY versionMajor DESC, versionMinor DESC, versionPatch DESC, versionPrerelease DESC, versionBuildmetadata DESC
+    )
+    If SQLCODE=0 { 
+      // found 
+      Return version
+    } Else {
+      // find the latest version in UpLinks 
+      Do ##class(ZPM.UpLink).LoadPackageFromAllUpLinks(pkg, "latest")
+      &sql(SELECT TOP 1 Version INTO :version FROM ZPM.Package WHERE Name = :pkg
+          ORDER BY versionMajor DESC, versionMinor DESC, versionPatch DESC, versionPrerelease DESC, versionBuildmetadata DESC
+      )
+      If SQLCODE=0 { 
+        // if manifest doesn''t exists - find in uplinks
+        Set package = ##class(ZPM.Package).%OpenId(pkg_"||"_version)
+        If ('$IsObject(package)) { Return ""}
+        If (package.manifest.Size) { Return version }
+        Do ##class(ZPM.UpLink).LoadPackageFromAllUpLinks(pkg, version)
+        Return version
+      } 
+      Return ""
+    }
+  } Else {
+    If ( ##class(ZPM.Package).NameUpLinkExists(pkg, "") ) {
+      If ..%ExistsId(pkg _ "||" _ version) {
+        Return version
+      } 
+      Return ""
+    } Else {
+      If ..%ExistsId(pkg _ "||" _ version) {
+        Set package = ##class(ZPM.Package).%OpenId(pkg_"||"_version)
+        If ('$IsObject(package)) { Return ""}
+        If (package.manifest.Size) { Return version }
+      } 
+      Do ##class(ZPM.UpLink).LoadPackageFromAllUpLinks(pkg, version)
+      If ..%ExistsId(pkg _ "||" _ version) {
+        Return version
+      }
+      Return ""
+    }
+  }
+}
+
+ClassMethod DeleteExistingPackages(pkg, version) As %Status
+{
+  // delete all "proxy" packages
+  &sql(DELETE FROM ZPM.Package WHERE Name = :pkg AND UpLink IS NOT NULL)
+
+  Do ..NameVersionDelete(pkg, version)
+  Return 1
+}
+
 ClassMethod versionValidate(pkg As %String = "", version As %String = "") As %String
 {
   if (pkg="") {
@@ -250,7 +314,7 @@ ClassMethod LoadPackage(Path As %String, repository As %String = "", silent As %
     WRITE "ArchiveSize: " _ archive.Size
   }
 
-  do ..%DeleteId(name _ "||" _version)
+  do ..DeleteExistingPackages(name,version)
   SET package = ..%New()
   SET package.name = name
   Set package.description = description
@@ -281,7 +345,7 @@ ClassMethod GetDefaultBranch(path As %String) As %String
   Set branch = "main"
   Set ht = ##class(%Net.HttpRequest).%New()
   Set ht.Server = "api.github.com"
-  Set ht.SSLConfiguration = ..GetSSLConfiguration(ht.Server)
+  Set ht.SSLConfiguration = ##class(ZPM.Utils).GetSSLConfiguration()
   Set ht.Https = 1
   Set ht.Location = "/repos"_path 
   $$$ThrowOnError(ht.Get())
@@ -299,7 +363,7 @@ ClassMethod DownloadPackageFromGitHub(url As %String, Output branch As %String)
   Set ht = ##class(%Net.HttpRequest).%New()
   Set ht.Server = tComponents("host")
   If $get(tComponents("scheme"))="https" {
-    Set ht.SSLConfiguration = ..GetSSLConfiguration(ht.Server)
+    Set ht.SSLConfiguration = ##class(ZPM.Utils).GetSSLConfiguration()
     Set ht.Https = 1
   }
   If $data(tComponents("port"), port), port'="" {
@@ -330,7 +394,7 @@ ClassMethod GetModuleMeta(url As %String) As %DynamicObject
   Set ht = ##class(%Net.HttpRequest).%New()
   Set ht.Server = "raw.githubusercontent.com"
   If $get(tComponents("scheme"))="https" {
-    Set ht.SSLConfiguration = ..GetSSLConfiguration(ht.Server)
+    Set ht.SSLConfiguration = ##class(ZPM.Utils).GetSSLConfiguration()
     Set ht.Https = 1
   }
   If $data(tComponents("port"), port), port'="" {
@@ -341,24 +405,13 @@ ClassMethod GetModuleMeta(url As %String) As %DynamicObject
   If (ht.HttpResponse.StatusCode'=200) {
     $$$ThrowStatus($$$ERROR(5001,"Unable to download XML "_ht.Location))
   }
-  Set tmpFolder = $$$FileTempDir
-
-  Set stream=##class(%Stream.FileCharacter).%New()
-  Do stream.LinkToFile(tmpFolder_"/module.xml")
-  Set stream.TranslateTable = "UTF8"
-  Do stream.CopyFrom(ht.HttpResponse.Data)
-  Do stream.%Save()
 
-  set binaryStream=##class(%Stream.FileBinary).%New()
-  do binaryStream.LinkToFile(tmpFolder_"/module.xml")
+  Set xPathArray("Module/Version")=""
+  Set xPathArray("Module/Name")=""
+  $$$ThrowOnError(##class(ZPM.Utils).ReadXMLElements(ht.HttpResponse.Data, .xPathArray))
 
-  $$$ThrowOnError(##class(%XML.XPATH.Document).CreateFromStream(binaryStream, .xpathdoc))
-  $$$ThrowOnError(xpathdoc.EvaluateExpression("/","Export/Document/Module/Version/text()", .tResults))
-  set jo.version = tResults.GetAt(1).ValueGet()
-  $$$ThrowOnError(xpathdoc.EvaluateExpression("/","Export/Document/Module/Name/text()", .tResults))
-  set jo.name = $$$lcase(tResults.GetAt(1).ValueGet())
-
-  Do ##class(%File).RemoveDirectoryTree(tmpFolder)
+  Set jo.name = $Get(xPathArray("Module/Name","text"))
+  Set jo.version = $Get(xPathArray("Module/Version","text"))
 
   return jo
 }
@@ -377,6 +430,9 @@ ClassMethod UpdatePackage(jo As %DynamicObject) As %Status
     If (meta.name = "") { $$$ThrowStatus($$$ERROR(5001, "Package name is empty")) }
     Set tName = meta.name
     Set id = ""
+
+    &sql(DELETE FROM ZPM.Package WHERE Name = :tName AND UpLink IS NOT NULL) // delete all "proxy" packages
+
     &sql(SELECT ID into :id FROM ZPM.Package WHERE name=:tName ORDER BY versionMajor DESC, versionMinor DESC, versionPatch DESC, versionPrerelease  DESC)
     If (id'="") { 
       If (..repositoryGetStored(id)'=url) { 
@@ -432,15 +488,66 @@ ClassMethod LoadFromGitHub(Url = "")
   do ##class(%File).RemoveDirectoryTree(outputFolder)
 }
 
-ClassMethod GetSSLConfiguration(host) As %String
+ClassMethod GetTopVersion(pkg As %String) As %String
 {
-  NEW $NAMESPACE
-  SET $NAMESPACE = "%SYS"
-  
-  IF '##class(Security.SSLConfigs).Exists(host) {
-    DO ##class(Security.SSLConfigs).Create(host)
+  Set version = ""
+  &sql(SELECT TOP 1 version INTO :version FROM ZPM.Package 
+        WHERE name = :pkg
+        ORDER BY versionMajor DESC, versionMinor DESC, versionPatch DESC, versionPrerelease DESC, versionBuildmetadata DESC ) 
+  Return version
+}
+
+/// returns latest versions of packages 
+ClassMethod GetLatest(searchTerms As %DynamicArray = "", Output pStatus As %Status) As %DynamicArray
+{
+  Set pStatus = 1
+  If ( '($CLASSNAME(searchTerms)="%Library.DynamicArray" ) || (searchTerms.%Size()=0) ) {
+    // empty query
+    Set searchCondition = " ( 1=1 ) "
+  } Else {
+    Set searchCondition = " ( "
+    Set iter = searchTerms.%GetIterator()
+    Set params = 0
+    While iter.%GetNext(.key , .value ) {
+      if (key=0) {
+        Set searchCondition = searchCondition _ " ((name %MATCHES ?) OR (description %MATCHES ?) OR (keywords %MATCHES ?)) "
+      } else {
+        Set searchCondition = searchCondition _ " OR ((name %MATCHES ?) OR (description %MATCHES ?) OR (keywords %MATCHES ?))"
+      }
+      Set params(params+1) = value
+      Set params(params+2) = value
+      Set params(params+3) = value
+      Set params = params + 3
+    }
+    Set searchCondition = searchCondition_ " ) "
+  }
+  Set sql = "SELECT name, repository, description "_
+            "FROM ZPM.Package p1 "_
+            "WHERE "_ searchCondition _
+            "GROUP BY name "_
+            "ORDER BY name"
+  Set tStatement = ##class(%SQL.Statement).%New()
+  Set tStatus = tStatement.%Prepare(sql)
+  If ($$$ISERR(tStatus)) { 
+    Set pStatus = tStatus
+    Return []
+  }
+  Set tResult = tStatement.%Execute(params...)
+  If (tResult.%SQLCODE<0) {
+    Set pStatus = $$$ERROR(5001, "Error executing sql statement")
+    Return []
+  }
+  Set tList = []
+  While tResult.%Next() {
+    Set tPkgInfo = {
+      "name": (tResult.name),
+      "description": (tResult.Description),
+      "repository": (tResult.repository),
+      "versions": [(..GetTopVersion(tResult.name))]
+    }
+    do tList.%Push(tPkgInfo)
   }
-  QUIT host
+  Return tList
 }
 
 Query ListLatest(searchTerm As %String = "") As %SQLQuery(ROWSPEC = "name:%String,description:%String,repository:%String,version:%String")
@@ -517,6 +624,9 @@ Storage Default
 <Value name="18">
 <Value>versionBuildmetadata</Value>
 </Value>
+<Value name="19">
+<Value>UpLink</Value>
+</Value>
 </Data>
 <Data name="dependencies">
 <Attribute>dependencies</Attribute>