Merge pull request #255 from SwissOpenEM/253-tryout-create-choco-package-hosted-on-github

phwissmann · web-flow · commit aad1f7ee4220 · 2026-02-09T16:15:24.000+01:00
253 tryout create choco package hosted on GitHub
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -8,6 +8,12 @@ on:
 
 permissions:
   contents: write
+  packages: write
+
+env:
+  NUGET_SOURCE_URL: "https://nuget.pkg.github.com/swissopenem/index.json" 
+  # For deploying within ETHZ w/o authorization that is required for public github access
+  GITLAB_SOURCE_URL: "https://sissource.ethz.ch/api/v4/projects/1201/packages/nuget/index.json"
 
 jobs:
   goreleaser:
@@ -30,6 +36,29 @@ jobs:
           args: release --clean
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up .NET
+        uses: actions/setup-dotnet@v2
+        with:
+          dotnet-version: 10.x.x 
+
+      # AI Generated step to parse the version
+      - name: Extract version from Git tag
+        id: extract_version
+        run: |
+          VERSION=${GITHUB_REF##*/} 
+          echo "VERSION=${VERSION/v/}" >> $GITHUB_ENV
+
+      - name: Create NuGet package
+        run: |
+          nuget pack openem-ingestor.nuspec -version ${{env.VERSION}} 
+
+      - name: Publish NuGet package to GitHub
+        run: |
+          nuget push *.nupkg -Source "${{env.NUGET_SOURCE_URL}}" -ApiKey ${{ secrets.GITHUB_TOKEN }}
+          nuget push *.nupkg -Source "${{env.GITLAB_SOURCE_URL}}" -ApiKey ${{ secrets.SISSOURCE_GITLAB_NUGET_REPO }} 
+
+
   build_docker_image_release:
     permissions:
       packages: write
diff --git a/.gitignore b/.gitignore
@@ -30,4 +30,5 @@ go.work.sum
 dist/**
 
 # backup files
-*.go.bak
+*.go.bak
+*.nupkg
diff --git a/.goreleaser.yaml b/.goreleaser.yaml
@@ -71,7 +71,7 @@ builds:
     flags:
       - -trimpath
     ldflags:
-      - "-s -w  -X 'main.version={{.Version}}'"
+      - "-s -w  -X 'main.version={{.Version}}' "
     env:
       - CGO_ENABLED=0
     dir: ./cmd/openem-ingestor-service
@@ -126,12 +126,10 @@ archives:
       {{- else }}{{ .Arch }}{{ end }}
       {{- if .Arm }}v{{ .Arm }}{{ end }}
     files:
-    - src: configs/ReadMe.md
-      dst: ./ReadMe.md
+    - src: ./README.md
+      dst: ./README.md
     - src: configs/*.yaml
       dst: ./
-    - src: configs/schemas/*.json
-      dst: schemas
       
 changelog:
   sort: asc
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
@@ -36,7 +36,7 @@
       "command": "go",
       "args": [
         "generate",
-        "./internal/*"
+        "./internal/..."
       ]
     },
     {
diff --git a/README.md b/README.md
@@ -56,6 +56,30 @@ The core package contains main functionality. It makes use of the [scicat-cli to
 - WebServer: [docs/authentication.md](docs/webserver.md)
 - Keycloak Setup for development: [docs/keycloak-setup.md](docs/keycloak-setup.md)
 
-## Deployment
+## Linux Deployment
 
 For deployment instruction and example setup see [openem-deployment](https://github.com/SwissOpenEM/openem-deployment) repository.
+
+## Windows Deployment
+
+### Executable
+
+The executable can be download directly from the [releases page](https://github.com/SwissOpenEM/Ingestor/releases) and executed. Alternatively,
+a [Chocolatey](https://docs.chocolatey.org/en-us/) package can be downloaded which installs the ingestor as a Windows service.
+
+### Windows Service Deployment
+
+1. Install Shawl <https://github.com/mtkennerly/shawl> with `machine` scope to run the ingestor as a Windows services
+        ```pwsh
+        winget install --scope "machine" -e --id mtkennerly.shawl
+        ```
+
+2. Install the ingestor using `chocolatey`
+
+    ```pwsh
+    choco install openem-ingestor --source=https://nuget.pkg.github.com/swissopenem/index.json --params="'/Scicat.Host=\"https://dacat.psi.ch/api/v3\"'" -y
+    ```
+
+> **Note**: You will be prompted for a Github username and password as Github does not allow for unauthenticated downloads.
+
+3. Verify the ingestor is up and running by entering `http://localhost:8888/version` in a browser. The install version should appear.
diff --git a/configs/openem-ingestor-config-template.yaml b/configs/openem-ingestor-config-template.yaml
@@ -0,0 +1,76 @@
+Scicat:
+  Host: ${SCICAT_HOST}/api/v3
+Transfer:
+  StorageLocation: "ETHZ"
+  Method: S3
+  S3:
+    Endpoint: ${S3_HOST}/archiver/api/v1
+    TokenUrl: ${KEYCLOAK_HOST}/realms/awi/protocol/openid-connect/token
+    ClientID: scopem-archiver-service-api
+    ChunkSizeMB: 128
+    ConcurrentFiles: 8
+    PoolSize: 4
+
+MetadataExtractors:
+  InstallationPath: C:\Users\Public\Documents\OpenEM\extractors
+  SchemasLocation: C:\Users\Public\Documents\OpenEM\schemas
+  DownloadMissingExtractors: true
+  Timeout: 20m
+  Extractors:
+  - Name: LS
+    GithubOrg: osc-em
+    GithubProject: oscem-extractor-life
+    Version: v2.0.2
+    Executable: oscem-extractor-life.exe
+    Checksum: 0f7f2ef8ec641795ef322c765a504fd6a5a287e70071a79f42f5f73a7c892b1f
+    ChecksumAlg: sha256
+    CommandLineTemplate: "-i '{{.SourceFolder}}' -o '{{.OutputFile}}' --cs 2.7 --folder_filter=^.*(?i)(metadata).*$"
+    Methods:
+    - Name: Single Particle
+      Schema: oscem_schemas_spa.schema.json
+      Url: https://w3id.org/oscem-schemas/latest/spa/jsonschema/oscem_schemas_spa.schema.json
+    - Name: Cellular Tomography
+      Schema: oscem_cellular_tomo.json
+      Url: https://w3id.org/oscem-schemas/latest/cellular_tomo/jsonschema/oscem_schemas_cellular_tomo.schema.json
+    - Name: Tomography
+      Schema: oscem_tomo.json
+      Url: https://w3id.org/oscem-schemas/latest/subtomo/jsonschema/oscem_schemas_subtomo.schema.json
+    - Name: EnvironmentalTomography
+      Schema: oscem_env_tomo.json
+      Url: https://w3id.org/oscem-schemas/latest/env_tomo/jsonschema/oscem_schemas_env_tomo.schema.json
+WebServer:
+  Auth:
+    Disable: true
+    SessionDuration: 28800
+    Frontend:
+      Origin: ${FRONTEND_HOST}
+      RedirectPath: "/ingestor"
+    OAuth2:
+      ClientID: "ingestor"
+      RedirectURL: "http://localhost:8888/callback"
+      Scopes:
+        - email
+    OIDC:
+     IssuerURL: "${KEYCLOAK_HOST}/realms/awi"
+    JWT:
+      UseJWKS: true
+      JwksURL: "${KEYCLOAK_HOST}/realms/awi/protocol/openid-connect/certs"
+      JwksSignatureMethods:
+        - RS256
+    RBAC:
+      AdminRole: "ingestor-admin-scopem"
+      CreateModifyTasksRole: "ingestor-write-scopem"
+      ViewTasksRole: "ingestor-read-scopem"
+  Paths:
+    CollectionLocations:
+        "${COLLECTION_LOCATION1}": "${COLLECTION_LOCATION1_PATH}" 
+        "${COLLECTION_LOCATION2}": "${COLLECTION_LOCATION2_PATH}" 
+        "${COLLECTION_LOCATION3}": "${COLLECTION_LOCATION3_PATH}" 
+  MetadataExtJobs:
+    ConcurrencyLimit: 10
+    QueueSize: 200
+  Other:
+    Port: 8888
+    LogLevel: Info
+    DisableServiceAccountCheck: true
+    BackendAddress: http://localhost:8888
diff --git a/internal/metadataextractor/extractor.go b/internal/metadataextractor/extractor.go
@@ -15,6 +15,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"regexp"
 	"runtime"
 	"sort"
@@ -106,7 +107,7 @@ func NewExtractorHandler(config ExtractorsConfig) *ExtractorHandler {
 		if config.DownloadMissingExtractors {
 			err := downloadExtractor(fullInstallPath, extractorConfig)
 			if err != nil {
-				log().Error("Failed to download extractor", "name", extractorConfig.Name)
+				log().Error("Failed to download extractor", "name", extractorConfig.Name, "error", err.Error())
 				continue
 			}
 		}
@@ -307,8 +308,8 @@ func downloadExtractor(fullInstallPath string, config ExtractorConfig) error {
 			return err
 		}
 		x := &xtractr.XFile{
-			FilePath:  path.Clean(file),
-			OutputDir: path.Dir(fullInstallPath),
+			FilePath:  filepath.Clean(file),
+			OutputDir: filepath.Clean(path.Dir(fullInstallPath)),
 		}
 
 		size, files, _, err := x.Extract()
diff --git a/internal/webserver/webserver.go b/internal/webserver/webserver.go
@@ -23,9 +23,6 @@ import (
 	"github.com/gin-contrib/slog"
 )
 
-// Copy the openapi specs to local folder so it can be embedded in order to statically serve it
-//go:generate cp ../../api/openapi.yaml ./openapi.yaml
-
 //go:embed openapi.yaml
 var swaggerYAML embed.FS
 
diff --git a/internal/webserver/webserver_linux.go b/internal/webserver/webserver_linux.go
@@ -0,0 +1,6 @@
+//go:build linux
+
+package webserver
+
+// Copy the openapi specs to local folder so it can be embedded in order to statically serve it
+//go:generate cp ../../api/openapi.yaml ./openapi.yaml
diff --git a/internal/webserver/webserver_win.go b/internal/webserver/webserver_win.go
@@ -0,0 +1,6 @@
+//go:build windows
+
+package webserver
+
+// Copy the openapi specs to local folder so it can be embedded in order to statically serve it
+//go:generate cmd.exe /c copy ..\..\api\openapi.yaml .\openapi.yaml
diff --git a/openem-ingestor-service b/openem-ingestor-service
diff --git a/openem-ingestor.nuspec b/openem-ingestor.nuspec
@@ -0,0 +1,44 @@
+<?xml version="1.0"?>
+<package>
+  <metadata>
+    <id>openem-ingestor</id>
+    <title>Openem Ingestor</title>
+    <version>9.9.9</version>
+    <authors>OpenEm</authors>
+    <owners>OpenEm</owners>
+    <projectUrl>https://github.com/SwissOpenEM/Ingestor</projectUrl>
+    <!-- <licenseUrl>https://github.com/SwissOpenEM/Ingestor/LICENSE</licenseUrl> -->
+    <summary>This repository provides a data ingestion service for dataset transfer and metadata registration in a catalog. It targets Scicat as dataset catalog.</summary>
+    <requireLicenseAcceptance>false</requireLicenseAcceptance>
+    <description>
+OpenEM Ingestor Package
+This package installs the ingestor as a windows network service. The config file is templated using the package parameters and saved
+at "$Env:ChocolateyInstall\lib\openem-ingestor-config.yaml", i.e. next to the binary. Any changes need to be done there.
+The service name "OpenEM-Ingestor" and needs to be stopped before making changes to the config file.
+
+The service depends on https://github.com/mtkennerly/shawl. Install with winget:
+
+`winget install --scope "machine" -e --id mtkennerly.shawl`
+
+The log file can be found in the directory of the shawl executable; using powershell this can be found by `get-command shawl`.
+
+#### Package Parameters
+The following package parameters can be set:
+
+  * `/Scicat.Host:` - Scicat backend URL - defaults to: "https://dacat.psi.ch"
+  * `/Frontend.Host:` - Scicat Frontend URL - defaults to "https://discovery.psi.ch"
+  * `/Keycloak.Host` - Keycloak URL - defaults to "https://kc.psi.ch"
+  * `/S3.Host:` - S3 host URL - defaults to "https://scopem-openem.ethz.ch"
+  * `/CollectionLocations:` List of collection location name and path - example --params="'/CollectionLocations:storage1:\\\\path\\to\\storage1;storage2:\\\\path\\to\\storage2'"
+
+To pass parameters, use `--params "''"` (e.g. `choco install openem-ingestor [other options] --params="'/ITEM:value /ITEM2:value2 /FLAG_BOOLEAN'"`).
+To have choco remember parameters on upgrade, be sure to set `choco feature enable -n=useRememberedArgumentsForUpgrades`.
+    </description>
+    <tags>swissopenem</tags>
+  </metadata>
+  <files>
+    <file src="tools\*.ps1" target="tools/"/>
+    <file src="configs\openem-ingestor-config-template.yaml" target="./"/>
+    <file src="dist/openem-ingestor-service_windows_amd64_v1/openem-ingestor-service.exe" target="./"/>
+  </files>
+</package>
diff --git a/tools/chocolateyInstall.ps1 b/tools/chocolateyInstall.ps1
@@ -0,0 +1,91 @@
+$ErrorActionPreference = 'Stop';
+
+$package = Get-Package -Name 'Openem-Ingestor' -ErrorAction SilentlyContinue
+
+
+
+if ($package) {
+    Write-Host "Package already installed. Uninstalling first."
+    choco uninstall openem-ingestor -y
+}
+
+$serviceName = "OpenEM-Ingestor"
+try {
+    $service = Get-Service -Name $serviceName -ErrorAction Stop
+    Write-Host "Service '$serviceName' exists."
+    Stop-Service -Name $serviceName
+
+    sc.exe delete $serviceName
+} catch {}
+
+
+$packageName = 'openem-ingestor'
+
+
+$pp = Get-PackageParameters
+
+if (!$pp['Scicat.Host']) { $pp['Scicat.Host'] = 'https://dacat.psi.ch' }
+if (!$pp['Frontend.Host']) { $pp['Frontend.Host'] = 'https://discovery.psi.ch' }
+if (!$pp['Keycloak.Host']) { $pp['Keycloak.Host'] = 'https://kc.psi.ch' }
+if (!$pp['S3.Host']) { $pp['S3.Host'] = 'https://scopem-openem.ethz.ch' }
+
+# Copy the executable
+
+# Define the string to find and the string to replace it with
+$parameters = @{}; 
+$parameters["SCICAT_HOST"] = $pp['Scicat.Host']
+$parameters["FRONTEND_HOST"] = $pp['Frontend.Host']
+$parameters["KEYCLOAK_HOST"] = $pp['Keycloak.Host']
+$parameters["S3_HOST"] = $pp['S3.Host']
+
+$locationPairs = $pp['CollectionLocations'] -split ';'
+for ($index = 0; $index -lt $locationPairs.Length; $index++) {
+    $pair = $locationPairs[$index] -split ':'
+    Write-Host "Adding collection Location: $($pair[0]): $($pair[1])"
+    $parameters["COLLECTION_LOCATION$($index + 1)"] = $pair[0]
+    $parameters["COLLECTION_LOCATION$($index + 1)_PATH"] = $pair[1]
+}
+
+
+$extractPath = "$Env:ChocolateyInstall\lib\$packageName"
+$binaryPath = "$extractPath\openem-ingestor-service.exe"
+
+$yamlFilePath = "$extractPath\openem-ingestor-config-template.yaml"
+$configFilePath = "$extractPath\openem-ingestor-config.yaml"
+
+Write-Host "Writing config file $configFilePath"
+$yamlContent = Get-Content -Path $yamlFilePath -Raw
+foreach ($key in $parameters.Keys) { 
+    $escapedKey = "\$\{$key\}"
+   $yamlContent = $yamlContent -replace $escapedKey, $parameters[$key]
+}; 
+
+# # Save the updated content back to the YAML file
+Set-Content -Path $configFilePath -Value $yamlContent
+
+Write-Host "Installing $packageName as a service."
+# Prompt for the password
+$password = Read-Host -Prompt "Enter admin password:" -AsSecureString
+
+# Convert the password to a credential object using the NETWORK SERVICE account
+$credential = New-Object System.Management.Automation.PSCredential("NT AUTHORITY\NETWORK SERVICE", $password)
+
+$shawlPath = (get-command shawl).path
+$shawlBinPath = "`"$shawlPath`" run --name `"$serviceName`" --cwd `"$extractPath`" -- `"$binarypath`""
+
+
+New-Service -Name $serviceName -DisplayName $serviceName -BinaryPathName $shawlBinPath -StartupType Automatic -Credential $credential
+$target = (get-item (Get-Item $shawlPath).Target).Directory.FullName
+
+Write-Host "Add firewall settings for $($binaryPath)"
+icacls $target /grant "NT AUTHORITY\NETWORK SERVICE:(OI)(CI)F" /T
+icacls $binaryPath /grant "NT AUTHORITY\NETWORK SERVICE:(OI)(CI)F" /T
+
+sc.exe config $serviceName obj="NT AUTHORITY\Network Service"
+
+Write-Host "Starting Service"
+Start-Service $serviceName
+
+Write-Host (Get-Service $serviceName)
+
+Write-Host "openem-ingestor installed successfully!"
diff --git a/tools/chocolateyUninstall.ps1 b/tools/chocolateyUninstall.ps1
@@ -0,0 +1,10 @@
+$ErrorActionPreference = 'Stop';
+
+$package = Get-Package -Name 'Openem-Ingestor' -ErrorAction SilentlyContinue
+
+$serviceName = "OpenEM-Ingestor"
+
+Stop-Service -Name $serviceName
+
+sc.exe delete $serviceName
+
diff --git a/tools/nuspec.yaml b/tools/nuspec.yaml

Original file line number	Diff line number	Diff line change
`@@ -36,7 +36,7 @@`
`36`	`36`	`"command": "go",`
`37`	`37`	`"args": [`
`38`	`38`	`"generate",`
`39`		`- "./internal/*"`
	`39`	`+ "./internal/..."`
`40`	`40`	`]`
`41`	`41`	`},`
`42`	`42`	`{`
Original file line number	Diff line number	Diff line change
`@@ -23,9 +23,6 @@ import (`
`23`	`23`	`"github.com/gin-contrib/slog"`
`24`	`24`	`)`
`25`	`25`
`26`		`-// Copy the openapi specs to local folder so it can be embedded in order to statically serve it`
`27`		`-//go:generate cp ../../api/openapi.yaml ./openapi.yaml`
`28`		`-`
`29`	`26`	`//go:embed openapi.yaml`
`30`	`27`	`var swaggerYAML embed.FS`
`31`	`28`