Macos intel build (#2446)

* remove conflicting dependencies

* intel preparation (untested)

* make orgName more flexible to test

* adjust set-version logic

* build-osx package target

* fix syntax

* fix syntax

* fix syntax

* append arch for upload

* syntax

* syntax

* fixes

* fix

* complete feature

* fix

* set-version now deleting the file instead of complaining if version-mismatch

* further improvements

* parametrize CI_PROJECT_ROOT_NAMESPACE with --gh-project

* more fixes

* tinyfix

* small fix

* fix packaging

* fix

* fix

* documentation and further polishing

* polishing

* bugfixes and feedback

* docs

* more polishing

* testrun

* kick

* kick

* kick again

* kick yet again

* kick

* kick again

* kick

* kick yet again

* fix

* kick

* kick again and again

* kick again

* kick

* kick again

* kick yet again

* remove comments for restriction

Author: k9ert

.gitlab-ci.yml

@@ -135,7 +135,7 @@ release_binary_windows:
     - .\pyinstaller\build-win-ci.bat $CI_COMMIT_TAG
     - python ./utils/github.py upload ./pyinstaller/release/specterd-$CI_COMMIT_TAG-win64.zip
     - cd ./pyinstaller/release
-    - python ..\..\utils\release-helper.py sha256sums specterd-$CI_COMMIT_TAG-win64.zip > SHA256SUMS-windows
+    - python ..\..\utils\release_helper.py sha256sums specterd-$CI_COMMIT_TAG-win64.zip > SHA256SUMS-windows
     - type SHA256SUMS-windows
     - echo $GPG_PASSPHRASE | c:\Program` Files` `(x86`)\GnuPg\bin\gpg --detach-sign --armor --no-tty --batch --yes --passphrase-fd 0  --pinentry-mode loopback SHA256SUMS-windows
   artifacts:
@@ -222,14 +222,14 @@ release_signatures:
     - pip3 install -e ".[test]"
     - ./utils/artifact_signer.sh init # prepare .gnupg
   script:
-    - python3 -m utils.release-helper download          # downloads the job-artifacts from gitlab
-    - python3 -m utils.release-helper downloadgithub    # downloads additional artifacts from github (if not there and is they have SHA256SUMS-something)
-    - python3 -m utils.release-helper checksigs         # checks the signatures of all SHA256SUMM*.asc files
-    - python3 -m utils.release-helper checkhashes       # checks all SHA256SUM* files (might modify files on the fly due to windows line endings)
-    - python3 -m utils.release-helper create            # creates a SHA256SUM
+    - python3 -m utils.release_helper download          # downloads the job-artifacts from gitlab
+    - python3 -m utils.release_helper downloadgithub    # downloads additional artifacts from github (if not there and is they have SHA256SUMS-something)
+    - python3 -m utils.release_helper checksigs         # checks the signatures of all SHA256SUMM*.asc files
+    - python3 -m utils.release_helper checkhashes       # checks all SHA256SUM* files (might modify files on the fly due to windows line endings)
+    - python3 -m utils.release_helper create            # creates a SHA256SUM
     - ./utils/artifact_signer.sh sign --artifact ./signing_dir/SHA256SUMS # Signs the SHA256SUM
-    - python3 -m utils.release-helper upload_shasums    # uploads SHA256SUMS to github
-    - python3 -m utils.release-helper upload_shasumssig # uploads SHA256SUMS.asc to github
+    - python3 -m utils.release_helper upload_shasums    # uploads SHA256SUMS to github
+    - python3 -m utils.release_helper upload_shasumssig # uploads SHA256SUMS.asc to github
 
 release_docker:
   stage: post_releasing
@@ -247,46 +247,31 @@ tag_specterext_dummy_repo:
   only:
     - tags
   before_script:
-    ## Install ssh-agent if not already installed, it is required by Docker.
-    ## (change apt-get to yum if you use an RPM-based image)
-    ##
-    - 'which ssh-agent || ( apk update && apk add --no-cache bash git openssh )'
-    - docker info
-    
-    ##
-    ## Run ssh-agent (inside the build environment)
-    ##
-    - eval $(ssh-agent -s)
-
-    ##
-    ## Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
-    ## We're using tr to fix line endings which makes ed25519 keys work
-    ## without extra base64 encoding.
-    ## https://gitlab.com/gitlab-examples/ssh-private-key/issues/1#note_48526556
-    ##
-    - echo "$SSH_SPECTEREXT_DEPLOY_KEY" | tr -d '\r' | ssh-add - > /dev/null
-
-    ##
-    ## Create the SSH directory and give it the right permissions
-    ##
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-
-    ##
-    ## Optionally, if you will be using any Git commands, set the user name and
-    ## and email.
-    ##
-    - git config --global user.email "specter@secretvalues"
-    - git config --global user.name "specter"
-    
-    ##
-    ## Assuming you created the SSH_KNOWN_HOSTS variable, uncomment the
-    ## following two lines.
-    ##
-    - echo "$KNOWN_HOSTS" > ~/.ssh/known_hosts
-    - chmod 644 ~/.ssh/known_hosts
-
+    # write access to [email protected]:cryptoadvance/specterext-dummy.git
+    - source ./utils/prepare_for_git_write.sh "$SSH_SPECTEREXT_DEPLOY_KEY"
   script:
     - echo "Now tagging ... [email protected]:${CI_PROJECT_ROOT_NAMESPACE}/specterext-dummy.git"
     - ./utils/tag_specterext_dummy.sh
 
+update_github:
+  stage: post_releasing
+  only:
+    - tags
+  before_script:
+    # write access to [email protected]:swan-bitcoin/specter-static.git
+    - source ./utils/prepare_for_git_write.sh "$SSH_SPECTERSTATIC_DEPLOY_KEY"
+  script:
+    - echo "Now updating https://github.com:${CI_PROJECT_ROOT_NAMESPACE}/specter-desktop/releases/tag/${CI_COMMIT_TAG:-v2.0.4-pre8}"
+    - ./utils/generate_downloadpage.sh --org_name ${CI_PROJECT_ROOT_NAMESPACE:-k9ert} --debug --version ${CI_COMMIT_TAG:-v2.0.4-pre8} generate github # default-value for testing
+
+update_webpage:
+  stage: post_releasing
+  only:
+    - tags
+  before_script:
+    # write access to [email protected]:swan-bitcoin/specter-static.git
+    - source ./utils/prepare_for_git_write.sh "$SSH_SPECTERSTATIC_DEPLOY_KEY"
+  script:
+    - echo "Now updating https://github.com:${CI_PROJECT_ROOT_NAMESPACE}/specter-static.git"
+    - ./utils/generate_downloadpage.sh --org_name ${CI_PROJECT_ROOT_NAMESPACE:-k9ert} --debug --version ${CI_COMMIT_TAG:-v2.0.4-pre8} generate webpage # default-value for testing
+

docs/release-guide.md

@@ -46,6 +46,12 @@ Update your master branch after the release notes PR ([example](https://github.c
 git tag v1.13.1 && git push upstream v1.13.1
 ```
 
+if you have a proper setup of github- AND gitlab forks (and the remote `origin` on git pointing to your github fork), you can push that tag to origin and this is useful to test the procedures. See "CI/CD-dev-env setup" in [continuous-integration](./continuous-integration.md).
+
+```bash
+git tag v1.13.1 && git push origin v1.13.1
+```
+
 ## GitLab - releasing stage
 
 Creating a tag triggers the release process of the GitLab runners.
@@ -73,30 +79,61 @@ For details look at `.gitlab-ci.yml`
 
 ## MacOS
 
-Ideally, directly after the tag is created, start with the MacOS release. This has to be done manually, for now. There is a script for this:
+Ideally, directly after the tag is created, start with the MacOS release. As the binaries of x86/arm64 are not compatible with each other, we need to build on two MacOS architectures.This has to be done manually, for now. There is a script for this. Start with the build on x86:
+
+### MacOS x64 build
+
+```bash
+./utils/build-osx.sh  --version v2.0.5-pre4 specterd package upload
+```
+
+You can also test this procedure without messing the original project via changing the `orgName` to your `orgName` in `pyinstaller/electron/downloadloc.js`.
+
+This will create three artifacts on github:
+* specterd-v2.0.5-pre4-osx_x64.zip
+* SHA256SUMS-macos_x64
+* SHA256SUMS-macos_x64.asc
+
+### MacOS arm64 build
+
+The electron application will get built on the arm architecture. As it needs to store the sha256 hash in the electron-app, the make-hash target
+will not only hash the specterd but also download the other specterd and hash it.
 
 ```bash
-./utils/build-osx.sh  --version v1.13.1 --appleid "Satoshi Nakamoto (appleid)" --mail "[email protected]" make-hash specterd electron sign upload
+./utils/build-osx.sh  --version v2.0.5-pre4 --appleid "Satoshi Nakamoto (appleid)" --mail "[email protected]" specterd make-hash electron sign package upload
 ```
 
-This script also runs `github.py upload `, so two more binares and the hash and signature files are uploaded to GitHub:
+This will create four artifacts on github:
+* Specter-v2.0.5-pre4.dmg
+* specterd-v2.0.5-pre4-osx_arm64.zip
+* SHA256SUMS-macos_arm64
+* SHA256SUMS-macos_arm64.asc
 
-- Specter-v1.13.1.dmg
-- specterd-v1.13.1-osx.zip
-- SHA256SUMS-macos
-- SHA256SUMS-macos.asc
 
 ## GitLab - post releasing
 
 Back to GitLab, the final stage is "post releasing".
 
-In this stage, the invididual SHA256-hashes and signatures are combined into two final files:
+### release_signatures
+
+In this job, the individual SHA256-hashes and signatures are combined into two final files:
 
 - SHA256SUMS
 - SHA256SUMS.asc
 
 Everything, apart from the MacOS files, are pulled from the GitLab environment, the MacOS files from GitHub.
-Don't forget to delete the two MacOS files (`SHA256SUMS-macos` and `SHA256SUMS-macos.asc`) on the GitHub release page in the end.
+Don't forget to delete the four MacOS files (`SHA256SUMS-macos_arm64` and `SHA256SUMS-macos_arm64.asc` and the two corresponding `_x64` files) on the GitHub release page in the end.
+
+This is difficult to automate as sometimes the manual steps has not succeeded while generating the SHASUM-files. As a result, those hashes are not included. So you might want to run this again. And you can, just delete the two generated files - `SHA256SUMS` and `SHA256SUMS.asc` and run the job again.
+
+### release_docker
+
+There are docker images created by the awesome [Chiang Mai LN dev](https://github.com/lncm/docker-specter-desktop). So the task of this job is to trigger their build-system which is done via `utils/trigger_docker_build.sh`. A prerequisite of this is a token in order to authenticate. That token is from Aaron, one of the maintainers of that repo, and can be found in the gitlab variables section of the CI/CD configuration.
+
+### tag_specterext_dummy_repo
+
+Sometimes there are changes on the plugin architecture. In order to create a plugin, it's quite important to know which version of the plugin system should be used. Because of that, we simply assume that the master of the [specterext-dummy](https://github.com/cryptoadvance/specterext-dummy) repo is compatible with the current master which was just tagged with the new version.
+So this job will tag that repo with the same tag and the creation of a plugin will take the version into account.
 
 ## Trouble shooting
 

pyinstaller/build-win-ci.bat

@@ -19,7 +19,7 @@ pip3 install build==0.10.0
 python -m build
 
 echo "    --> Installing pypi package"
-python .\utils\release-helper.py install_wheel %1%
+python .\utils\release_helper.py install_wheel %1%
 
 cd pyinstaller
 

pyinstaller/electron/downloadloc.js

@@ -1,14 +1,23 @@
 
+function orgName() {
+    // This can be changed in order to make download possible from other github orgs
+    return "cryptoadvance"
+}
+
 function getDownloadLocation(version, platformname) {
-    return `https://github.com/cryptoadvance/specter-desktop/releases/download/${version}/specterd-${version}-${platformname}.zip`
+    if (platformname != "osx") {
+        return `https://github.com/${orgName()}/specter-desktop/releases/download/${version}/specterd-${version}-${platformname}.zip`
+    }
+    return `https://github.com/${orgName()}/specter-desktop/releases/download/${version}/specterd-${version}-${platformname}_${process.arch}.zip`
 }
 
 function appName() {
     return "Specter"
 }
 
 module.exports = {
-    getDownloadLocation: getDownloadLocation, 
-    appName: appName
+    getDownloadLocation, 
+    appName,
+    orgName
 }
 

pyinstaller/electron/main.js

@@ -146,7 +146,7 @@ app.whenReady().then(() => {
   if (!appSettings.versionInitialized || appSettings.versionInitialized != versionData.version) {
     logger.info(`Updating ${appSettingsPath} : ${JSON.stringify(appSettings)}`)
     appSettings.specterdVersion = versionData.version
-    appSettings.specterdHash = versionData.sha256
+    appSettings.specterdHash = versionData.sha256[process.arch]
     appSettings.versionInitialized = versionData.version
     fs.writeFileSync(appSettingsPath, JSON.stringify(appSettings))
   }

pyinstaller/electron/set-version.js

@@ -1,25 +1,60 @@
-const fs = require('fs')
-const crypto = require('crypto')
-const version = process.argv[2]
+const fs = require('fs');
+const crypto = require('crypto');
+const versionDataFile = './version-data.json';
 
 async function setVersion() {
-    let package = require('./package.json')
-    package.version = version
-    fs.writeFileSync('./package.json', JSON.stringify(package, undefined, 2))
-    
-    if (process.argv[3]) {
-        let versionData = {
-            version,
-            sha256: (await createHashFromFile(process.argv[3]))
-        }
-    
-        fs.writeFileSync('./version-data.json', JSON.stringify(versionData, undefined, 2))
-    }    
+  const version = process.argv[2];
+  const file = process.argv[3];
+  const arch = process.argv[4] || process.arch;
+
+  // Set version in package.json
+  let packageJson = require('./package.json');
+  packageJson.version = version;
+  fs.writeFileSync('./package.json', JSON.stringify(packageJson, undefined, 2));
+
+  // Set version in version-data.json
+  if (version && file) {
+    let versionData;
+    try {
+      versionData = require(versionDataFile);
+      
+      if (versionData.version != version) {
+        console.log(`Version mismatch. Deleting ${versionDataFile} and creating anew.`);
+        fs.unlinkSync(versionDataFile); // Delete the existing version-data.json file
+        versionData = createNewVersionData(version); // Create new version data object
+      }
+      
+    } catch (error) {
+      console.log(`No ${versionDataFile} found. Creating anew.`);
+      versionData = createNewVersionData(version);
+    }
+    // Compute SHA256 hash of the provided file
+    versionData.sha256[arch] = await createHashFromFile(file);
+    // Write new version data to file
+    fs.writeFileSync(versionDataFile, JSON.stringify(versionData, undefined, 2));
+    console.log("version-data.js: ")
+    console.log("----------------------------------------------------------")
+    console.log(versionData);
+    console.log("----------------------------------------------------------")
+  } else if (arch || file) {
+    throw new Error("Declare both arch and file or none.");
+  }
+}
+
+function createNewVersionData(version) {
+  // Return a new version data object
+  return {
+    version,
+    sha256: {}
+  };
 }
 
-const createHashFromFile = filePath => new Promise(resolve => {
-    const hash = crypto.createHash('sha256');
-    fs.createReadStream(filePath).on('data', data => hash.update(data)).on('end', () => resolve(hash.digest('hex')));
+const createHashFromFile = filePath => new Promise((resolve, reject) => {
+  const hash = crypto.createHash('sha256');
+  fs.createReadStream(filePath)
+    .on('data', data => hash.update(data))
+    .on('end', () => resolve(hash.digest('hex')))
+    .on('error', reject);
 });
 
-setVersion()
+setVersion().catch(console.error);

pyinstaller/electron/src/config.js

@@ -44,7 +44,7 @@ function getAppSettings() {
     tor: false,
     proxyURL: "socks5://127.0.0.1:9050",
     specterdVersion: (versionData && versionData.version !== undefined) ? versionData.version : 'unknown',
-    specterdHash: (versionData && versionData.sha256 !== undefined) ? versionData.sha256 : 'unknown',
+    specterdHash: (versionData && versionData.sha256 !== undefined) ? versionData.sha256[process.arch] : 'unknown',
     specterdCLIArgs: "",
     versionInitialized: false
   }

pyinstaller/electron/src/download.js

@@ -3,7 +3,7 @@ const fs = require('fs')
 const { app, Menu } = require('electron')
 const extract = require('extract-zip')
 const { getDownloadLocation } = require('../downloadloc.js')
-const { appName, appSettings, platformName, appNameLower, versionData, versionDataPath } = require('./config.js')
+const { appName, appSettings, platformName, appNameLower, versionDataPath } = require('./config.js')
 const { isMac, getFileHash } = require('./helpers.js')
 const { logger } = require('./logging.js')
 const ProgressBar = require('electron-progressbar')

pyinstaller/requirements.in

@@ -2,5 +2,3 @@ pyinstaller==5.2
 pefile==2022.5.30
 macholib
 pywin32-ctypes
-babel==2.12.1
-pytz==2022.1

pyinstaller/requirements.txt

@@ -10,10 +10,6 @@ altgraph==0.17 \
     # via
     #   macholib
     #   pyinstaller
-babel==2.12.1 \
-    --hash=sha256:b4246fb7677d3b98f501a39d43396d3cafdc8eadb045f4a31be01863f655c610 \
-    --hash=sha256:cc2d99999cd01d44420ae725a21c9e3711b3aadc7976d6147f622d8581963455
-    # via -r requirements.in
 future==0.18.2 \
     --hash=sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d
     # via pefile
@@ -43,10 +39,6 @@ pyinstaller-hooks-contrib==2022.8 \
     --hash=sha256:c4210fc50282c9c6a918e485e0bfae9405592390508e3be9fde19acc2213da56 \
     --hash=sha256:e46f099934dd4577fb1ddcf37a99fa04027c92f8f5291c8802f326345988d001
     # via pyinstaller
-pytz==2022.1 \
-    --hash=sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7 \
-    --hash=sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c
-    # via -r requirements.in
 pywin32-ctypes==0.2.0 \
     --hash=sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942 \
     --hash=sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98