Merge pull request #5193 from Lezek123/archive_script

Colossus archive script

Author: mnaamani

.gitignore

@@ -48,3 +48,5 @@ runtime-inputs/
 devops/infrastructure
 
 joystream.tar.gz
+
+.venv

colossus.Dockerfile

@@ -30,7 +30,10 @@ RUN yarn workspace storage-node build
 RUN yarn cache clean
 
 FROM node:18 as final
+
 WORKDIR /joystream
+# 7zip and zstd are required by the archive script
+RUN apt-get update && apt-get install -y p7zip-full zstd
 COPY --from=builder /joystream /joystream
 RUN yarn --frozen-lockfile --production
 

setup.sh

@@ -8,7 +8,7 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then
     # code build tools
     sudo apt-get update -y
     sudo apt-get install -y coreutils clang llvm jq curl gcc xz-utils sudo pkg-config \
-      unzip libc6-dev make libssl-dev python3 cmake protobuf-compiler libprotobuf-dev
+      unzip libc6-dev make libssl-dev python3 cmake protobuf-compiler libprotobuf-dev p7zip-full
 
     # Docker: do not replace existing installation to avoid distrupting running containers
     if ! command -v docker &> /dev/null
@@ -23,7 +23,7 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then
     fi
     # install additional packages
     brew update
-    brew install coreutils gnu-tar jq curl llvm gnu-sed cmake protobuf || :
+    brew install coreutils gnu-tar jq curl llvm gnu-sed cmake protobuf p7zip || :
     echo "It is recommended to setup Docker desktop from: https://www.docker.com/products/docker-desktop"
     echo "It is also recommended to install qemu emulators with following command:"
     echo "docker run --privileged --rm tonistiigi/binfmt --install all"

storage-node/CHANGELOG.md

@@ -1,3 +1,7 @@
+### 4.3.0
+
+- Adds `archive` mode / command, which allows downloading, compressing and uploading all data objects to an external S3 bucket that can be used as a backup.
+
 ### 4.2.0
 
 - Fix `util:cleanup` script (call `loadDataObjectIdCache` first)

storage-node/README.md

@@ -1,14 +1,16 @@
 {
   "name": "storage-node",
   "description": "Joystream storage subsystem.",
-  "version": "4.2.0",
+  "version": "4.3.0",
   "author": "Joystream contributors",
   "bin": {
     "storage-node": "./bin/run"
   },
   "bugs": "https://github.com/Joystream/joystream/issues",
   "dependencies": {
     "@apollo/client": "^3.3.21",
+    "@aws-sdk/client-s3": "^3.675.0",
+    "@aws-sdk/s3-request-presigner": "^3.675.0",
     "@elastic/ecs-winston-format": "^1.3.1",
     "@joystream/metadata-protobuf": "^2.15.0",
     "@joystream/opentelemetry": "1.0.0",
@@ -36,6 +38,7 @@
     "await-lock": "^2.1.0",
     "base64url": "^3.0.1",
     "blake3-wasm": "^2.1.5",
+    "chokidar": "4.0.1",
     "cors": "^2.8.5",
     "cross-fetch": "^3.1.4",
     "express": "4.17.1",
@@ -52,6 +55,7 @@
     "node-cache": "^5.1.2",
     "openapi-editor": "^0.3.0",
     "promise-timeout": "^1.3.0",
+    "proper-lockfile": "^4.1.2",
     "react": "^18.2.0",
     "read-chunk": "^3.2.0",
     "rimraf": "^3.0.2",
@@ -81,6 +85,7 @@
     "@types/mocha": "^5",
     "@types/node": "^18.6.0",
     "@types/pg": "^8.6.1",
+    "@types/proper-lockfile": "^4.1.4",
     "@types/swagger-ui-express": "^4.1.2",
     "@types/ws": "^5.1.2",
     "@typescript-eslint/eslint-plugin": "3.8.0",