diff --git a/.eslintrc.js b/.eslintrc.js new file mode 100644 index 0000000..da34396 --- /dev/null +++ b/.eslintrc.js @@ -0,0 +1,24 @@ +module.exports = { + "env": { + "commonjs": true, + "es2021": true, + "node": true + }, + "extends": [ + "plugin:jest/all", + "standard-with-typescript" + ], + "parserOptions": { + "ecmaVersion": "latest", + "sourceType": "module", + "project": [ + "./tsconfig.json" + ] + }, + "plugins": [ + "jest" + ], + "rules": { + "jest/prefer-expect-assertions": 0 + } +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd89594..fc08ffc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,10 +24,10 @@ jobs: node-version: 18.17.0 - name: Install dependencies - run: npm ci + run: npm i - name: Run ESLint - run: npm run eslint + run: npx lerna run eslint - name: Run tests - run: npm run test + run: npx lerna run test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e892766 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.idea/ +dist/ +node_modules/ +lerna-debug.log +package-lock.json diff --git a/lerna.json b/lerna.json new file mode 100644 index 0000000..f6604bd --- /dev/null +++ b/lerna.json @@ -0,0 +1,4 @@ +{ + "$schema": "node_modules/lerna/schemas/lerna-schema.json", + "version": "0.0.0" +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..d4f5803 --- /dev/null +++ b/package.json @@ -0,0 +1,23 @@ +{ + "name": "root", + "private": false, + "workspaces": [ + "packages/*" + ], + "devDependencies": { + "@azure/storage-blob": "12.15.0", + "@types/jest": "29.5.3", + "@typescript-eslint/eslint-plugin": "5.52.0", + "eslint": "8.46.0", + "eslint-config-standard-with-typescript": "37.0.0", + "eslint-plugin-import": "2.28.0", + "eslint-plugin-jest": "27.2.3", + "eslint-plugin-n": "16.0.1", + "eslint-plugin-promise": "6.1.1", + "jest": "29.6.2", + "lerna": "7.1.4", + "ts-jest": "29.1.1", + "ts-node": "10.9.1", + "typescript": "5.1.6" + } +} diff --git a/packages/core/README.md b/packages/core/README.md new file mode 100644 index 0000000..51c6ecf --- /dev/null +++ b/packages/core/README.md @@ -0,0 +1,11 @@ +# `core` + +> TODO: description + +## Usage + +``` +const core = require('core'); + +// TODO: DEMONSTRATE API +``` diff --git a/packages/core/__tests__/manifest/ManifestValidator.test.ts b/packages/core/__tests__/manifest/ManifestValidator.test.ts new file mode 100644 index 0000000..646d0b8 --- /dev/null +++ b/packages/core/__tests__/manifest/ManifestValidator.test.ts @@ -0,0 +1,85 @@ +import { ManifestValidator } from '../../src/manifest/ManifestValidator' + +describe('manifest validator test', () => { + it('should not throw exception when json is valid', () => { + // given + const manifest = { + version: '1.0', + deploy: [ + { + name: 'resources', + include: 'v1-rc6/resources/*', + headers: { + 'content-cache': 'public, max-age=2592000' + } + }, + { + name: 'main', + include: 'v1-rc6/*.js', + headers: { + 'content-encoding': 'gzip', + 'content-type': 'text/javascript', + 'content-cache': 'public, max-age=2592000' + } + }, + { + name: 'chat-loader', + include: 'chat-loader.js', + headers: { + 'content-encoding': 'gzip', + 'content-typ': 'text/javascript', + 'content-cache': 'public, max-age=300' + } + } + ] + } + + const manifestValidator = new ManifestValidator() + + // when/then + expect(() => { + manifestValidator.validate(manifest) + }).not.toThrow() + }) + + it('should throw exception when json is not valid', () => { + // given + const manifest = { + version: 'invalid-version', + deploy: [ + { + name: 'resources', + include: 'v1-rc6/resources/*', + headers: { + 'content-cache': 'public, max-age=2592000' + } + }, + { + name: 'main', + include: 'v1-rc6/*.js', + headers: { + 'content-encoding': 'gzip', + 'content-type': 'text/javascript', + 'content-cache': 'public, max-age=2592000' + } + }, + { + name: 'chat-loader', + include: 'chat-loader.js', + headers: { + 'content-encoding': 'gzip', + 'content-typ': 'text/javascript', + 'content-cache': 'public, max-age=300' + } + } + ] + } + + const manifestValidator = new ManifestValidator() + + // when/then + expect(() => { + manifestValidator.validate(manifest) + }).toThrow('Storage shipper manifest is not valid. Error: [path: \'/version\'] [message: \'must match pattern "^([1-9]\\d*)\\.(?:0|[1-9]\\d*)$"\']') + }) +}) diff --git a/packages/core/jest.config.ts b/packages/core/jest.config.ts new file mode 100644 index 0000000..75355dd --- /dev/null +++ b/packages/core/jest.config.ts @@ -0,0 +1,7 @@ +import { type JestConfigWithTsJest } from 'ts-jest' + +const config: JestConfigWithTsJest = { + preset: 'ts-jest' +} + +export default config diff --git a/packages/core/package.json b/packages/core/package.json new file mode 100644 index 0000000..b922d45 --- /dev/null +++ b/packages/core/package.json @@ -0,0 +1,33 @@ +{ + "name": "@thulium/storage-shipper", + "version": "0.0.0", + "description": "Core module for ship files to storage", + "homepage": "https://github.com/thulium/storage-shipper#readme", + "license": "MIT", + "main": "src/core.ts", + "directories": { + "lib": "src", + "test": "__tests__" + }, + "files": [ + "src" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/thulium/storage-shipper.git" + }, + "scripts": { + "build": "tsc", + "test": "jest", + "eslint": "eslint \"src/**\"" + }, + "bugs": { + "url": "https://github.com/thulium/storage-shipper/issues" + }, + "dependencies": { + "ajv": "8.12.0", + "jszip": "3.10.1", + "minimatch": "9.0.3", + "remeda": "1.24.0" + } +} diff --git a/packages/core/src/Destination.ts b/packages/core/src/Destination.ts new file mode 100644 index 0000000..fa8d71b --- /dev/null +++ b/packages/core/src/Destination.ts @@ -0,0 +1,3 @@ +export interface Destination { + container: string +} diff --git a/packages/core/src/StorageShipper.ts b/packages/core/src/StorageShipper.ts new file mode 100644 index 0000000..1e944ed --- /dev/null +++ b/packages/core/src/StorageShipper.ts @@ -0,0 +1,24 @@ +import { type StorageUploader } from './StorageUploader' +import { type Destination } from './Destination' +import { type ArtifactRepositoryFactory } from './artifact/ArtifactRepositoryFactory' +import { type Artifact } from './artifact/Artifact' + +export class StorageShipper { + constructor ( + private readonly artifactRepositoryFactory: ArtifactRepositoryFactory, + private readonly storageUploader: StorageUploader + ) { + } + + public async shipIt (artifact: Artifact, destination: Destination): Promise { + const artifactRepository = await this.artifactRepositoryFactory.create(artifact) + + const manifest = await artifactRepository.getManifest() + + const deploys = manifest.deploy + for (const deploy of deploys) { + const artifactFiles = await artifactRepository.getMatchingArtifactFiles(deploy.include) + this.storageUploader.upload(artifactFiles, destination, artifact) + } + } +} diff --git a/packages/core/src/StorageUploader.ts b/packages/core/src/StorageUploader.ts new file mode 100644 index 0000000..2d8c75c --- /dev/null +++ b/packages/core/src/StorageUploader.ts @@ -0,0 +1,6 @@ +import { type Destination } from './Destination' +import { type Artifact, type ArtifactFile } from './artifact/Artifact' + +export interface StorageUploader { + upload: (artifactFiles: ArtifactFile[], destination: Destination, artifact: Artifact) => void +} diff --git a/packages/core/src/artifact/Artifact.ts b/packages/core/src/artifact/Artifact.ts new file mode 100644 index 0000000..acbe767 --- /dev/null +++ b/packages/core/src/artifact/Artifact.ts @@ -0,0 +1,17 @@ +export interface Artifact { + parentDir?: string + path: string +} + +export interface ArtifactFile { + name: string + data: ArrayBuffer | null +} + +export function sanitizedParentDir (artifact: Artifact): string | null | undefined { + const parentDir = artifact.parentDir + if (parentDir == null) { + return parentDir + } + return parentDir.endsWith('/') ? '' : '/' +} diff --git a/packages/core/src/artifact/ArtifactRepository.ts b/packages/core/src/artifact/ArtifactRepository.ts new file mode 100644 index 0000000..caa015e --- /dev/null +++ b/packages/core/src/artifact/ArtifactRepository.ts @@ -0,0 +1,8 @@ +import { type Manifest } from '../manifest/Manifest' +import { type ArtifactFile } from './Artifact' + +export interface ArtifactRepository { + getManifest: () => Promise + + getMatchingArtifactFiles: (pattern: string) => Promise +} diff --git a/packages/core/src/artifact/ArtifactRepositoryFactory.ts b/packages/core/src/artifact/ArtifactRepositoryFactory.ts new file mode 100644 index 0000000..03832aa --- /dev/null +++ b/packages/core/src/artifact/ArtifactRepositoryFactory.ts @@ -0,0 +1,6 @@ +import { type Artifact } from './Artifact' +import { type ArtifactRepository } from './ArtifactRepository' + +export interface ArtifactRepositoryFactory { + create: (artifact: Artifact) => Promise +} diff --git a/packages/core/src/artifact/zip/ZipArtifactRepository.ts b/packages/core/src/artifact/zip/ZipArtifactRepository.ts new file mode 100644 index 0000000..8a36a7a --- /dev/null +++ b/packages/core/src/artifact/zip/ZipArtifactRepository.ts @@ -0,0 +1,63 @@ +import type JSZip from 'jszip' +import { minimatch } from 'minimatch/dist/mjs' +import { type Artifact, type ArtifactFile, sanitizedParentDir } from '../Artifact' +import { ManifestValidator } from '../../manifest/ManifestValidator' +import { type Manifest } from '../../manifest/Manifest' +import { type ArtifactRepository } from '../ArtifactRepository' +import * as R from 'remeda' + +export class ZipArtifactRepository implements ArtifactRepository { + private readonly manifestValidator: ManifestValidator = new ManifestValidator() + + constructor ( + private readonly artifact: Artifact, + private readonly jsZip: JSZip + ) { + } + + public async getManifest (): Promise { + const manifests = this.jsZip.filter(relativePath => { + return relativePath.includes('storage-shipper-manifest.json') + }) + + const manifestString = await this.getManifestString(manifests) + const manifest: Manifest = JSON.parse(manifestString) + this.manifestValidator.validate(manifest) + + return manifest + } + + public async getMatchingArtifactFiles (pattern: string): Promise { + const jsZipObjects = this.jsZip.filter(relativePath => { + const parentDir = sanitizedParentDir(this.artifact) + if (parentDir != null) { + relativePath = relativePath.replace(parentDir, '') + } + return minimatch(relativePath, pattern) + }) + + const artifactFilesPromise = R.map(jsZipObjects, async (jsZipObject): Promise => { + const parentDir = sanitizedParentDir(this.artifact) ?? '' + const data = await jsZipObject.async('arraybuffer') ?? null + return { + name: jsZipObject.name.replace(parentDir, ''), + data + } + }) + return await Promise.all(artifactFilesPromise) + } + + private async getManifestString (manifests: JSZip.JSZipObject[]): Promise { + if (manifests.length !== 1) { + throw new Error('Cannot determine correct manifest file') + } + + const manifestFile = manifests.at(0) + const manifestString = await manifestFile?.async('text') ?? null + if (manifestString === null) { + throw new Error('Cannot read manifest file') + } + + return manifestString + } +} diff --git a/packages/core/src/artifact/zip/ZipArtifactRepositoryFactory.ts b/packages/core/src/artifact/zip/ZipArtifactRepositoryFactory.ts new file mode 100644 index 0000000..313caf1 --- /dev/null +++ b/packages/core/src/artifact/zip/ZipArtifactRepositoryFactory.ts @@ -0,0 +1,22 @@ +import JSZip from 'jszip' +import { type ArtifactRepository } from '../ArtifactRepository' +import * as fs from 'fs' +import { type ArtifactRepositoryFactory } from '../ArtifactRepositoryFactory' +import { type Artifact } from '../Artifact' +import { ZipArtifactRepository } from './ZipArtifactRepository' + +export class ZipArtifactRepositoryFactory implements ArtifactRepositoryFactory { + private readonly jsZip = new JSZip() + + public async create (artifact: Artifact): Promise { + const path = artifact.path + + if (!fs.existsSync(path)) { + throw new Error(`Artifact '${path}' does not exists`) + } + + const archive = fs.readFileSync(path) + const jsZip = await this.jsZip.loadAsync(archive) + return new ZipArtifactRepository(artifact, jsZip) + } +} diff --git a/packages/core/src/core.ts b/packages/core/src/core.ts new file mode 100644 index 0000000..ba0e39f --- /dev/null +++ b/packages/core/src/core.ts @@ -0,0 +1,12 @@ +export * from './Destination' +export * from './StorageShipper' +export * from './StorageUploader' + +export * from './artifact/Artifact' +export * from './artifact/ArtifactRepositoryFactory' + +export * from './artifact/zip/ZipArtifactRepository' +export * from './artifact/zip/ZipArtifactRepositoryFactory' + +export * from './manifest/Manifest' +export * from './manifest/ManifestValidator' diff --git a/packages/core/src/manifest/Manifest.ts b/packages/core/src/manifest/Manifest.ts new file mode 100644 index 0000000..09df244 --- /dev/null +++ b/packages/core/src/manifest/Manifest.ts @@ -0,0 +1,10 @@ +export interface Manifest { + version: string + deploy: Deploy[] +} + +export interface Deploy { + name: string + include: string + headers?: object +} diff --git a/packages/core/src/manifest/ManifestValidator.ts b/packages/core/src/manifest/ManifestValidator.ts new file mode 100644 index 0000000..4847015 --- /dev/null +++ b/packages/core/src/manifest/ManifestValidator.ts @@ -0,0 +1,23 @@ +import * as schema from '../../../../schema/manifest-1.0.json' +import Ajv2020 from 'ajv/dist/2020' +import * as R from 'remeda' +import { type Manifest } from './Manifest' + +export class ManifestValidator { + private readonly ajv = new Ajv2020() + + public validate (manifest: Manifest | undefined): void { + const validateFunction = this.ajv.compile(schema) + const valid = validateFunction(manifest) + const errors = validateFunction.errors ?? [] + if (!valid) { + const errorMessagesString = R + .map(errors, (error) => { + return `[path: '${error.instancePath}'] [message: '${error.message ?? 'unknown error'}']` + }) + .join(',') + + throw new Error(`Storage shipper manifest is not valid. Error: ${errorMessagesString}`) + } + } +} diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json new file mode 100644 index 0000000..8439751 --- /dev/null +++ b/packages/core/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "./dist" + }, + "include": [ + "./src" + ] +} diff --git a/schema/manifest-1.0.json b/schema/manifest-1.0.json new file mode 100644 index 0000000..6b2c251 --- /dev/null +++ b/schema/manifest-1.0.json @@ -0,0 +1,49 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://thulium.io/schema/storage-shipper/manifest", + "title": "Manifest", + "description": "Storage sipper manifest schema", + "type": "object", + "properties": { + "version": { + "type": "string", + "description": "Version of the manifest", + "pattern": "^([1-9]\\d*)\\.(?:0|[1-9]\\d*)$" + }, + "deploy": { + "type": "array", + "description": "List of deployment rules", + "items": { + "$ref": "#/definitions/deploy-item" + } + } + }, + "required": [ + "version", + "deploy" + ], + "definitions": { + "deploy-item": { + "type": "object", + "description": "Single deploy rule", + "properties": { + "name": { + "type": "string", + "description": "Name of the deployment rule" + }, + "include": { + "type": "string", + "description": "Pattern for selecting files (minimatch)" + }, + "headers": { + "type": "object", + "description": "Name value header pair" + } + }, + "required": [ + "name", + "include" + ] + } + } +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..8f4940d --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "allowJs": false, + "allowSyntheticDefaultImports": true, + "declaration": true, + "esModuleInterop": true, + "module": "CommonJS", + "moduleResolution": "node", + "resolveJsonModule": true, + "strict": true, + "target": "ES6" + } +}