Merge pull request #134 from VeloraDEX/feature/DVOPS-310

DVOPS-310: Secrets detection pre-commit

Author: ovceev

.github/workflows/ci.yaml

@@ -0,0 +1,41 @@
+name: CI
+on: push
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: ['22.x']
+    name: Node ${{ matrix.node }} sample
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+
+      - name: Install Dependencies
+        run: yarn
+
+      - name: Run Trivy vulnerability scanner in fs mode
+        uses: aquasecurity/trivy-action@0.29.0
+        with:
+          scan-type: 'fs'
+          scan-ref: '.'
+          ignore-unfixed: true
+          severity: 'CRITICAL'
+          exit-code: 0
+          hide-progress: true
+          scanners: vuln,secret,misconfig
+          output: 'trivy-scan-results'
+
+      - name: Upload Trivy report as a Github artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: trivy-report
+          path: '${{ github.workspace }}/trivy-scan-results'
+          retention-days: 7

.gitignore

@@ -7,7 +7,11 @@ yarn-error.log
 
 dist
 
-.env
+.env*
+!.env.example
+!.env.sample
+
+.husky/.git-secrets-installed
 
 TODO
 

.husky/pre-commit

@@ -0,0 +1,25 @@
+if ! command -v git-secrets &> /dev/null
+then
+    echo "git-secrets is not installed. Please run 'brew install git-secrets' or visit https://github.com/awslabs/git-secrets#installing-git-secrets"
+    exit 1
+fi
+
+
+HOOK_FLAG=".husky/.git-secrets-installed"
+if [[ ! -f "$HOOK_FLAG" ]]; then
+    git-secrets --register-aws > /dev/null
+    git secrets --add -- 'ghp_[A-Za-z0-9_]\{36\}'
+    git secrets --add -- 'github_pat_[A-Za-z0-9_]\{36\}'
+    git secrets --add -- 'xox[apb]-[0-9]\{12\}-[0-9]\{12\}-[A-Za-z0-9]\{24\}'
+    git secrets --add -- 'sk_live_[A-Za-z0-9]\{24\}'
+    git secrets --add -- 'pk_live_[A-Za-z0-9]\{24\}'
+    git secrets --add -- 'AIza[0-9A-Za-z_-]\{35\}'
+    git secrets --add -- '^[0-9a-f]\{8\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{12\}$'
+    git secrets --add -- '[A-Za-z0-9_-]\{20,\}\.[A-Za-z0-9_-]\{20,\}\.[A-Za-z0-9_-]\{20,\}'
+    git secrets --add -- 'postgres:\/\/[A-Za-z0-9@:\-_.\/?%=+]+'
+    git secrets --add -- 'BEGIN'
+    touch "$HOOK_FLAG"
+fi
+
+echo "🔍 Running git-secrets..."
+git-secrets --pre_commit_hook -- "$@"

Dockerfile

@@ -1,21 +1,21 @@
 # compile typescript to ./dist
-FROM node:16-alpine AS tsc
+FROM node:22-alpine AS tsc
 WORKDIR /app
 COPY ["package*.json", "tsconfig.json", "yarn.lock", "./"]
 RUN yarn install
 COPY . ./
 RUN yarn build
 
 # install production only dependencies
-FROM node:16-alpine as yarn-prod
+FROM node:22-alpine AS yarn-prod
 WORKDIR /app
 COPY --from=tsc /app/package*.json ./
 COPY --from=tsc /app/yarn.lock ./
 COPY --from=tsc /app/dist ./
 RUN yarn install --production
 
 # copy only production artifacts (get rid of yarn cache)
-FROM node:16-alpine as image
+FROM node:22-alpine AS image
 RUN apk --no-cache add dumb-init curl && rm -rf /var/cache/apk/*
 WORKDIR /app
 COPY --from=yarn-prod /app ./

Dockerfile.grp

@@ -1,13 +1,13 @@
 # compile typescript to ./dist
-FROM node:16-alpine AS tsc
+FROM node:22-alpine AS tsc
 WORKDIR /app
 COPY ["package*.json", "tsconfig.json", "yarn.lock", "./"]
 RUN yarn install
 COPY . ./
 RUN yarn build:grp
 
 # install production only dependencies
-FROM node:16-alpine as yarn-prod
+FROM node:22-alpine AS yarn-prod
 WORKDIR /app
 COPY --from=tsc /app/package*.json ./
 COPY --from=tsc /app/yarn.lock ./
@@ -17,7 +17,7 @@ RUN yarn install --production
 RUN ./node_modules/.bin/patch-package
 
 # copy only production artifacts (get rid of yarn cache)
-FROM node:16-alpine as image
+FROM node:22-alpine AS image
 WORKDIR /app
 COPY --from=yarn-prod /app ./
 CMD ["yarn", "gas-refund:prod:compute-gas-refund-save-db"]

package.json

@@ -19,7 +19,8 @@
     "gas-refund:computeDistributionFilesAndPersistIPFS": "patch-package && NODE_ENV=development ts-node scripts/gas-refund-program/distribution/computeDistributionFilesAndPersistIPFS",
     "migrate:up": "source .env && DATABASE_URL=$DATABASE_URL npx sequelize-cli db:migrate # <- executes any new migrations that are not in sequalize meta table yet, sorted alphabetically",
     "migrate:undo": "source .env && DATABASE_URL=$DATABASE_URL npx sequelize-cli db:migrate:undo # <- undoes the last migration from sequalize meta table, sorted alphabetically",
-    "test": "jest"
+    "test": "jest",
+    "prepare": "husky"
   },
   "husky": {
     "hooks": {
@@ -41,7 +42,7 @@
     "@types/node": "14.14.41",
     "@types/stoppable": "1.1.1",
     "@types/validator": "13.6.6",
-    "husky": "7.0.4",
+    "husky": "^9.1.7",
     "jest": "^27.5.1",
     "jest-chance": "^0.1.11",
     "mkdirp": "^3.0.1",
@@ -79,7 +80,7 @@
     "log4js": "6.3.0",
     "merkletreejs": "^0.2.31",
     "moment": "2.29.1",
-    "newrelic": "11.15.0",
+    "newrelic": "12.24.0",
     "p-limit": "^3.1.0",
     "p-memoize": "^4.0.4",
     "parse-duration": "1.0.2",

src/lib/access-log.ts

@@ -0,0 +1,18 @@
+import { Request, Response } from 'express';
+import { connectLogger } from 'log4js';
+
+import { getRemoteAddress } from './remote-address';
+import { ACCESS_LOG_CATEGORY } from './log4js';
+
+const logger = global.LOGGER(ACCESS_LOG_CATEGORY, true);
+
+export default function AccessLogMiddleware() {
+  return connectLogger(logger, {
+    level: 'auto',
+
+    format: (req: Request, res: Response, format: (str: string) => string) => {
+      const baseLog = `${getRemoteAddress(req)} :hostname HTTP/:http-version :method ":url" :status :content-length - :response-timems`;
+      return format(baseLog);
+    },
+  });
+}

src/lib/log4js.ts

@@ -27,6 +27,8 @@ function calculateCategory() {
   return parsed?.fileName.split('.')[0].split(path.sep).join('.');
 }
 
+export const ACCESS_LOG_CATEGORY = 'ACCESS-LOG';
+
 const configuration: log4js.Configuration = {
   appenders: {
     console: {
@@ -45,7 +47,7 @@ const configuration: log4js.Configuration = {
       level: process.env['LOGGER_LEVEL'] || isDev ? 'trace' : 'info',
       enableCallStack: isDev,
     },
-    ACCESS_LOG_CATEGORY: {
+    [ACCESS_LOG_CATEGORY]: {
       appenders: ['console'],
       level: 'info',
     },

src/lib/middleware.ts

@@ -1,7 +1,6 @@
 import * as express from 'express';
 import * as parser from 'body-parser';
-import * as compression from 'compression';
-import * as cookie from 'cookie-parser';
+import AccessLogMiddleware from './access-log';
 
 const logger = global.LOGGER();
 
@@ -40,6 +39,9 @@ export default class Middleware {
     app.use(parser.json({ strict: false }));
     app.use(parser.text());
 
+    // generates access log message
+    app.use(AccessLogMiddleware());
+
     app.get('/robots.txt', function (req, res) {
       res.type('text/plain');
       res.send(`