fix: configurable db setup and users for workdir

Author: mahendraintelops

.github/workflows/app-ci.yml

@@ -116,8 +116,8 @@ jobs:
           context: ./app
           # Note: tags has to be all lower-case
           tags: ${{ env.REGISTRY }}/${{ github.repository }}/app:${{ github.run_id }}
-          # ${{ steps.md.metadata.outputs.tags }}
-          labels: ${{ steps.md.metadata.outputs.labels }}
+          # ${{ steps.metadata.outputs.tags }}
+          labels: ${{ steps.metadata.outputs.labels }}
           # build on feature branches, push only on main branch
           push: true
 #      - uses: goodwithtech/dockle-action@main

.github/workflows/core-ci.yml

@@ -115,11 +115,11 @@ jobs:
         with:
           # relative path to the place where source code with Dockerfile is located
           context: ./core
-          # Note: tags has to be all lower-case
+          # Note: tags have to be all lower-case
           tags: ${{ env.REGISTRY }}/${{ github.repository }}/core:${{ github.run_id }}
-          # ${{ steps.md.metadata.outputs.tags }}
-          labels: ${{ steps.md.metadata.outputs.labels }}
-          # build on feature branches, push only on main branch
+          # ${{ steps.metadata.outputs.tags }}
+          labels: ${{ steps.metadata.outputs.labels }}
+          # build on feature branches, push only on the main branch
           push: true
 #      - uses: goodwithtech/dockle-action@main
 #        with:

app/Dockerfile

@@ -16,8 +16,11 @@ RUN apk update && apk add git
 COPY --from=builder /app/dist /server/dist
 COPY --from=builder /app/node_modules /server/node_modules
 COPY --from=builder /app/protobufs /server/protobufs
-RUN mkdir -p /compage/workdir
-RUN chmod -R 777 /compage/workdir
+### Add new user
+RUN adduser -D -g '' compageuser
+RUN mkdir -p $HOME/.compage/workdir
+RUN chmod -R 777 $HOME/.compage/workdir
+USER compageuser
 ENV NODE_ENV=production
 EXPOSE 5000
 CMD [ "node", "dist/src/app.js" ]

app/README.md

@@ -1,11 +1,20 @@
 # app
-- This is a component which connects to different git-platforms and push-pull repositories. 
-- This also uses Cassandra or SQLite (based on configuration) databases to persist projects, gitplatforms and users related details.
-- It's an Express.js based REST server and gRPC client to core component.
+- This is a component that connects to different git-platforms and push-pull repositories. 
+- This also uses Cassandra or SQLite (based on configuration) databases to persist projects, git platforms, and users related details.
+- It's an Express.js based REST server and gRPC client to a core component.
+- The sqlite is used as a database for development. To use cassandra, you need to set the following environment variables in your terminal along with a `environment DB_TYPE='cassandra'`.
+```bash
+    export CASSANDRA_CONTACT_POINTS = 'localhost';
+    export CASSANDRA_LOCAL_DATA_CENTER = 'datacenter1';
+    export CASSANDRA_KEYSPACE = 'compage';
+    export CASSANDRA_USERNAME = 'cassandra';
+    export CASSANDRA_PASSWORD = 'cassandra';
+```
+- To start the cassandra server on local, you can fire command `docker-compose up -d` from the setup directory of compage/app.
 
 #### How to run this component?
 - Navigate to app directory [`cd app`] from root directory of compage
 - Fire `npm install` to install the dependencies
 - Run `npm run dev` command to start the express-server. This command will auto-reload the changes you make to app directory.
-- Kill process on port 5000
+- If something goes wrong, you want to kill the process on port 5000, run the following command
 ```sudo kill -9 `sudo lsof -t -i:5000```

app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "app",
-  "version": "0.0.1",
+  "version": "1.0.0",
   "main": "app.ts",
   "repository": "https://github.com/intelops/compage.git",
   "author": "Mahendra <[email protected]>",

app/src/integrations/git-platforms/index.ts

@@ -5,6 +5,7 @@ import {GitPlatformEntity} from '../../models/gitPlatform';
 import {NewProjectGitServerRequest} from '../simple-git/models';
 import {pushNewProjectToGitServer} from '../simple-git/newProject';
 import {GitPlatformService} from '../../services/gitPlatformService';
+import {WORKDIR_PATH} from '../../utils/constants';
 
 const gitPlatformService = new GitPlatformService();
 
@@ -23,7 +24,7 @@ export const createRepository = async (projectEntity: ProjectEntity) => {
 };
 
 export const makeInitialCommit = async (projectEntity: ProjectEntity) => {
-    const createdProjectPath = `/compage/workdir/workdir/${projectEntity.id}`;
+    const createdProjectPath = `${WORKDIR_PATH}/${projectEntity.id}`;
     fs.mkdirSync(createdProjectPath, {recursive: true});
     fs.writeFileSync(`${createdProjectPath}/README.md`, `# ${projectEntity.repository_name}`);
     const gitPlatform: GitPlatformEntity = await gitPlatformService.getGitPlatform(projectEntity.owner_email, projectEntity.git_platform_name);
@@ -45,6 +46,7 @@ export const makeInitialCommit = async (projectEntity: ProjectEntity) => {
     if (error.length !== 0) {
         throw new Error(error);
     }
+    // TODO: uncomment this line
     // delete the project from temp folder once it's pushed to git server.
     // fs.rmSync(createdProjectPath, {recursive: true});
 };

app/src/integrations/simple-git/common.ts

@@ -37,7 +37,7 @@ export const gitOperations = async (git: SimpleGit, repositoryBranch?: string, p
 
     let branchName: string = '';
     if (!repositoryBranch && !projectVersion) {
-        branchName = 'master';
+        branchName = 'main';
     } else {
         branchName = repositoryBranch + '-' + projectVersion;
     }

app/src/integrations/simple-git/newProject.ts

@@ -4,6 +4,7 @@ import Logger from '../../utils/logger';
 import {NewProjectGitServerRequest} from './models';
 
 export const pushNewProjectToGitServer = async (newProjectGitServerRequest: NewProjectGitServerRequest): Promise<string> => {
+    console.log('pushNewProjectToGitServer' + newProjectGitServerRequest.generatedProjectPath);
     const options: Partial<SimpleGitOptions> = {
         baseDir: newProjectGitServerRequest.generatedProjectPath,
         binary: 'git',
@@ -27,7 +28,6 @@ export const pushNewProjectToGitServer = async (newProjectGitServerRequest: NewP
     if (error.length > 0) {
         return error;
     }
-    Logger.debug('git init done');
 
     // add local git config like username and email
     await git.addConfig('user.email', newProjectGitServerRequest.gitProviderDetails.platformEmail);

app/src/routes/codeOperations.ts

@@ -10,7 +10,7 @@ import {requireEmailMiddleware} from '../middlewares/auth';
 import Logger from '../utils/logger';
 import {rimraf} from 'rimraf';
 import tar from 'tar';
-import {X_EMAIL_HEADER} from '../utils/constants';
+import {WORKDIR_PATH, X_EMAIL_HEADER} from '../utils/constants';
 import {GenerateCodeRequest, getGenerateCodeError, getGenerateCodeResponse, Project} from '../models/code';
 import {GitPlatformEntity} from '../models/gitPlatform';
 import {Metadata, OldVersion, ProjectEntity} from '../models/project';
@@ -87,7 +87,7 @@ codeOperationsRouter.post('/generate', requireEmailMiddleware, async (request, r
     }
 
     // create directory hierarchy here itself as creating it after receiving data will not be proper.
-    const originalProjectPath = `/compage/workdir/${projectEntity.display_name}`;
+    const originalProjectPath = `${WORKDIR_PATH}/${projectEntity.display_name}`;
     // this is a path where the project will be downloaded
     const downloadedProjectPath = `${originalProjectPath}_downloaded`;
     // this is a path where the project will be cloned

app/src/utils/constants.ts

@@ -8,7 +8,10 @@ import os from 'os';
 export const X_EMAIL_HEADER = 'X-Email-ID';
 export const DEVELOPMENT = 'development';
 export const TEST = 'test';
+export const CONFIG_PATH = `${os.homedir()}/.compage`;
+export const WORKDIR_PATH = `${CONFIG_PATH}/workdir`;
 
+// read secret files from the given path - happens only in production [not in development or test] in containers
 const readSecretFile = (secretPath: string) => {
     const keyValuePairs: Map<string, string> = new Map();
     const entries = fs.readdirSync(secretPath, {withFileTypes: true});
@@ -20,6 +23,7 @@ const readSecretFile = (secretPath: string) => {
     return keyValuePairs;
 };
 
+// set NODE_ENV to development if not set
 if (!process.env.NODE_ENV) {
     process.env.NODE_ENV = DEVELOPMENT;
 }
@@ -31,25 +35,41 @@ let config: Config;
 if (isDevelopment) {
     // use verbose logging
     // use server in development mode
-    dotenv.config({path: `${os.homedir()}/.compage/.env`});
-    assert.ok(process.env.COMPAGE_CORE_URL, `The 'COMPAGE_CORE_URL' environment variable is required`);
-    config = {
-        // db config
-        db: {
+    dotenv.config({path: `${CONFIG_PATH}/.env`});
+    let db: DbConfig;
+    if (process.env.DB_TYPE === 'cassandra') {
+        db = {
+            cassandra: {
+                contactPoints: process.env.CASSANDRA_CONTACT_POINTS?.split(','),
+                localDataCenter: process.env.CASSANDRA_LOCAL_DATA_CENTER,
+                keyspace: process.env.CASSANDRA_KEYSPACE,
+                credentials: {
+                    username: process.env.CASSANDRA_USERNAME,
+                    password: process.env.CASSANDRA_PASSWORD
+                }
+            }
+        };
+    } else {
+        db = {
             sqlite: {
                 storage: 'db.sqlite'
             }
-        },
+        };
+    }
+    assert.ok(process.env.COMPAGE_CORE_URL, `The 'COMPAGE_CORE_URL' environment variable is required`);
+    config = {
+        // db config
+        db,
         // app server config
         serverPort: process.env.PORT || 5000,
-        // core url
+        // compage-core url
         compageCoreUrl: process.env.COMPAGE_CORE_URL,
     };
 } else if (isTest) {
     config = {
         // app server config
         serverPort: 5000,
-        // db config
+        // db config for testing is sqlite
         db: {
             sqlite: {
                 storage: 'db.sqlite'
@@ -101,22 +121,30 @@ if (isDevelopment) {
     };
 }
 
+export interface CassandraDBConfig {
+    contactPoints?: string[];
+    localDataCenter?: string;
+    keyspace?: string;
+    credentials?: {
+        username?: string;
+        password?: string;
+    };
+}
+
+export interface SqliteDBConfig {
+    storage: string;
+}
+
+export interface DbConfig {
+    cassandra?: CassandraDBConfig;
+    sqlite?: SqliteDBConfig;
+}
+
 export interface Config {
     // app server config
     serverPort?: string | number;
-    // cassandra config
-    db?: {
-        cassandra?: {
-            contactPoints?: string[];
-            localDataCenter?: string;
-            keyspace?: string;
-            credentials?: {
-                username: string;
-                password: string;
-            }
-        },
-        sqlite?: {}
-    };
+    // db config
+    db?: DbConfig;
     // core url
     compageCoreUrl?: string;
 }