Merge pull request #1189 from jescalada/985-optimize-pullRemote-for-large-repos

feat: push speed optimizations

Author: jescalada

src/proxy/processors/push-action/checkAuthorEmails.ts

@@ -30,8 +30,6 @@ const isEmailAllowed = (email: string): boolean => {
 };
 
 const exec = async (req: any, action: Action): Promise<Action> => {
-  console.log({ req, action });
-
   const step = new Step('checkAuthorEmails');
 
   const uniqueAuthorEmails = [

src/proxy/processors/push-action/checkCommitMessages.ts

@@ -51,8 +51,6 @@ const isMessageAllowed = (commitMessage: string): boolean => {
 
 // Execute if the repo is approved
 const exec = async (req: any, action: Action): Promise<Action> => {
-  console.log({ req, action });
-
   const step = new Step('checkCommitMessages');
 
   const uniqueCommitMessages = [...new Set(action.commitData?.map((commit) => commit.message))];

src/proxy/processors/push-action/pullRemote.ts

@@ -9,15 +9,14 @@ const exec = async (req: any, action: Action): Promise<Action> => {
   const step = new Step('pullRemote');
 
   try {
-    action.proxyGitPath = `${dir}/${action.timestamp}`;
-
-    step.log(`Creating folder ${action.proxyGitPath}`);
+    action.proxyGitPath = `${dir}/${action.id}`;
 
     if (!fs.existsSync(dir)) {
       fs.mkdirSync(dir);
     }
 
     if (!fs.existsSync(action.proxyGitPath)) {
+      step.log(`Creating folder ${action.proxyGitPath}`);
       fs.mkdirSync(action.proxyGitPath, 0o755);
     }
 
@@ -33,15 +32,12 @@ const exec = async (req: any, action: Action): Promise<Action> => {
       fs,
       http: gitHttpClient,
       url: action.url,
-      onAuth: () => ({
-        username,
-        password,
-      }),
       dir: `${action.proxyGitPath}/${action.repoName}`,
+      onAuth: () => ({ username, password }),
+      singleBranch: true,
+      depth: 1,
     });
 
-    console.log('Clone Success: ', action.url);
-
     step.log(`Completed ${cmd}`);
     step.setContent(`Completed ${cmd}`);
   } catch (e: any) {

src/proxy/routes/index.ts

@@ -216,7 +216,8 @@ const getRouter = async () => {
         proxyReqOptDecorator: proxyReqOptDecorator,
         proxyReqBodyDecorator: proxyReqBodyDecorator,
         proxyErrorHandler: proxyErrorHandler,
-      }),
+        stream: true,
+      } as any),
     );
   });
 
@@ -229,7 +230,8 @@ const getRouter = async () => {
     proxyReqOptDecorator: proxyReqOptDecorator,
     proxyReqBodyDecorator: proxyReqBodyDecorator,
     proxyErrorHandler: proxyErrorHandler,
-  });
+    stream: true,
+  } as any);
 
   console.log('proxy keys registered: ', JSON.stringify(proxyKeys));
 

src/service/routes/push.js

@@ -78,8 +78,6 @@ router.post('/:id/reject', async (req, res) => {
 });
 
 router.post('/:id/authorise', async (req, res) => {
-  console.log({ req });
-
   const questions = req.body.params?.attestation;
   console.log({ questions });
 

test/processors/clearBareClone.test.js

@@ -6,11 +6,13 @@ const { Action } = require('../../src/proxy/actions/Action');
 chai.should();
 
 const expect = chai.expect;
+
+const actionId = '123__456';
 const timestamp = Date.now();
 
 describe('clear bare and local clones', async () => {
   it('pull remote generates a local .remote folder', async () => {
-    const action = new Action('123', 'type', 'get', timestamp, 'finos/git-proxy.git');
+    const action = new Action(actionId, 'type', 'get', timestamp, 'finos/git-proxy.git');
     action.url = 'https://github.com/finos/git-proxy.git';
 
     const authorization = `Basic ${Buffer.from('JamieSlome:test').toString('base64')}`;
@@ -24,14 +26,14 @@ describe('clear bare and local clones', async () => {
       action,
     );
 
-    expect(fs.existsSync(`./.remote/${timestamp}`)).to.be.true;
+    expect(fs.existsSync(`./.remote/${actionId}`)).to.be.true;
   }).timeout(20000);
 
   it('clear bare clone function purges .remote folder and specific clone folder', async () => {
-    const action = new Action('123', 'type', 'get', timestamp, 'finos/git-proxy.git');
+    const action = new Action(actionId, 'type', 'get', timestamp, 'finos/git-proxy.git');
     await clearBareClone(null, action);
     expect(fs.existsSync(`./.remote`)).to.throw;
-    expect(fs.existsSync(`./.remote/${timestamp}`)).to.throw;
+    expect(fs.existsSync(`./.remote/${actionId}`)).to.throw;
   });
 
   afterEach(() => {