set up optimized pipelines

Author: bunsenstraat

.circleci/config.yml

@@ -1,5 +1,14 @@
 version: 2.1
 parameters:
+  run_all_tests:
+    type: boolean
+    default: false
+  run_pr_tests:
+    type: boolean
+    default: false
+  run_metamask_tests:
+    type: boolean
+    default: false  
   run_flaky_tests:
     type: boolean
     default: false   
@@ -12,12 +21,7 @@ parameters:
   linux:
     type: boolean
     default: false
-  web:
-    type: boolean
-    default: true
-  run_metamask_tests:
-    type: boolean
-    default: false   
+
 orbs:
   browser-tools: circleci/[email protected]
   win: circleci/[email protected]
@@ -756,6 +760,8 @@ jobs:
         type: string
       script:
         type: string
+      scriptparameter:
+        type: string
       job:
         type: string
       jobsize:
@@ -816,7 +822,7 @@ jobs:
                 install-chromedriver: false
             - run: yarn install_webdriver
             - run: firefox --version
-      - run: ./apps/remix-ide/ci/<< parameters.script >> << parameters.browser >> << parameters.jobsize >> << parameters.job >>
+      - run: ./apps/remix-ide/ci/<< parameters.script >> << parameters.browser >> << parameters.jobsize >> << parameters.job >> << parameters.scriptparameter >>
       - store_test_results:
           path: ./reports/tests
       - store_artifacts:
@@ -969,6 +975,22 @@ workflows:
             - build-remixdesktop-linux
             - test-remixdesktop-linux
 
+  run_pr_tests:
+    when: << pipeline.parameters.run_pr_tests >>
+    jobs:
+      - build
+      - remix-ide-browser:
+          requires:
+            - build
+          matrix:
+            parameters:
+              browser: ["chrome"]
+              script: ["singletest"]
+              job: ["nogroup"]
+              jobsize: ["1"]
+              parallelism: [1]
+              scriptparameter: ["pr"]
+
   run_flaky_tests:
     when: << pipeline.parameters.run_flaky_tests >>
     jobs:
@@ -979,10 +1001,11 @@ workflows:
           matrix:
             parameters:
               browser: ["chrome", "firefox"]
-              script: ["flaky.sh"]
+              script: ["singletest"]
               job: ["nogroup"]
               jobsize: ["1"]
               parallelism: [5]
+              scriptparameter: ["flaky"]
 
   run_metamask_tests:
     when: << pipeline.parameters.run_metamask_tests >>
@@ -999,7 +1022,7 @@ workflows:
               jobsize: ["1"]
               parallelism: [1]
   web:
-    when: << pipeline.parameters.web >>
+    when: << pipeline.parameters.run_all_tests >>
     jobs:
       - build
       - build-plugin:

apps/remix-ide-e2e/src/buildGroupTests.js

@@ -15,23 +15,23 @@ fs.readdirSync(testFolder).forEach(file => {
   if (!file.includes('group')) {
     const content = fs.readFileSync(testFolder + file, 'utf8')
     const matches = content.match(/group\d+/g)
-    createFlakyTestFiles(file, content)
+    createTaggedTestFiles(file, content)
     createFiles(file, matches)
   }
 })
 
-function createFiles(file, matches, flaky = false) {
+function createFiles(file, matches, tag = false) {
   if (matches) {
     const unique = matches.filter(onlyUnique)
     unique.map((group) => {
       const rewrite = source.replace('#groupname', group).replace('#file', file.replace('.ts', ''))
       const extension = file.split('.')
       extension.shift()
       let filename
-      if (!flaky) {
+      if (!tag) {
         filename = `${testFolder}${file.split('.').shift()}_${group}.${extension.join('.')}`
       } else {
-        filename = `${testFolder}${file.split('.').shift()}_${group}.flaky.ts`
+        filename = `${testFolder}${file.split('.').shift()}_${group}.${tag.toLowerCase()}.ts`
       }
       fs.writeFileSync(filename, rewrite)
     })
@@ -42,14 +42,17 @@ function onlyUnique(value, index, self) {
   return self.indexOf(value) === index
 }
 
-function createFlakyTestFiles(file, text) {
+function createTaggedTestFiles(file, text) {
   const lines = text.split('\n')
-  lines.forEach((line, index) => {
-    // if line contains #flaky
-    if (line.includes('#flaky')) {
+  lines.forEach((line) => {
+    if (line.includes('#flaky') || line.includes('#pr') || line.includes('#PR')) {
       const matches = line.match(/group\d+/g)
-      const unique = matches.filter(onlyUnique)
-      createFiles(file, matches, true)
+      if (matches) {
+        const tags = line.match(/#(flaky|pr|PR)/gi).map(t => t.replace('#', '').toLowerCase());
+        tags.forEach(tag => {
+          createFiles(file, matches, tag);
+        });
+      }
     }
   })
 }

apps/remix-ide-e2e/src/tests/ballot.test.ts

@@ -20,7 +20,7 @@ module.exports = {
     browser
       .addFile('Untitled.sol', sources[0]['Untitled.sol'])
   },
-  'Deploy Ballot #group1': function (browser: NightwatchBrowser) {
+  'Deploy Ballot #flaky #group1 #pr': function (browser: NightwatchBrowser) {
     browser
       .waitForElementVisible('*[data-id="remixIdeIconPanel"]', 10000)
       .clickLaunchIcon('solidity')

apps/remix-ide/ci/flaky.sh renamed to apps/remix-ide/ci/singletest.sh

@@ -2,14 +2,14 @@
 
 set -e
 
-TESTFILES=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "\.flaky" | sort )
+TESTFILES=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "\.${4}" | sort )
 
 # count test files
-fileCount=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "\.flaky" | wc -l )
+fileCount=$(grep -IRiL "\'@disabled\': \?true" "dist/apps/remix-ide-e2e/src/tests" | grep "\.${4}" | wc -l )
 # if fileCount is 0
 if [ $fileCount -eq 0 ]
 then
-  echo "No flaky tests found"
+  echo "No flaky or PR tests found"
   exit 0
 fi
 

projects.json

@@ -4362,7 +4362,7 @@
               "hash": "b99ed2d6b005cdaeffe543b9f95b0c1ccafee7f3"
             },
             {
-              "file": "apps/remix-ide/ci/flaky.sh",
+              "file": "apps/remix-ide/ci/singletest",
               "hash": "15fefe4c3eae0f283031043bc3c491679e23cf39"
             },
             {