util: add test folders and workflow

Author: habibayassin

.github/workflows/github-actions-cron-util-test.yml

@@ -0,0 +1,36 @@
+name: Test Util Scripts
+on:
+  schedule:
+  - cron: "0 8 * * SUN"
+  push:
+    paths:
+      - 'flow/util/'
+      - 'flow/util/test'
+  pull_request:
+    paths:
+      - 'flow/util/'
+      - 'flow/util/test'
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:     
+  testUtilScripts:
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+          submodules: recursive
+      - name: Install Dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y python3-pip
+      - name: Run Tests
+        run: |
+          cd flow/util/test
+          for file in *.py; do
+              python "$file"
+          done

flow/util/genElapsedTime.py

@@ -26,28 +26,30 @@
 # Loop on all log files in the directory
 for f in sorted(pathlib.Path(args.logDir).glob('**/[0-9]_*.log')):
     # Extract Elapsed Time line from log file
-    for line in open(str(f)):
-        elapsedTime = 0
-        
-        if 'Elapsed time' in line:
-            # Extract the portion that has the time
-            timePor = line.strip().replace('Elapsed time: ', '')
-            # Remove the units from the time portion
-            timePor = timePor.split('[h:]', 1)[0]
-            # Ensure that hours, min and seconds are separated by ':' not '.'
-            timePor = timePor.replace('.',':')
-            # Calculate elapsed time that has this format 'h:m:s'
-            timeList = timePor.split(':')
-            if len(timeList) == 2:
-                # Only minutes and seconds are present
-                elapsedTime = int(timeList[0])*60 + int(timeList[1])
-            elif len(timeList) == 3:
-                # Hours, minutes, and seconds are present
-                elapsedTime = int(timeList[0])*3600 + int(timeList[1])*60 + int(timeList[2])
-    if elapsedTime == 0:
-        print('No elapsed time found in',  str(f), file=sys.stderr)
+    with open(str(f)) as logfile:
+        for line in logfile:
+            elapsedTime = 0
+            
+            if 'Elapsed time' in line:
+                # Extract the portion that has the time
+                timePor = line.strip().replace('Elapsed time: ', '')
+                # Remove the units from the time portion
+                timePor = timePor.split('[h:]', 1)[0]
+                # Ensure that hours, min and seconds are separated by ':' not '.'
+                timePor = timePor.replace('.',':')
+                # Calculate elapsed time that has this format 'h:m:s'
+                timeList = timePor.split(':')
+                if len(timeList) == 2:
+                    # Only minutes and seconds are present
+                    elapsedTime = int(timeList[0])*60 + int(timeList[1])
+                elif len(timeList) == 3:
+                    # Hours, minutes, and seconds are present
+                    elapsedTime = int(timeList[0])*3600 + int(timeList[1])*60 + int(timeList[2])
+        if elapsedTime == 0:
+            print('No elapsed time found in',  str(f), file=sys.stderr)
 
     # Print the name of the step and the corresponding elapsed time
-    print('%-25s %10s' % (os.path.splitext(os.path.basename(str(f)))[0], elapsedTime))
+    if elapsedTime != 0:
+        print('%-25s %10s' % (os.path.splitext(os.path.basename(str(f)))[0], elapsedTime))
 
 

flow/util/test_genElapsedTime.py flow/util/test/test_genElapsedTime.pyflow/util/test_genElapsedTime.py renamed to flow/util/test/test_genElapsedTime.py

@@ -11,15 +11,18 @@
 from datetime import datetime
 import argparse  # argument parsing
 from pathlib import Path
+import importlib  # module reloading
 
 # make sure the working dir is flow/
-os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)) , '..'))
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
+os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
 class TestElapsedTime(unittest.TestCase):
 
     def setUp(self):
         self.tmp_dir = tempfile.TemporaryDirectory()
         self.log_file = os.path.join(self.tmp_dir.name, "1_test.log")
+        self.module_name = "genElapsedTime"
 
     @patch("sys.stdout", new_callable=StringIO)
     def test_elapsed_time(self, mock_stdout):
@@ -28,23 +31,24 @@ def test_elapsed_time(self, mock_stdout):
             f.write("Some log entry\n")
             f.write("Elapsed time: 01:30:00[h:]min:sec.\n")
         # call the script with the test log file
-        sys.argv = ["util/genElapsedTime.py", "--logDir", str(self.tmp_dir.name)]
+        sys.argv = ["./genElapsedTime.py", "--logDir", str(self.tmp_dir.name)]
         with patch.object(sys, 'argv', sys.argv):
-                import genElapsedTime
+            module = importlib.import_module(self.module_name)
         # check if output is correct
         expected_output = "1_test                          5400\n"
         self.assertEqual(mock_stdout.getvalue(), expected_output)
 
     @patch("sys.stdout", new_callable=StringIO)
-    def test_longer_elapsed_time(self, mock_stdout):
+    def test_elapsed_time_longer_duration(self, mock_stdout):
         # create a log file with elapsed time
         with open(self.log_file, "w") as f:
             f.write("Some log entry\n")
             f.write("Elapsed time: 12:24.14[h:]min:sec. CPU time: user 5081.82 sys 170.18 (705%). Peak memory: 9667132KB.\n")
         # call the script with the test log file
         sys.argv = ["util/genElapsedTime.py", "--logDir", str(self.tmp_dir.name)]
         with patch.object(sys, 'argv', sys.argv):
-                import genElapsedTime
+            module = importlib.import_module(self.module_name)
+            importlib.reload(module)
         # check if output is correct
         expected_output = "1_test                         44654\n"
         self.assertEqual(mock_stdout.getvalue(), expected_output)
@@ -53,20 +57,21 @@ def test_missing_arg(self):
         with self.assertRaises(SystemExit):
             with patch('sys.stderr', new=StringIO()) as fake_err_output:
                 with patch('sys.argv', ["util/genElapsedTime.py"]):
-                    import genElapsedTime
+                    module = importlib.import_module(self.module_name)
+                    importlib.reload(module)
             self.assertIn('--logDir', fake_err_output.getvalue())
 
-
     def test_no_elapsed_time(self):
         with open(self.log_file, "w") as f:
             f.write('Other log message')
         with patch('sys.argv', ["util/genElapsedTime.py", "--logDir", str(self.tmp_dir.name)]):
-                with patch('sys.stderr', new=StringIO()) as fake_err_output:
-                    import genElapsedTime
-                self.assertIn('No elapsed time found in', fake_err_output.getvalue())
+            with patch('sys.stderr', new=StringIO()) as fake_err_output:
+                module = importlib.import_module(self.module_name)
+                importlib.reload(module)
+            self.assertIn('No elapsed time found in', fake_err_output.getvalue())
 
     def tearDown(self):
        self.tmp_dir.cleanup()
 
 if __name__ == '__main__':
-    unittest.main()
+    unittest.main()