Merge pull request #20 from UCL-ARC/test-drive-mesh-generator

Test drive mesh generator

Author: connoraird

CMakeLists.txt

@@ -8,7 +8,7 @@ project(
   DESCRIPTION "Fortran tooling"
 )
 
-set(GFORTRANFLAGS "-ffree-line-length-none")
+set(GFORTRAN_FLAGS "-ffree-line-length-none")
 set(CMAKE_Fortran_FLAGS "${GFORTRAN_FLAGS}") # Update this to match the flags for the chosen compiler
 
 # Define src files
@@ -19,3 +19,10 @@ file(GLOB PROJ_POISSON_SOURCES "${PROJ_SRC_DIR}/poisson/*.f90")
 # Build src executables
 add_executable("${PROJECT_NAME}-mesh-generator" "${PROJ_MESH_GENERATOR_SOURCES}")
 add_executable("${PROJECT_NAME}-poisson" "${PROJ_POISSON_SOURCES}")
+
+enable_testing()
+
+#--------------------------------------#
+#             test-drive               #
+#--------------------------------------#
+add_subdirectory("testing/test-drive")

README.md

@@ -58,3 +58,7 @@ This will produce executables for the two src codes, `fortran-tooling-mesh-gener
 ```sh
 ./build/fortran-tooling-poisson # then respond to prompt with the mesh name, likely to be `square_mesh`
 ```
+
+## Running the tests
+
+To run the tests run `ctest` from within the `build` directory.

src/mesh_generator/mesh_generator.f90

@@ -69,8 +69,8 @@ subroutine calculate_mesh(num_edges_per_boundary, num_nodes, num_elements, num_b
         counter = 1
         do i = 1, num_nodes_per_boundary
             do j = 1, num_nodes_per_boundary
-                nodes(1, counter) = i
-                nodes(2, counter) = j
+                nodes(1, counter) = i ! x coordinate
+                nodes(2, counter) = j ! y coordinate
                 counter = counter + 1
             end do
         end do
@@ -80,9 +80,9 @@ subroutine calculate_mesh(num_edges_per_boundary, num_nodes, num_elements, num_b
             do j = 1, num_edges_per_boundary
                 bottom_left_node = j + (i - 1) * num_nodes_per_boundary
 
-                elements(1, counter) = bottom_left_node
-                elements(2, counter) = bottom_left_node + 1
-                elements(3, counter) = bottom_left_node + 1 + num_nodes_per_boundary
+                elements(1, counter) = bottom_left_node                              ! bottom left node
+                elements(2, counter) = bottom_left_node + 1                          ! Next node anti-clockwise
+                elements(3, counter) = bottom_left_node + 1 + num_nodes_per_boundary ! Next node anti-clockwise
 
                 elements(1, counter + num_edges_per_boundary) = bottom_left_node
                 elements(2, counter + num_edges_per_boundary) = bottom_left_node + num_nodes_per_boundary + 1
@@ -98,7 +98,7 @@ subroutine calculate_mesh(num_edges_per_boundary, num_nodes, num_elements, num_b
             ! bottom boundary 
             boundary_edges(1, i) = i       ! left node
             boundary_edges(2, i) = i + 1   ! right node
-            boundary_edges(3, i) = i*2 - 1 ! element
+            boundary_edges(3, i) = i       ! element
 
             ! right boundary
             boundary_edges(1, i + num_edges_per_boundary) = i       * num_nodes_per_boundary
@@ -108,7 +108,7 @@ subroutine calculate_mesh(num_edges_per_boundary, num_nodes, num_elements, num_b
             ! top boundary
             boundary_edges(1, i + num_edges_per_boundary * 2) = num_nodes - i + 1
             boundary_edges(2, i + num_edges_per_boundary * 2) = num_nodes - i
-            boundary_edges(2, i + num_edges_per_boundary * 2) = num_elements - i + 1
+            boundary_edges(3, i + num_edges_per_boundary * 2) = num_elements - i + 1
 
             ! left boundary
             boundary_edges(1, i + num_edges_per_boundary * 3) = (num_nodes_per_boundary - i) * num_nodes_per_boundary + 1

testing/README.md

@@ -1,19 +1,15 @@
 # fortran-unit-testing
-Fortran code bases often have next to zero test coverage. Any tests implemented are fully end to end and often just check that a single value in some
-output file is what we expect it to be. Whilst a test like this can catch a breaking change or bug, it will be unlikely to indicate where that 
-breaking change has been introduced. The solution to this issue is unit testing. 
+Fortran code bases often have next to zero test coverage. Any tests implemented are fully end to end and often just check that a single value in some output file is what we expect it to be. Whilst a test like this can catch a breaking change or bug, it will be unlikely to indicate where that breaking change has been introduced. The solution to this issue is unit testing. 
 
-There are several tools/frameworks written for unit testing Fortran code. However, these are not widely adopted by research codebases. We would like
-to understand why that is and help to change it. 
+There are several tools/frameworks written for unit testing Fortran code. However, these are not widely adopted by research codebases. We would like to understand why that is and help to change it. 
 
 There are several examples of good unit testing tools for other languages, such as 
 
 - pytest (Python)
 - QtTest (Qt in c++)
 - J-Unit (JavaScript)
 
-These will be used as the basis for what the recommended Fortran unit testing tool should look like. Therefore, key features from these tools shall be 
-individually tested for each Fortran unit testing tool we select to test.
+These will be used as the basis for what the recommended Fortran unit testing tool should look like. Therefore, key features from these tools shall be individually tested for each Fortran unit testing tool we select to test.
 
 ## Aims
 - Test multiple fortran unit testing tools to determine which we would recommend. To do this we will

testing/tests/pFUnit/README.md renamed to testing/pFUnit/README.md

@@ -1,14 +1,4 @@
-cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
-
-# Set project name
-project(
-  "fortran-unit-testing-with-test-drive"
-  LANGUAGES "Fortran"
-  VERSION "0.0.1"
-  DESCRIPTION "Playground project for testing test-drive"
-)
-
-# Get test-drive lib
+# Install the test-drive dependency
 set(test_lib "test-drive")
 set(test_pkg "TEST_DRIVE")
 set(test_url "https://github.com/fortran-lang/test-drive")
@@ -32,42 +22,43 @@ if(NOT EXISTS "${${test_pkg}_BINARY_DIR}/include")
     make_directory("${${test_pkg}_BINARY_DIR}/include")
 endif()
 
-# Define src and test files
-set(PROJ_SRC_DIR "${PROJECT_SOURCE_DIR}/../../src")
-set(PROJ_TEST_DIR "${PROJECT_SOURCE_DIR}")
-
-file(GLOB PROJ_SOURCES          "${PROJ_SRC_DIR}/*.f90")
-file(GLOB PROJ_TEST_SOURCES     "${PROJ_TEST_DIR}/tests/*.f90")
+# Define test files
+set(PROJ_TEST_DRIVE_DIR "${PROJECT_SOURCE_DIR}/testing/test-drive")
+file(GLOB PROJ_TEST_DRIVE_SOURCES "${PROJ_TEST_DRIVE_DIR}/tests/*.f90")
 
-# Filter out the main.cxx file. We can only have one main() function in our tests
-set(PROJ_SOURCES_EXC_MAIN ${PROJ_SOURCES})
-list(FILTER PROJ_SOURCES_EXC_MAIN EXCLUDE REGEX ".*main.f90")
-
-# Build src executable
-add_executable("${PROJECT_NAME}" "${PROJ_SOURCES}")
+# Filter out the main.f90 files. We can only have one main() function in our tests
+set(PROJ_POISSON_SOURCES_EXC_MAIN ${PROJ_POISSON_SOURCES})
+set(PROJ_MESH_GENERATOR_SOURCES_EXC_MAIN ${PROJ_MESH_GENERATOR_SOURCES})
+list(FILTER PROJ_POISSON_SOURCES_EXC_MAIN EXCLUDE REGEX ".*main.f90")
+list(FILTER PROJ_MESH_GENERATOR_SOURCES_EXC_MAIN EXCLUDE REGEX ".*main.f90")
 
 # Unit testing
 set(
   tests
-  "calc_pi"
+  "mesh_generator"
+)
+set(
+  test-srcs
+  "main.f90"
 )
 foreach(t IN LISTS tests)
   string(MAKE_C_IDENTIFIER ${t} t) 
-  list(APPEND test-srcs "${PROJ_TEST_DIR}/test_${t}.f90")
+  list(APPEND test-srcs "${PROJ_TEST_DRIVE_DIR}/test_${t}.f90")
 endforeach()
 
 add_executable(
-  "${PROJECT_NAME}-tester"
-  "${PROJ_TEST_DIR}/main.f90"
-  "${PROJ_TEST_SOURCES}"
-  "${PROJ_SOURCES_EXC_MAIN}"
+  "test_${PROJECT_NAME}-test-drive"
+  "${test-srcs}"
+  "${PROJ_TEST_DRIVE_SOURCES}"
+  "${PROJ_POISSON_SOURCES_EXC_MAIN}"
+  "${PROJ_MESH_GENERATOR_SOURCES_EXC_MAIN}"
 )
 target_link_libraries(
-  "${PROJECT_NAME}-tester"
+  "test_${PROJECT_NAME}-test-drive"
   PRIVATE
   "test-drive::test-drive"
 )
 
 foreach(t IN LISTS tests)
-  add_test("${PROJECT_NAME}/${t}" "${PROJECT_NAME}-tester" "${t}")
+  add_test("${PROJECT_NAME}-test-drive/${t}" "test_${PROJECT_NAME}-test-drive" "${t}")
 endforeach()

testing/src/calc_pi.f90

@@ -0,0 +1,54 @@
+# test-drive
+This project offers a lightweight, procedural unit testing framework based on nothing but standard Fortran. Integration with [meson](https://mesonbuild.com/), [cmake](https://cmake.org/) and [Fortran package manager (fpm)](https://github.com/fortran-lang/fpm) is available. Alternatively, the testdrive.F90 source file can be redistributed in the project's testsuite as well.
+
+## Running the tests
+
+The test-drive tests will run with the rest of the [tests](../README.md#running-the-tests) in the repo.
+
+There is a known issue that the tests for test-drive itself will also be ran by `ctest` as shown below
+```sh
+$ ctest   
+Test project /Users/connoraird/work/fortran-tooling/build
+    Start 1: fortran-tooling-test-drive/mesh_generator
+1/4 Test #1: fortran-tooling-test-drive/mesh_generator ...   Passed    0.33 sec
+    Start 2: test-drive/all-tests
+2/4 Test #2: test-drive/all-tests ........................   Passed    0.33 sec
+    Start 3: test-drive/check
+3/4 Test #3: test-drive/check ............................   Passed    0.01 sec
+    Start 4: test-drive/select
+4/4 Test #4: test-drive/select ...........................   Passed    0.01 sec
+
+100% tests passed, 0 tests failed out of 4
+
+Total Test time (real) =   0.69 sec
+```
+
+
+## Features matrix
+
+Compilers tested: gfortran (homebrew) 
+
+| Feature | Implemented natively | Implemented manually |
+|---------|----------------------|----------------------|
+| Can run individual tests | No | Yes, see [main.f90](./main.f90). However, this requires running the test executable directly without ctest. |
+| Mocking | No | Not implemented |
+| Stubbing | No | Not implemented |
+| Data driven tests | No | Yes, see verify_calculate_mesh_parameters and verify_calculate_mesh in [test_mesh_generator.f90](./test_mesh_generator.f90)
+| Coverage report | Yes, with fpm | N/A |
+| Skip tests | Yes, see test_skip_example in [test_calc_pi.f90](./test_mesh_generator.f90) | N/A |
+
+## Pros
+- Lightweight, procedural unit testing framework based on nothing but standard Fortran.
+- Since test-drive is written is standard fortran, any customisation we implement could be carried over to the open-source test-drive code.
+
+## Cons 
+- Slow at releasing. Last release was in 2021 (3 years ago).
+- Most standard test festures such as test paramaterisation and mocking are not provided by the test-drive library.
+- A fair amount of boiler plate code is required to get things up and running.
+
+## Building
+- `fpm` provided a very convenient way to [get started](https://fpm.fortran-lang.org/tutorial/dependencies.html#adding-a-testing-framework) with `test-drive`. However, it is unlikely that a long running Fortran project written in Fortran 95, for example, will be using `fpm`. It is more likely that `cmake` is in use.  
+- `cmake` can be used to install
+
+## Resources
+- Repository: https://github.com/fortran-lang/test-drive

testing/src/main.f90

@@ -4,7 +4,7 @@ program tester
     use testdrive, only : run_testsuite, new_testsuite, testsuite_type, &
         & select_suite, run_selected, get_argument
 
-    use test_cal_pi, only : collect_calc_pi_suite
+    use test_mesh_generator, only : collect_mesh_generator_testsuite
 
     implicit none
 
@@ -16,7 +16,7 @@ program tester
     stat = 0
 
     testsuites = [ &
-        new_testsuite("suite1", collect_calc_pi_suite) &
+        new_testsuite("mesh_generator", collect_mesh_generator_testsuite) &
         ]
 
     call get_argument(1, suite_name)
@@ -25,27 +25,27 @@ program tester
     if (allocated(suite_name)) then
         is = select_suite(testsuites, suite_name)
         if (is > 0 .and. is <= size(testsuites)) then
-        if (allocated(test_name)) then
-            write(error_unit, fmt) "Suite:", testsuites(is)%name
-            call run_selected(testsuites(is)%collect, test_name, error_unit, stat)
-            if (stat < 0) then
-            error stop 1
+            if (allocated(test_name)) then
+                write(error_unit, fmt) "Suite:", testsuites(is)%name
+                call run_selected(testsuites(is)%collect, test_name, error_unit, stat)
+                if (stat < 0) then
+                    error stop 1
+                end if
+            else
+                write(error_unit, fmt) "Testing:", testsuites(is)%name
+                call run_testsuite(testsuites(is)%collect, error_unit, stat)
             end if
         else
-            write(error_unit, fmt) "Testing:", testsuites(is)%name
-            call run_testsuite(testsuites(is)%collect, error_unit, stat)
-        end if
-        else
-        write(error_unit, fmt) "Available testsuites"
-        do is = 1, size(testsuites)
-            write(error_unit, fmt) "-", testsuites(is)%name
-        end do
-        error stop 1
+            write(error_unit, fmt) "Available testsuites"
+            do is = 1, size(testsuites)
+                write(error_unit, fmt) "-", testsuites(is)%name
+            end do
+            error stop 1
         end if
     else
         do is = 1, size(testsuites)
-        write(error_unit, fmt) "Testing:", testsuites(is)%name
-        call run_testsuite(testsuites(is)%collect, error_unit, stat)
+            write(error_unit, fmt) "Testing:", testsuites(is)%name
+            call run_testsuite(testsuites(is)%collect, error_unit, stat)
         end do
     end if