Clean up & improve execution tests (#512)

* Clean up execution test files

* Start ursim as part of the execution tests

* Dont use custom dockerursim for humble and rolling execution tests

* Clean up test implementations

* pep257 fixes

* Perform rolling and humble execution tests as part of normal pipelines

* Increase admissable timeouts as the CI needs to pull ursim first

* Add more debug messages during tests

* Wait until robot is in POWER_OFF mode before trying to power it on

* Fix error introduced in last commit

* Add additional cmake option to enable integration tests

* Increase timeout for robot tests

* Add CMake comment describing the execution test integration

* Run source tests on pull request

This is only here for testing the test setup! Remove before merging

* call resend_robot_program twice

This seems to be necessary, as otherwise the robot hangs after bootup.
The first program execution (that gets automatically started at driver
startup because of the headless_mode) gets paused, since it is sent while
the robotis not yet switched on. To mitigate this, we send the robot program
again after switching on the robot, but this seems to stop the robot program.
Sending it again seems to set it correctly to a started state.

* Increase timeouts for dashboard_client tests

Otherwise they can fail, since in parallel we pull and start the docker
container.

Co-authored-by: Felix Exner <[email protected]>

Author: RobertWilbrandt

.github/workflows/humble-binary-build.yml

@@ -15,13 +15,14 @@ jobs:
         ROS_REPO: [main, testing]
     env:
       UPSTREAM_WORKSPACE: Universal_Robots_ROS2_Driver-not-released.${{ matrix.ROS_DISTRO }}.repos
-      NOT_TEST_BUILD: true
-      NOT_TEST_DOWNSTREAM: true
+      DOCKER_RUN_OPTS: '-v /var/run/docker.sock:/var/run/docker.sock --network ursim_net'
     steps:
       - uses: actions/checkout@v1
         with:
           ref: humble
+      - run: docker network create --subnet=192.168.56.0/24 ursim_net
       - uses: 'ros-industrial/industrial_ci@master'
         env:
           ROS_DISTRO: ${{ matrix.ROS_DISTRO }}
           ROS_REPO: ${{ matrix.ROS_REPO }}
+          CMAKE_ARGS: -DUR_ROBOT_DRIVER_BUILD_INTEGRATION_TESTS=ON

.github/workflows/humble-execution-test.yml

@@ -15,13 +15,14 @@ jobs:
         ROS_REPO: [main, testing]
     env:
       UPSTREAM_WORKSPACE: Universal_Robots_ROS2_Driver.${{ matrix.ROS_DISTRO }}.repos
-      NOT_TEST_BUILD: true
-      NOT_TEST_DOWNSTREAM: true
+      DOCKER_RUN_OPTS: '-v /var/run/docker.sock:/var/run/docker.sock --network ursim_net'
     steps:
       - uses: actions/checkout@v1
         with:
           ref: humble
+      - run: docker network create --subnet=192.168.56.0/24 ursim_net
       - uses: 'ros-industrial/industrial_ci@master'
         env:
           ROS_DISTRO: ${{ matrix.ROS_DISTRO }}
           ROS_REPO: ${{ matrix.ROS_REPO }}
+          CMAKE_ARGS: -DUR_ROBOT_DRIVER_BUILD_INTEGRATION_TESTS=ON

.github/workflows/humble-semi-binary-build.yml

@@ -21,11 +21,12 @@ jobs:
         ROS_REPO: [main, testing]
     env:
       UPSTREAM_WORKSPACE: Universal_Robots_ROS2_Driver-not-released.${{ matrix.ROS_DISTRO }}.repos
-      NOT_TEST_BUILD: true
-      NOT_TEST_DOWNSTREAM: true
+      DOCKER_RUN_OPTS: '-v /var/run/docker.sock:/var/run/docker.sock --network ursim_net'
     steps:
       - uses: actions/checkout@v1
+      - run: docker network create --subnet=192.168.56.0/24 ursim_net
       - uses: 'ros-industrial/industrial_ci@master'
         env:
           ROS_DISTRO: ${{ matrix.ROS_DISTRO }}
           ROS_REPO: ${{ matrix.ROS_REPO }}
+          CMAKE_ARGS: -DUR_ROBOT_DRIVER_BUILD_INTEGRATION_TESTS=ON

.github/workflows/rolling-binary-build.yml

@@ -21,11 +21,12 @@ jobs:
         ROS_REPO: [main, testing]
     env:
       UPSTREAM_WORKSPACE: Universal_Robots_ROS2_Driver.${{ matrix.ROS_DISTRO }}.repos
-      NOT_TEST_BUILD: true
-      NOT_TEST_DOWNSTREAM: true
+      DOCKER_RUN_OPTS: '-v /var/run/docker.sock:/var/run/docker.sock --network ursim_net'
     steps:
       - uses: actions/checkout@v1
+      - run: docker network create --subnet=192.168.56.0/24 ursim_net
       - uses: 'ros-industrial/industrial_ci@master'
         env:
           ROS_DISTRO: ${{ matrix.ROS_DISTRO }}
           ROS_REPO: ${{ matrix.ROS_REPO }}
+          CMAKE_ARGS: -DUR_ROBOT_DRIVER_BUILD_INTEGRATION_TESTS=ON

.github/workflows/rolling-execution-test.yml

@@ -1,5 +1,8 @@
 name: Rolling Source Build
 on:
+  pull_request:
+    branches:
+      - main
   push:
     branches:
       - main
@@ -37,7 +40,7 @@ jobs:
             https://raw.githubusercontent.com/ros2/ros2/master/ros2.repos
             https://raw.githubusercontent.com/${{ github.repository }}/${{ github.sha }}/Universal_Robots_ROS2_Driver.${{ env.ROS_DISTRO }}.repos
           colcon-mixin-repository: https://raw.githubusercontent.com/colcon/colcon-mixin-repository/master/index.yaml
-          skip-tests: true
+          extra-cmake-args: -DUR_ROBOT_DRIVER_BUILD_INTEGRATION_TESTS=ON
       - uses: actions/upload-artifact@v1
         with:
           name: colcon-logs-${{ matrix.os }}

.github/workflows/rolling-semi-binary-build.yml

@@ -89,29 +89,6 @@ Check also [presentations and videos](ur_robot_driver/doc/resources/README.md) a
       </a>
     </td>
   </tr>
-  <tr>
-    <th>Execution Test</th>
-    <td>&nbsp;</td>
-    <td>
-      <a href="https://github.com/UniversalRobots/Universal_Robots_ROS2_Driver/actions/workflows/galactic-execution-test.yml?query=branch%3Amain+">
-         <img src="https://github.com/UniversalRobots/Universal_Robots_ROS2_Driver/actions/workflows/galactic-execution-test.yml/badge.svg?branch=main"
-              alt="Execution Testing"/>
-      </a>
-   </td>
-   <td>
-      <a href="https://github.com/UniversalRobots/Universal_Robots_ROS2_Driver/actions/workflows/humble-execution-test.yml?query=branch%3Amain+">
-         <img src="https://github.com/UniversalRobots/Universal_Robots_ROS2_Driver/actions/workflows/humble-execution-test.yml/badge.svg?branch=main"
-              alt="Execution Testing"/>
-      </a>
-   </td>
-   <td>
-      <a href="https://github.com/UniversalRobots/Universal_Robots_ROS2_Driver/actions/workflows/rolling-execution-test.yml?query=branch%3Amain+">
-         <img src="https://github.com/UniversalRobots/Universal_Robots_ROS2_Driver/actions/workflows/rolling-execution-test.yml/badge.svg?branch=main"
-              alt="Execution Testing"/>
-      </a>
-   </td>
-
-  </tr>
 </table>
 
 
@@ -129,6 +106,11 @@ Check also [presentations and videos](ur_robot_driver/doc/resources/README.md) a
 
 1. Source build - also core ROS packages are build from source. It shows potential issues in the mid future.
 
+Each of these stages also performs integration tests using ursim. In order to execute these tests locally, they have to be enabled:
+  ```
+  colcon build --packages-select ur_robot_driver --cmake-args -DUR_ROBOT_DRIVER_BUILD_INTEGRATION_TESTS=On
+  ```
+
 
 ## Packages in the Repository:
 

.github/workflows/rolling-source-build.yml

@@ -1,6 +1,14 @@
 cmake_minimum_required(VERSION 3.5)
 project(ur_robot_driver)
 
+# Default to off as this only works in environments where a new docker container can be started with the appropriate networking, which
+# might not be possible e.g. inside the buildfarm
+option(
+  UR_ROBOT_DRIVER_BUILD_INTEGRATION_TESTS
+  "Build integration tests using the start_ursim script"
+  OFF
+)
+
 add_compile_options(-Wall)
 add_compile_options(-Wextra)
 add_compile_options(-Wno-unused-parameter)
@@ -156,6 +164,15 @@ if(BUILD_TESTING)
   find_package(ur_description REQUIRED)
   find_package(ur_msgs REQUIRED)
   find_package(launch_testing_ament_cmake)
-  add_launch_test(test/integration_test_1.py)
-  add_launch_test(test/integration_test_2.py)
+
+  if(${UR_ROBOT_DRIVER_BUILD_INTEGRATION_TESTS})
+    add_launch_test(test/dashboard_client.py
+      TIMEOUT
+        180
+    )
+    add_launch_test(test/robot_driver.py
+      TIMEOUT
+        500
+    )
+  endif()
 endif()

README.md

@@ -58,6 +58,7 @@
   <exec_depend>velocity_controllers</exec_depend>
   <exec_depend>xacro</exec_depend>
 
+  <test_depend>docker.io</test_depend>
   <test_depend>launch_testing_ament_cmake</test_depend>
 
   <export>