Large refactor of test framework

Makes many changes to test infrastructure:
* Split test_util into separate files for maintainability
* Move new files into tests/framework/util
* Move FolderManager into separate file from test_environment
* Rename COMMON_UNIX_PLATFORMS to TESTING_COMMON_UNIX_PLATFORMS
* Rewrote unix path redirection logic, relies on FileSystemManager
  * Relative paths are actually relative now
* Split unsecured folder location into unsecured driver/implicit layer/explicit layer/settings
* Fix bugs in tests that are attributable to old path redirection logic.
* Use normalized paths in test framework - ie normalize paths so they can be compared easily
* Make running with elevated privileges a framework level setting
* Split tests that change elevated privileges into separate tests
* Framework can set home & xdg env-vars at startup
* Platform specific folders are only defined on those platforms
* PlatformShim::add_manifest is now windows only, reflecting that only windows implemented it

Author: charles-lunarg

scripts/generators/dispatch_table_helper_generator.py

@@ -60,7 +60,7 @@ def generate(self):
 #include <vulkan/vulkan.h>
 #include <vulkan/vk_layer.h>
 #include <string.h>
-#include "vk_layer_dispatch_table.h"
+#include "loader/generated/vk_layer_dispatch_table.h"
 
 ''')
 

tests/CMakeLists.txt

@@ -68,6 +68,8 @@ endif()
 
 option(ENABLE_LIVE_VERIFICATION_TESTS "Enable tests which expect to run on live drivers. Meant for manual verification only" OFF)
 
+set(TEST_EXECUTION_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/executing_tests")
+
 include(GoogleTest)
 add_subdirectory(framework)
 
@@ -94,7 +96,11 @@ add_executable(
         loader_testing_main.cpp
         loader_fuzz_tests.cpp
 )
-target_link_libraries(test_fuzzing PUBLIC testing_dependencies vulkan)
+target_link_libraries(test_fuzzing PUBLIC testing_dependencies)
+# testing_dependencies already links to vulkan when linking statically
+if (NOT APPLE_STATIC_LOADER)
+    target_link_libraries(test_fuzzing PUBLIC vulkan)
+endif()
 target_include_directories(test_fuzzing PUBLIC ${CMAKE_SOURCE_DIR}/loader ${CMAKE_SOURCE_DIR}/loader/generated)
 
 # Threading tests live in separate executabe just for threading tests as it'll need support

tests/framework/CMakeLists.txt

@@ -15,38 +15,15 @@
 # limitations under the License.
 # ~~~
 
-add_library(testing_framework_util STATIC test_util.cpp)
-target_link_libraries(testing_framework_util PUBLIC loader_common_options Vulkan::Headers)
-
-if(UNIX OR APPLE)
-    target_link_libraries(testing_framework_util PUBLIC ${CMAKE_DL_LIBS})
-endif()
-
-if(UNIX)
-    target_compile_options(testing_framework_util PUBLIC -fPIC)
-endif()
-# Gives access to all headers in the framework folder, in the framework binary, and in the whole project (mainly for loader/generated)
-target_include_directories(testing_framework_util PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" "${PROJECT_SOURCE_DIR}" "${PROJECT_SOURCE_DIR}/loader/generated")
+#configure the framework_config.h.in file - used to locate all the binaries generated so that it can be used in the tests
+#setup framework_config_temp.h.in in the current binary directory
+configure_file("${CMAKE_CURRENT_SOURCE_DIR}/framework_config.h.in" "${CMAKE_CURRENT_BINARY_DIR}/framework_config_temp.h.in")
 
-if (UNIX)
-    if (LOADER_ENABLE_ADDRESS_SANITIZER)
-        target_compile_options(testing_framework_util PUBLIC -fsanitize=address,undefined)
-        target_link_options(testing_framework_util PUBLIC -fsanitize=address,undefined)
-    endif()
-    if (LOADER_ENABLE_THREAD_SANITIZER)
-        target_compile_options(testing_framework_util PUBLIC -fsanitize=thread)
-        target_link_options(testing_framework_util PUBLIC -fsanitize=thread)
-        target_compile_options(gtest PUBLIC -fsanitize=thread)
-        target_link_options(gtest PUBLIC -fsanitize=thread)
-    endif()
-endif()
+# setup framework_config_$<CONFIG> using framework_config_temp.h.in as a source
+file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/framework_config_$<CONFIG>.h" INPUT "${CMAKE_CURRENT_BINARY_DIR}/framework_config_temp.h.in")
 
-if (MSVC)
-    # silence hidden class member warnings in test framework
-    target_compile_options(testing_framework_util PUBLIC /wd4458)
-    # Make sure exception handling is enabled for the test framework
-    target_compile_options(testing_framework_util PUBLIC /EHsc)
-endif()
+# Create a variable to hold the path to the correct header file, used as a compiler definition in the utils target
+set(FRAMEWORK_CONFIG_HEADER_PATH "${CMAKE_CURRENT_BINARY_DIR}/framework_config_$<CONFIG>.h")
 
 function(AddSharedLibrary LIBRARY_NAME)
     set(singleValueArgs DEF_FILE)
@@ -63,26 +40,14 @@ function(AddSharedLibrary LIBRARY_NAME)
     endif()
 endfunction()
 
+add_subdirectory(util)
 add_subdirectory(data)
 add_subdirectory(shim)
 add_subdirectory(icd)
 add_subdirectory(layer)
 
-#configure the framework_config.h.in file - used to locate all the binaries generated so that it can be used in the tests
-#setup framework_config_temp.h.in in the current binary directory
-configure_file("${CMAKE_CURRENT_SOURCE_DIR}/framework_config.h.in" "${CMAKE_CURRENT_BINARY_DIR}/framework_config_temp.h.in")
-
-# setup framework_config_$<CONFIG> using framework_config_temp.h.in as a source
-file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/framework_config_$<CONFIG>.h" INPUT "${CMAKE_CURRENT_BINARY_DIR}/framework_config_temp.h.in")
-
-# Add a compiler definition for the path to framework_config.h with the correct config
-target_compile_definitions(testing_framework_util PUBLIC FRAMEWORK_CONFIG_HEADER="framework_config_$<CONFIG>.h")
-
 add_library(testing_dependencies STATIC test_environment.cpp test_environment.h)
 target_link_libraries(testing_dependencies
     PUBLIC gtest Vulkan::Headers testing_framework_util shim-library)
 target_include_directories(testing_dependencies PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
-if (APPLE_STATIC_LOADER)
-    target_compile_definitions(testing_dependencies PUBLIC "APPLE_STATIC_LOADER=1")
-    target_link_libraries(testing_dependencies PUBLIC vulkan)
-endif()
+

tests/framework/README.md

@@ -136,8 +136,7 @@ There are many utilities that the test framework and tests have access to. These
 * Environment Variable Wrapper: `EnvVarWrapper` for creating, setting, getting, and removing environment variables in a RAII manner
 * Windows API error handling helpers
 * filesystem abstractions:
-  * `create_folder`/`delete_folder`
-  * `FolderManager`
+  * `Folder`
     * Creates a new folder with the given name at construction time.
     * Allows writing manifests and files (eg, icd or layer binaries)
     * Automatically destroys the folder and all contained files at destruction

tests/framework/framework_config.h.in

@@ -26,7 +26,7 @@
  */
 
 #pragma once
-#define FRAMEWORK_BUILD_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+#define TEST_EXECUTION_DIRECTORY "${TEST_EXECUTION_DIRECTORY}"
 
 #define FRAMEWORK_VULKAN_LIBRARY_PATH "$<TARGET_FILE:vulkan>"
 

tests/framework/icd/test_icd.cpp

@@ -29,6 +29,13 @@
 
 #include "test_icd.h"
 
+#include <assert.h>
+#include <cstring>
+
+#include <iostream>
+
+#include "equality_helpers.h"
+
 // export vk_icdGetInstanceProcAddr
 #if !defined(TEST_ICD_EXPORT_ICD_GIPA)
 #define TEST_ICD_EXPORT_ICD_GIPA 0

tests/framework/icd/test_icd.h

@@ -27,7 +27,13 @@
 
 #pragma once
 
-#include "test_util.h"
+#include <array>
+#include <filesystem>
+#include <ostream>
+
+#include "util/dispatchable_handle.h"
+#include "util/platform_wsi.h"
+#include "util/functions.h"
 
 #include "layer/layer_util.h"
 
@@ -70,6 +76,9 @@ inline std::ostream& operator<<(std::ostream& os, const InterfaceVersionCheck& r
     }
     return os << static_cast<uint32_t>(result);
 }
+
+using VulkanUUID = std::array<uint8_t, VK_UUID_SIZE>;
+
 // clang-format on
 
 // Move only type because it holds a DispatchableHandle<VkPhysicalDevice>

tests/framework/json_writer.h

@@ -28,7 +28,7 @@
 #include <vulkan/vulkan.h>
 #include <vulkan/vk_layer.h>
 #include <string.h>
-#include "vk_layer_dispatch_table.h"
+#include "loader/generated/vk_layer_dispatch_table.h"
 
 // clang-format off
 static inline void layer_init_device_dispatch_table(VkDevice device, VkLayerDispatchTable *table, PFN_vkGetDeviceProcAddr gpa) {

tests/framework/layer/generated/vk_dispatch_table_helper.h

@@ -27,7 +27,9 @@
 
 #pragma once
 
-#include "test_util.h"
+#include "builder_defines.h"
+
+#include "vulkan_object_wrappers.h"
 
 struct LayerDefinition {
     BUILDER_VALUE(std::string, layerName)
@@ -38,10 +40,10 @@ struct LayerDefinition {
 
     VkLayerProperties get() const noexcept {
         VkLayerProperties props{};
-        copy_string_to_char_array(layerName, &props.layerName[0], VK_MAX_EXTENSION_NAME_SIZE);
+        layerName.copy(props.layerName, VK_MAX_EXTENSION_NAME_SIZE);
         props.specVersion = specVersion;
         props.implementationVersion = implementationVersion;
-        copy_string_to_char_array(description, &props.description[0], VK_MAX_DESCRIPTION_SIZE);
+        description.copy(props.description, VK_MAX_DESCRIPTION_SIZE);
         return props;
     }
 };