Merge pull request #109 from bbockelm/posc

Create a "Persist on Successful Close" plugin

Author: bbockelm

.github/workflows/test.yml

@@ -33,29 +33,31 @@ jobs:
       with:
         go-version: '1.23.5'
     - name: install deps
+      working-directory: ${{runner.workspace}}
       run: |
-        sudo curl -L https://xrootd.web.cern.ch/repo/RPM-GPG-KEY.txt -o /etc/apt/trusted.gpg.d/xrootd.asc
-        sudo /bin/sh -c 'echo "deb https://xrootd.web.cern.ch/ubuntu noble stable" >> /etc/apt/sources.list.d/xrootd.list'
-        sudo apt update && sudo apt-get install -y cmake libcurl4-openssl-dev libcurl4 pkg-config libssl-dev xrootd-server libxrootd-dev libxrootd-server-dev libgtest-dev
-        sudo curl -L https://dl.min.io/server/minio/release/linux-amd64/minio -o /usr/local/bin/minio
-        sudo chmod +x /usr/local/bin/minio
-        sudo curl -L https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc
-        sudo chmod +x /usr/local/bin/mc
+        # Build deps
+        sudo apt update && sudo apt-get install -y cmake libz-dev uuid-dev libcurl4-openssl-dev libcurl4 pkg-config libssl-dev g++ libscitokens-dev libgtest-dev
+
+        # Build our preferred set of patches on xrootd
+        git clone https://github.com/xrootd/xrootd.git
+        cd xrootd
+        git remote add github_pelican https://github.com/PelicanPlatform/xrootd.git
+        git fetch github_pelican
+        git checkout -b v5.8.4-pelican -t github_pelican/v5.8.4-pelican
+        mkdir -p build/release_dir
+        cd build
+        cmake .. -DCMAKE_INSTALL_PREFIX=$PWD/release_dir -DENABLE_ASAN=TRUE
+        make -j $(($(nproc) + 2)) install
 
     - name: Create Build Environment
       # Some projects don't allow in-source building, so create a separate build directory
       # We'll use this as our working directory for all subsequent commands
       run: cmake -E make_directory ${{runner.workspace}}/build
 
     - name: Configure CMake
-      # Use a bash shell so we can use the same syntax for environment variable
-      # access regardless of the host operating system
       shell: bash
       working-directory: ${{runner.workspace}}/build
-      # Note the current convention is to use the -S and -B options here to specify source
-      # and build directories, but this is only available with CMake 3.13 and higher.
-      # The CMake binaries on the Github Actions machines are (as of this writing) 3.12
-      run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_TESTING=yes -DXROOTD_PLUGINS_EXTERNAL_GTEST=${{ matrix.external-gtest }}
+      run: CMAKE_PREFIX_PATH=$PWD/../xrootd/build/release_dir/lib/cmake/XRootD cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DENABLE_TESTS=yes -DENABLE_ASAN=true -DXROOTD_PLUGINS_EXTERNAL_GTEST=${{ matrix.external-gtest }}
 
     - name: Build
       working-directory: ${{runner.workspace}}/build
@@ -73,7 +75,7 @@ jobs:
     - name: Start xrootd
       working-directory: ${{runner.workspace}}/build
       shell: bash
-      run: xrootd -c ${{runner.workspace}}/xrootd-s3-http/test/s3-xrootd-test.cfg &
+      run: ASAN_OPTIONS=detect_odr_violation=0 LD_LIBRARY_PATH=$PWD/../xrootd/build/release_dir/lib $PWD/../xrootd/build/release_dir/bin/xrootd -c ${{runner.workspace}}/xrootd-s3-http/test/s3-xrootd-test.cfg &
 
     - name: Get a file
       working-directory: ${{runner.workspace}}/build

CMakeLists.txt

@@ -4,7 +4,7 @@ project( xrootd-http/s3 )
 
 option( XROOTD_PLUGINS_EXTERNAL_GTEST "Use an external/pre-installed copy of GTest" OFF )
 option( VALGRIND "Run select unit tests under valgrind" OFF )
-option( ASAN "Build the plugin with the address sanitizer" OFF )
+option( ENABLE_ASAN "Build the plugin with the address sanitizer" OFF )
 
 set( CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake )
 if( "${CMAKE_BUILD_TYPE}" STREQUAL "" )
@@ -20,7 +20,7 @@ if(VALGRIND)
   find_program(VALGRIND_BIN valgrind REQUIRED)
 endif()
 
-if(ASAN)
+if( ENABLE_ASAN )
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address")
   set(CMAKE_LINKER_FLAGS "${CMAKE_LINKER_FLAGS} -fsanitize=address")
@@ -146,6 +146,16 @@ target_link_libraries(XrdOssGlobusObj ${XRootD_UTILS_LIBRARIES} ${XRootD_SERVER_
 add_library(XrdOssGlobus MODULE "$<TARGET_OBJECTS:XrdOssGlobusObj>")
 target_link_libraries(XrdOssGlobus XrdOssGlobusObj)
 
+#######################
+## libXrdOssPosc     ##
+#######################
+add_library( XrdOssPoscObj OBJECT src/Posc.cc )
+set_target_properties( XrdOssPoscObj PROPERTIES POSITION_INDEPENDENT_CODE ON )
+target_link_libraries( XrdOssPoscObj XRootD::XrdServer XRootD::XrdUtils )
+
+add_library( XrdOssPosc MODULE "$<TARGET_OBJECTS:XrdOssPoscObj>" )
+target_link_libraries( XrdOssPosc XrdOssPoscObj )
+
 # Customize module's suffix and, on Linux, hide unnecessary symbols
 if( APPLE )
   set_target_properties( XrdS3 PROPERTIES OUTPUT_NAME "XrdS3-${XRootD_PLUGIN_VERSION}" SUFFIX ".so" )
@@ -154,23 +164,25 @@ if( APPLE )
   set_target_properties( XrdOssHttp PROPERTIES OUTPUT_NAME "XrdOssHttp-${XRootD_PLUGIN_VERSION}" SUFFIX ".so" )
   set_target_properties( XrdOssFilter PROPERTIES OUTPUT_NAME "XrdOssFilter-${XRootD_PLUGIN_VERSION}" SUFFIX ".so" )
   set_target_properties( XrdOssGlobus PROPERTIES OUTPUT_NAME "XrdOssGlobus-${XRootD_PLUGIN_VERSION}" SUFFIX ".so" )
+  set_target_properties( XrdOssPosc PROPERTIES OUTPUT_NAME "XrdOssPosc-${XRootD_PLUGIN_VERSION}" SUFFIX ".so" )
 else()
   set_target_properties( XrdS3 PROPERTIES OUTPUT_NAME "XrdS3-${XRootD_PLUGIN_VERSION}" SUFFIX ".so" LINK_FLAGS "-Wl,--version-script=${CMAKE_SOURCE_DIR}/configs/export-lib-symbols" )
   set_target_properties( XrdHTTPServer PROPERTIES OUTPUT_NAME "XrdHTTPServer-${XRootD_PLUGIN_VERSION}" SUFFIX ".so" LINK_FLAGS "-Wl,--version-script=${CMAKE_SOURCE_DIR}/configs/export-lib-symbols" )
   set_target_properties( XrdOssS3 PROPERTIES OUTPUT_NAME "XrdOssS3-${XRootD_PLUGIN_VERSION}" SUFFIX ".so" LINK_FLAGS "-Wl,--version-script=${CMAKE_SOURCE_DIR}/configs/export-lib-symbols" )
   set_target_properties( XrdOssHttp PROPERTIES OUTPUT_NAME "XrdOssHttp-${XRootD_PLUGIN_VERSION}" SUFFIX ".so" LINK_FLAGS "-Wl,--version-script=${CMAKE_SOURCE_DIR}/configs/export-lib-symbols" )
   set_target_properties( XrdOssFilter PROPERTIES OUTPUT_NAME "XrdOssFilter-${XRootD_PLUGIN_VERSION}" SUFFIX ".so" LINK_FLAGS "-Wl,--version-script=${CMAKE_SOURCE_DIR}/configs/export-lib-symbols" )
   set_target_properties( XrdOssGlobus PROPERTIES OUTPUT_NAME "XrdOssGlobus-${XRootD_PLUGIN_VERSION}" SUFFIX ".so" LINK_FLAGS "-Wl,--version-script=${CMAKE_SOURCE_DIR}/configs/export-lib-symbols" )
+  set_target_properties( XrdOssPosc PROPERTIES OUTPUT_NAME "XrdOssPosc-${XRootD_PLUGIN_VERSION}" SUFFIX ".so" LINK_FLAGS "-Wl,--version-script=${CMAKE_SOURCE_DIR}/configs/export-lib-symbols" )
 endif()
 
 include(GNUInstallDirs)
 
 install(
-  TARGETS XrdS3 XrdHTTPServer XrdOssS3 XrdOssHttp XrdOssFilter XrdOssGlobus
+  TARGETS XrdS3 XrdHTTPServer XrdOssS3 XrdOssHttp XrdOssFilter XrdOssGlobus XrdOssPosc
   LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
 )
 
-if( BUILD_TESTING )
+if( ENABLE_TESTS )
   # Create shared libraries for testing from the existing objects
   add_library(XrdS3Testing SHARED "$<TARGET_OBJECTS:XrdS3Obj>")
   target_link_libraries(XrdS3Testing XrdS3Obj)
@@ -188,6 +200,10 @@ if( BUILD_TESTING )
   target_link_libraries(XrdOssGlobusTesting XrdOssGlobusObj)
   target_include_directories(XrdOssGlobusTesting INTERFACE ${XRootD_INCLUDE_DIRS})
 
+  add_library( XrdOssPoscTesting SHARED "$<TARGET_OBJECTS:XrdOssPoscObj>" )
+  target_link_libraries( XrdOssPoscTesting XrdOssPoscObj )
+  target_include_directories( XrdOssPoscTesting INTERFACE ${XRootD_INCLUDE_DIRS} )
+
   find_program(GoWrk go-wrk HINTS "$ENV{HOME}/go/bin")
   if( NOT GoWrk )
     # Try installing the go-wrk variable to generate a reasonable stress test

README.md

@@ -1,6 +1,17 @@
+# Pelican filesystem plugins for XRootD
+This repository contains various plugins for [XRootD](https://github.com/xrootd/xrootd)'s "Open Storage System" (OSS) layer,
+which affects how XRootD serves objects from storage.
+
+The plugins in the repository include:
+- `XrdOssHttp`: Exposes a backend HTTP(S) server as a backend storage.
+- `XrdOssS3`: Exposes a backend S3-compatible interface as a backend storage.
+- `XrdOssFilter`: A "stacking" plugin (meant to be loaded on top of another OSS plugin)
+  that exports only files/directories matching a given list of Unix globs.
+- `XrdOssPosc`: A "stacking" plugin that causes in-progress files being written from being
+  visible to the filesystem and to be deleted if not successfully closed.  Here, "POSC"
+  stands for "Persist on Successful Close" and is similar in spirit to the POSC functionality
+  in the core XRootD (with the addition of making in-progress files not-visible in the namespace).
 
-# S3/HTTP filesystem plugins for XRootD
-These filesystem plugins for [XRootD](https://github.com/xrootd/xrootd) allow you to serve objects from S3 and HTTP backends through an XRootD server.
 
 ## Building and Installing
 Assuming XRootD, CMake>=3.13 and gcc>=8 are already installed, run:
@@ -18,28 +29,25 @@ make
 If building XRootD from source instead, add `-DXROOTD_DIR` to the CMake command line
 to point it at the installed directory.
 
+Dependency packages (tinyxml, gtest, and minio/mc for tests) are automatically downloaded
+if they are not present in the build environment.
+
 ### Building with Tests
 
 Unit tests for this repository require `gtest`, which is included as a submodule of this repo. The tests can be compiled with a slight modification to your build command:
 
 ```
 mkdir build
 cd build
-cmake -DXROOTD_PLUGINS_BUILD_UNITTESTS=ON ..
+cmake -DBUILD_TESTING=ON ..
 make
 ```
 
-This creates the directory `build/test` with two unit test executables that can be run:
-- `build/test/s3-gtest`
-- `build/test/http-gtest`
+To run the test framework, execute `ctest` from the build directory:
 
-Alternatively, `gtest` can be installed externally. For example, on RHEL-based linux distributions:
-
-```bash
-dnf install gtest
 ```
-
-Add `-DXROOTD_PLUGINS_EXTERNAL_GTEST=ON` to your `cmake` command if you're using an external installation.
+ctest
+```
 
 ## Configuration
 
@@ -48,7 +56,7 @@ Add `-DXROOTD_PLUGINS_EXTERNAL_GTEST=ON` to your `cmake` command if you're using
 To configure the HTTP server plugin, add the following line to the Xrootd configuration file:
 
 ```
-ofs.osslib </path/to/libXrdHTTPServer.so>
+ofs.osslib libXrdOssHttp.so
 ```
 
 Here's a minimal config file
@@ -64,9 +72,9 @@ xrd.protocol http:1094 libXrdHttp.so
 all.export  </exported/server/prefix>
 
 # Setting up HTTP plugin
-ofs.osslib libXrdHTTPServer.so
-# Use this if libXrdHTTPServer.so is in a development directory
-# ofs.osslib /path/to/libXrdHTTPServer.so
+ofs.osslib libXrdOssHttp.so
+# Use this if libXrdOssHttp.so is in a development directory
+# ofs.osslib /path/to/libXrdOssHttp.so
 
 # Upon last testing, the plugin did not yet work in async mode
 xrootd.async off
@@ -81,7 +89,7 @@ httpserver.host_url <host url>
 To configure the S3 plugin, add the following line to the Xrootd configuration file:
 
 ```
-ofs.osslib </path/to/libXrdS3.so>
+ofs.osslib libXrdOssS3.so
 ```
 
 Here's a minimal config file
@@ -96,11 +104,11 @@ xrd.protocol http:1094 libXrdHttp.so
 all.export  </exported/server/prefix>
 
 # Setting up S3 plugin
-ofs.osslib libXrdS3.so
-# Use this if libXrdS3.so is in a development directory
-# ofs.osslib /path/to/libXrdS3.so
+ofs.osslib libXrdOssS3.so
+# Use this if libXrdOssS3.so is in a development directory
+# ofs.osslib /path/to/libXrdOssS3.so
 
-# Upon last testing, the plugin did not yet work in async mode
+# The plugin does not support plugin mode
 xrootd.async off
 
 #example url
@@ -148,34 +156,145 @@ s3.trace debug
 
 ```
 
+### Configure the filter plugin
 
-## Startup and Testing
+The filter plugin allows you to provide multiple globs to export specific
+files and directories inside a filesystem (allowing some to be made inaccessible).
 
-### HTTP Server Backend
+Note this only affects the namespace seen by XRootD.  A user that can create symlinks
+on the filesystem outside XRootD can expose hidden parts of the namespace via a specially
+crafted symlink.
 
-Assuming you named the config file `xrootd-http.cfg`, as a non-rootly user run:
+To load, invoke `ofs.osslib` in "stacking" mode:
 
 ```
-xrootd -d -c xrootd-http.cfg
+ofs.osslib ++ libXrdOssFilter.so
 ```
 
-In a separate terminal, run
+(an absolute path may be given if `libXrdOssFilter-5.so` does not reside in a system directory)
+
+There are three configuration commands for the filter module:
 
 ```
-curl -v http://localhost:1094/<host name>/<URL path to object>
+filter.trace [all|error|warning|info|debug|none]
+filter.glob [-a] glob1 [glob2] [...]
+filter.prefix prefix1 [prefix2] [...]
 ```
 
-### S3 Server Backend
-Startup and Testing
+ - `filter.trace`: Controls the logging verbosity of the module.  Can be specified multiple times
+   (values are additive) and multiple values can be given per line.  Example:
+
+   ```
+   filter.trace info
+   ```
+
+   The default level is `warning`.
+ - `filter.glob`: Controls the visibility of files in the storage; only files matching one of the
+   configured globs or prefixes can be accessed.  Can be specified multiple times (values are additive)
+   and multiple globs can be given per line.
 
-Assuming you named the config file `xrootd-s3.cfg`, as a non-rootly user run:
+   The `-a` flag indicates that a wildcard should match all filenames, including those prefixed with a
+   `.` character.  Otherwise, such "dotfiles" are not visible.
+
+   The glob language supported by the platform's `fnmatch` C runtime function is used; additionally, the
+   globstar operator (`**`) matches any names in zero-or-more directory hierarchies
+
+   Example:
+
+   ```
+   filter.glob /foo/*/*.txt /bar/**/*.csv
+   ```
+
+   With the above configuration, the files `/foo/1/test.txt` and `/bar/2/3/data.csv` would be visible
+   but the files `/foo/4/5/test.txt` and `/bar/test.txt` would not.
+ - `filter.prefix`: Controls the prefixes exported.  Every file or directory under the provided prefix
+   will be visible.  Can be specified multiple times (values are additive) and multiple globs can be
+   given per line.
+
+   Example:
+
+   ```
+   filter.prefix /foo /bar
+   ```
+
+   The above example would be equivalent to setting:
+
+   ```
+   filter.glob -a /foo/** /bar/**
+   ```
+
+### Configure the POSC (Persist on Successful Close) plugin
+
+The POSC plugin allows you to make files being uploaded into the storage invisible
+from readers until they have been successfully closed.  If the writer never calls
+`close()` or the server crashes mid-transfer, the files will be deleted.
+
+Note the visibility only affects the namespace seen by XRootD.  A user that can create symlinks
+on the filesystem outside XRootD can expose the in-progress files if desired (but may not be
+able to read them, depending on the filesystem permissions set).
+
+To load, invoke `ofs.osslib` in "stacking" mode:
 
 ```
-xrootd -d -c xrootd-s3.cfg
+ofs.osslib ++ libXrdOssPosc.so
+```
+
+(an absolute path may be given if `libXrdOssPosc-5.so` does not reside in a system directory)
+
+There are three configuration commands for the POSC module:
+
+```
+posc.trace [all|error|warning|info|debug|none]
+posc.prefix posc_directory
+```
+
+ - `posc.trace`: Controls the logging verbosity of the module.  Can be specified multiple times
+   (values are additive) and multiple values can be given per line.  Example:
+
+   ```
+   filter.trace info
+   ```
+
+   The default level is `warning`.
+
+ - `posc.prefix`: Controls the directory where in-progress files will be written.  Once completed,
+   they will be renamed into the final location.  If this is not specified, the module will fail
+   to start; if specified multiple times, the last value wins.  Example:
+
+   ```
+   posc.prefix /in-progress
+   ```
+
+   The `posc.prefix` directory is not exported into the namespace; users will not be able to list
+   its contents.
+
+Files in the `posc.prefix` directory will have the following structure:
+
+```
+/$(posc.prefix)/$(username)/in-progress.$(timestamp).$(random)
+```
+
+where `$(username)` is the username as determined by the security framework (`anonymous` if unset).
+The user directory and files will be created with the user's credential as possible.  If using
+the multi-user plugin, this the `posc.prefix` directory should be world writable with the "sticky bit"
+set to allow any user to create a subdirectory (but not allow users to delete other directories).
+
+XRootD will never perform filesystem operations as root; it will not create as root and "chown" to the
+user.
+
+World-writable directories are difficult to manage.  Another option would be for the filesystem owner
+to pre-create all the potential user temporary directories.
+
+## Startup and Testing
+
+Assuming you named the config file `xrootd-http.cfg`, as a non-rootly user run:
+
+```
+xrootd -d -c xrootd-http.cfg
 ```
 
 In a separate terminal, run
 
 ```
-curl -v http://localhost:1094/<path name>/<object name>
+curl -v http://localhost:1094/<host name>/<URL path to object>
 ```

rpm/xrootd-s3-http.spec

@@ -45,6 +45,7 @@ cmake --build redhat-linux-build --verbose
 %{_libdir}/libXrdOssGlobus-5.so
 %{_libdir}/libXrdOssS3-5.so
 %{_libdir}/libXrdOssFilter-5.so
+%{_libdir}/libXrdOssPosc-5.so
 %doc README.md
 %license LICENSE