Add CTest for XRootd Remote support

.github/workflows/everything.yml

@@ -114,7 +114,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/ornladios/adios2:ci-spack-${{ matrix.os }}-${{ matrix.compiler }}
+      image: ghcr.io/ornladios/adios2:ci-tmp-spack-${{ matrix.os }}-${{ matrix.compiler }}
       options: --shm-size=1g
       env:
         GH_YML_JOBNAME: ${{ matrix.os }}-${{ matrix.compiler }}${{ matrix.shared == 'static' && '-static' || ''}}-${{ matrix.parallel }}
@@ -535,7 +535,7 @@ jobs:
     name: CodeQL
     runs-on: ubuntu-latest
     container:
-      image: 'ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-gcc8'
+      image: 'ghcr.io/ornladios/adios2:ci-tmp-spack-ubuntu20.04-gcc8'
       env:
         GH_YML_JOBNAME: ubuntu20.04-gcc8-serial-codeql
         GH_YML_BASE_OS: Linux

cmake/DetectOptions.cmake

@@ -583,6 +583,15 @@ elseif(ADIOS2_USE_XRootD)
 endif()
 if(XRootD_FOUND)
   set(ADIOS2_HAVE_XRootD TRUE)
+  find_program(XROOTD_SERVER_BINARY xrootd
+               HINTS
+               ${XROOTD_DIR}
+               $ENV{XROOTD_DIR}
+               /usr
+               /opt/xrootd
+               PATH_SUFFIXES bin
+  )
+
 endif()
 
 # Campaign Management

scripts/ci/images/0002-adios2-add-xrootd-variant.patch

@@ -0,0 +1,40 @@
+From dfe014d1eea8b0d0533b7519b4a4f58785e43298 Mon Sep 17 00:00:00 2001
+From: Vicente Adolfo Bolea Sanchez <[email protected]>
+Date: Tue, 18 Jun 2024 18:23:06 -0400
+Subject: [PATCH] adios2: add xrootd variant
+
+---
+ var/spack/repos/builtin/packages/adios2/package.py | 3 +++
+ 1 file changed, 3 insertions(+)
+
+diff --git a/var/spack/repos/builtin/packages/adios2/package.py b/var/spack/repos/builtin/packages/adios2/package.py
+index f6a5aba446..97f8322ff3 100644
+--- a/var/spack/repos/builtin/packages/adios2/package.py
++++ b/var/spack/repos/builtin/packages/adios2/package.py
+@@ -95,6 +95,7 @@ class Adios2(CMakePackage, CudaPackage):
+     variant("cuda", default=False, when="@2.8:", description="Enable CUDA support")
+     variant("python", default=False, description="Enable the Python bindings")
+     variant("fortran", default=True, description="Enable the Fortran bindings")
++    variant("xrootd", default=True, description="Enable the XRootD")
+
+     # Requires mature C++11 implementations
+     conflicts("%gcc@:4.7")
+@@ -148,6 +149,7 @@ class Adios2(CMakePackage, CudaPackage):
+     depends_on("[email protected]:", when="+mpi +python", type=("build", "run"))
+     depends_on("aws-sdk-cpp", when="+aws")
+     depends_on("libcatalyst@2", when="+libcatalyst")
++    depends_on("xrootd~davix", when="+xrootd")
+
+     # Fix findmpi when called by dependees
+     # See https://github.com/ornladios/ADIOS2/pull/1632
+@@ -197,6 +199,7 @@ def cmake_args(self):
+             from_variant("CMAKE_POSITION_INDEPENDENT_CODE", "pic"),
+             from_variant("BUILD_SHARED_LIBS", "shared"),
+             from_variant("ADIOS2_USE_AWSSDK", "aws"),
++            from_variant("ADIOS2_USE_XRootD", "xrootd"),
+             from_variant("ADIOS2_USE_Blosc", "blosc"),
+             from_variant("ADIOS2_USE_BZip2", "bzip2"),
+             from_variant("ADIOS2_USE_DataMan", "dataman"),
+-- 
+2.35.3
+

scripts/ci/images/Dockerfile.ci-spack-ubuntu20.04-base

@@ -6,6 +6,7 @@ ARG E4S_VERSION=23.08
 ARG ENABLED_ENVS="serial,ompi,mpich"
 ARG EXTERNAL_PACKAGES
 ARG EXTRA_VARIANTS
+ARG PATCH_VARIANT_XROOTD
 
 
 # Clone and patch spack
@@ -17,12 +18,14 @@ RUN if ! [ -d /spack ]; then \
     fi
 
 COPY 0001-mpich-support-ch3-sock.patch /spack
+COPY 0002-adios2-add-xrootd-variant.patch /spack
+
 COPY packages.yaml /etc/spack/packages.yaml
 
 WORKDIR /spack
 # TODO: Patch only needed until we merge https://github.com/spack/spack/pull/40964
 RUN if [ ${E4S_VERSION%.*} -le 23 ]; then git apply 0001-mpich-support-ch3-sock.patch; fi
-
+RUN if [ -n "${PATCH_VARIANT_XROOTD}" ]; then git apply 0002-adios2-add-xrootd-variant.patch; fi
 # Install base specs
 RUN . /spack/share/spack/setup-env.sh && \
     if test -n "${EXTERNAL_PACKAGES}"; then spack external find ${EXTERNAL_PACKAGES}; fi && \

scripts/ci/images/Dockerfile.ci-spack-ubuntu20.04-clang

@@ -1,4 +1,4 @@
-FROM ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-base
+FROM ghcr.io/ornladios/adios2:ci-tmp-spack-ubuntu20.04-base
 ARG CLANG_VERSION=10
 
 RUN apt-get update && apt-get install -y \

scripts/ci/images/Dockerfile.ci-spack-ubuntu20.04-gcc

@@ -1,4 +1,4 @@
-FROM ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-base
+FROM ghcr.io/ornladios/adios2:ci-tmp-spack-ubuntu20.04-base
 ARG GCC_VERSION=8
 
 RUN apt-get update && apt-get install -y \

scripts/ci/images/Dockerfile.ci-spack-ubuntu20.04-intel

@@ -1,4 +1,4 @@
-FROM ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-base
+FROM ghcr.io/ornladios/adios2:ci-tmp-spack-ubuntu20.04-base
 
 RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg && \
     echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \

scripts/ci/images/build-ubuntu.sh

@@ -3,16 +3,21 @@
 set -ex
 
 # Build the base image
-docker build --progress=plain --build-arg EXTRA_VARIANTS="+blosc+ssc ^mgard@2023-01-10" --rm -f ./Dockerfile.ci-spack-ubuntu20.04-base -t ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-base .
+docker build --progress=plain \
+  --build-arg EXTRA_VARIANTS="+blosc+ssc ^mgard@2023-01-10" \
+  --build-arg PATCH_VARIANT_XROOTD=ON \
+  --rm -f ./Dockerfile.ci-spack-ubuntu20.04-base \
+  -t ghcr.io/ornladios/adios2:ci-tmp-spack-ubuntu20.04-base \
+  .
 
 # Build the gcc8, gcc9, and gcc10 images
-docker build --rm --build-arg GCC_VERSION=8 -f ./Dockerfile.ci-spack-ubuntu20.04-gcc -t ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-gcc8 .
-docker build --rm --build-arg GCC_VERSION=10 -f ./Dockerfile.ci-spack-ubuntu20.04-gcc -t ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-gcc10 .
-docker build --rm --build-arg GCC_VERSION=9 -f ./Dockerfile.ci-spack-ubuntu20.04-gcc -t ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-gcc9 .
+docker build --rm --build-arg GCC_VERSION=8 -f ./Dockerfile.ci-spack-ubuntu20.04-gcc -t ghcr.io/ornladios/adios2:ci-tmp-spack-ubuntu20.04-gcc8 .
+docker build --rm --build-arg GCC_VERSION=10 -f ./Dockerfile.ci-spack-ubuntu20.04-gcc -t ghcr.io/ornladios/adios2:ci-tmp-spack-ubuntu20.04-gcc10 .
+docker build --rm --build-arg GCC_VERSION=9 -f ./Dockerfile.ci-spack-ubuntu20.04-gcc -t ghcr.io/ornladios/adios2:ci-tmp-spack-ubuntu20.04-gcc9 .
 
 # Build the clang6 and clang10 images
-docker build --rm --build-arg CLANG_VERSION=6.0 -f ./Dockerfile.ci-spack-ubuntu20.04-clang -t ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-clang6 .
-docker build --rm --build-arg CLANG_VERSION=10 -f ./Dockerfile.ci-spack-ubuntu20.04-clang -t ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-clang10 .
+docker build --rm --build-arg CLANG_VERSION=6.0 -f ./Dockerfile.ci-spack-ubuntu20.04-clang -t ghcr.io/ornladios/adios2:ci-tmp-spack-ubuntu20.04-clang6 .
+docker build --rm --build-arg CLANG_VERSION=10 -f ./Dockerfile.ci-spack-ubuntu20.04-clang -t ghcr.io/ornladios/adios2:ci-tmp-spack-ubuntu20.04-clang10 .
 
 # Build the ubuntu 22.04 img
 docker build \
@@ -22,14 +27,14 @@ docker build \
   --build-arg E4S_VERSION="24.05" \
   --build-arg EXTRA_VARIANTS="+blosc2" \
   -f ./Dockerfile.ci-spack-ubuntu20.04-base \
-  -t ghcr.io/ornladios/adios2:ci-spack-ubuntu22.04-gcc11 \
+  -t ghcr.io/ornladios/adios2:ci-tmp-spack-ubuntu22.04-gcc11 \
   .
 
 # Push images to github container registry
-docker push ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-base
-docker push ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-gcc8
-docker push ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-gcc9
-docker push ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-gcc10
-docker push ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-clang6
-docker push ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-clang10
-docker push ghcr.io/ornladios/adios2:ci-spack-ubuntu22.04-gcc11
+docker push ghcr.io/ornladios/adios2:ci-tmp-spack-ubuntu20.04-base
+docker push ghcr.io/ornladios/adios2:ci-tmp-spack-ubuntu20.04-gcc8
+docker push ghcr.io/ornladios/adios2:ci-tmp-spack-ubuntu20.04-gcc9
+docker push ghcr.io/ornladios/adios2:ci-tmp-spack-ubuntu20.04-gcc10
+docker push ghcr.io/ornladios/adios2:ci-tmp-spack-ubuntu20.04-clang6
+docker push ghcr.io/ornladios/adios2:ci-tmp-spack-ubuntu20.04-clang10
+docker push ghcr.io/ornladios/adios2:ci-tmp-spack-ubuntu22.04-gcc11

source/adios2/toolkit/remote/XrootdRemote.cpp

@@ -125,12 +125,12 @@ myRequest *myRequest::currentRequest = 0; // Pointer to current request
 
 void myRequest::Alert(XrdSsiRespInfoMsg &aMsg)
 {
-    char *theMsg;
-    int theMsz;
+    // char *theMsg;
+    // int theMsz;
 
     // Get the message
     //
-    theMsg = aMsg.GetMsg(theMsz);
+    // theMsg = aMsg.GetMsg(theMsz);
 
     // Print what we received
     //
@@ -140,7 +140,7 @@ void myRequest::Alert(XrdSsiRespInfoMsg &aMsg)
 
     // Recycle the message
     //
-    aMsg.RecycleMsg();
+    // aMsg.RecycleMsg();
 }
 
 /******************************************************************************/
@@ -337,7 +337,7 @@ Remote::GetHandle XrootdRemote::Get(char *VarName, size_t Step, size_t BlockID,
                                     Dims &Start, void *dest)
 {
 #ifdef ADIOS2_HAVE_XROOTD
-    char rName[512] = "/home/eisen/xroot/data";
+    char rName[512] = "/etc";
     XrdSsiResource rSpec((std::string)rName);
     myRequest *reqP;
     std::string reqData = "get Filename=" + std::string(m_Filename) + std::string("&RMOrder=") +

source/utils/xrootd-plugin/AdiosFilePool.cpp

@@ -10,7 +10,6 @@ AnonADIOSFile *ADIOSFilePool::GetFree(std::string Filename, bool RowMajorArrays)
     auto res = map.find(index);
 
     SubPool *subpool;
-    AnonADIOSFile *ret;
     if (res != map.end())
     {
         subpool = res->second.get();

source/utils/xrootd-plugin/XrdSsiSvService.cpp

@@ -64,13 +64,6 @@ using namespace std;
 
 namespace
 {
-void *SvWecho(void *svP)
-{
-    XrdSsiSvService *sessP = (XrdSsiSvService *)svP;
-    sessP->doWecho();
-    return 0;
-}
-
 void *SvAdiosGet(void *svP)
 {
     XrdSsiSvService *sessP = (XrdSsiSvService *)svP;
@@ -501,7 +494,6 @@ void XrdSsiSvService::ProcessRequest4Me(XrdSsiRequest *rqstP)
         pthread_t tid;
         auto engine = poolEntry->m_engine;
         auto io = poolEntry->m_io;
-        auto var = io.InquireVariable(VarName);
         adios2::Box<adios2::Dims> varSel(Start, Count);
         adios2::DataType TypeOfVar = io.InquireVariableType(VarName);
         try
@@ -569,11 +561,9 @@ int XrdSsiSvService::Copy2Buff(char *dest, int dsz, const char *src, int ssz)
 
 void XrdSsiSvService::Respond(const char *rData, const char *mData)
 {
-    XrdSsiResponder::Status rc;
-    int rLen;
-
     throw std::logic_error("Respond shouldn't be called");
 }
+
 void XrdSsiSvService::AdiosRespond(const char *rData, const char *mData)
 {
     XrdSsiResponder::Status rc;

source/utils/xrootd-plugin/XrdSsiSvService.hh

@@ -44,20 +44,11 @@
 class XrdSsiSvService : public XrdSsiService, public XrdSsiResponder
 {
 public:
-    //-----------------------------------------------------------------------------
-    //! Attach to a previously detached server-side request.
-    //-----------------------------------------------------------------------------
-
-    virtual void Attach(XrdSsiRequest &reqRef, std::string handle) {}
-
     //-----------------------------------------------------------------------------
     //! Do delayed echo (internal)
     //-----------------------------------------------------------------------------
 
-    void doWecho()
-    {
-        Respond(m_responseBuffer, m_respMeta);
-    }
+    void doWecho() { Respond(m_responseBuffer, m_respMeta); }
 
     void doAdiosGet() { AdiosRespond(m_responseBuffer, m_respMeta); }
 
@@ -103,7 +94,7 @@ protected:
     //! Destructor is protected. You cannot use delete on a service, use the
     //! Stop() method to effectively delete the service object.
     //-----------------------------------------------------------------------------
-  ~XrdSsiSvService();
+    ~XrdSsiSvService();
 
 private:
     int Copy2Buff(char *dest, int dsz, const char *src, int ssz);
@@ -118,7 +109,7 @@ private:
     static int alertNum;
     char *sName;
     char *m_responseBuffer = NULL;
-    int m_responseBufferSize = 0;
+    size_t m_responseBufferSize = 0;
     char m_respMeta[512];
     char m_respData[1024];
     int streamRdSz;

testing/adios2/engine/bp/CMakeLists.txt

@@ -150,6 +150,26 @@ if ((NOT WIN32) AND ADIOS2_HAVE_SST)
    add_file_remote_tests_helper(WriteMemorySelectionRead)
 endif()
 
+if (ADIOS2_HAVE_XRootD)
+   macro(add_get_xrremote_tests_helper testname)
+      add_test(NAME "Remote.BP${testname}.GetXRRemote" COMMAND Test.Engine.BP.${testname}.Serial bp5)
+      set_tests_properties(Remote.BP${testname}.GetXRRemote PROPERTIES FIXTURES_REQUIRED XRServer ENVIRONMENT "DoXRootD=1")
+   endmacro()
+
+   add_test(NAME generateXRootDConfig COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/generateXRootDConfig.sh $<TARGET_FILE:adios2_xrootd>  )
+   add_test(NAME remoteXRServerSetup COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/start_xrootd.sh ${XROOTD_SERVER_BINARY} ${CMAKE_CURRENT_BINARY_DIR})
+
+   set_tests_properties(remoteXRServerSetup PROPERTIES FIXTURES_SETUP XRServer)
+   set_tests_properties(remoteXRServerSetup PROPERTIES DEPENDS "generateXRootDConfig")
+
+   add_test(NAME remoteXRServerCleanup COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/kill_xrootd.sh )
+   set_tests_properties(remoteXRServerCleanup         PROPERTIES FIXTURES_CLEANUP    XRServer)
+
+   ##### add remote tests below this line
+   add_get_xrremote_tests_helper(WriteReadADIOS2stdio)
+   add_get_xrremote_tests_helper(WriteMemorySelectionRead)
+endif()
+
 if(ADIOS2_HAVE_MPI)
   list(APPEND BP5LargeMeta "Engine.BP.BPLargeMetadata.BPWrite1D_LargeMetadata.BP5.MPI" "Engine.BP.BPLargeMetadata.ManyLongStrings.BP5.MPI")
 endif()

testing/adios2/engine/bp/generateXRootDConfig.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+echo "Generating config for XRootD with plugin library at $1"
+mkdir -p xroot/var/spool
+mkdir -p xroot/run/xrootd
+mkdir -p xroot/etc/xrootd
+{
+    echo "xrootd.fslib libXrdSsi.so";
+    echo ""
+    echo "all.export /etc nolock r/w";
+    echo ""
+    echo "oss.statlib -2 libXrdSsi.so";
+    echo ""
+    echo "ssi.svclib $1";
+    echo ""
+} > xroot/etc/xrootd/xrootd-ssi.cfg

testing/adios2/engine/bp/kill_xrootd.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+kill -2 "$(cat /tmp/xrootd.pid)"
+#rm /tmp/xrootd.pid

testing/adios2/engine/bp/start_xrootd.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -x
+
+if [ "$(id -u)" -eq 0 ]; then
+    # we run as root in CI in docker images, this is OK, but we have to tell XRootD that it's OK.
+    XROOTD_USER=("-R" "daemon")
+fi
+
+"$1" -b -l /tmp/xroot.log -c "$2"/xroot/etc/xrootd/xrootd-ssi.cfg "${XROOTD_USER[@]}"