Modify remote requests to include step count in a single request ...

source/adios2/CMakeLists.txt

@@ -265,7 +265,7 @@ if(ADIOS2_HAVE_Campaign)
   target_link_libraries(adios2_core PRIVATE SQLite::SQLite3 ZLIB::ZLIB)
   if(ADIOS2_HAVE_Sodium)
     target_link_libraries(adios2_core PRIVATE sodium)
-  endif(ADIOS2_HAVE_SODIUM)
+  endif(ADIOS2_HAVE_Sodium)
 endif(ADIOS2_HAVE_Campaign)
 
 if(ADIOS2_HAVE_DAOS)

source/adios2/engine/bp5/BP5Reader.cpp

@@ -498,7 +498,7 @@ void BP5Reader::PerformRemoteGetsWithKVCache()
         const DataType varType = m_IO.InquireVariableType(Req.VarName);
         VariableBase *VB = m_BP5Deserializer->GetVariableBaseFromBP5VarRec(Req.VarRec);
 
-        std::string keyPrefix = m_Fingerprint + "|" + Req.VarName + std::to_string(Req.RelStep);
+        std::string keyPrefix = m_Fingerprint + "|" + Req.VarName + "|";
         if (Req.BlockID != std::numeric_limits<std::size_t>::max())
         {
             MinVarInfo *minBlocksInfo = nullptr;
@@ -527,11 +527,15 @@ void BP5Reader::PerformRemoteGetsWithKVCache()
             }
         }
 
-        RequestInfo ReqInfo(Req.Count.size());
+        // Start/Count in cache includes steps as first dimension
+        adios2::Dims cacheStart = helper::DimsWithStep(Req.RelStep, Req.Start);
+        adios2::Dims cacheCount = helper::DimsWithStep(Req.StepCount, Req.Count);
+
+        RequestInfo ReqInfo(cacheCount.size());
         ReqInfo.ReqSeq = req_seq;
         ReqInfo.TypeSize = helper::GetDataTypeSize(varType);
 
-        kvcache::QueryBox targetBox(Req.Start, Req.Count);
+        kvcache::QueryBox targetBox(cacheStart, cacheCount);
         std::string targetKey = keyPrefix + targetBox.toString();
 
         // Exact Match: check if targetKey exists
@@ -581,10 +585,12 @@ void BP5Reader::PerformRemoteGetsWithKVCache()
                 ReqInfo.Data = malloc(ReqInfo.ReqSize * ReqInfo.TypeSize);
                 std::vector<size_t> start;
                 std::vector<size_t> count;
-                box.StartToVector(start);
-                box.CountToVector(count);
-                auto handle = m_Remote->Get(Req.VarName, Req.RelStep, Req.BlockID, count, start,
-                                            VB->m_AccuracyRequested, ReqInfo.Data);
+                box.StartToVector(start, 1); // start without step
+                box.CountToVector(count, 1); // count without step
+                size_t stepStart = box.Start[0];
+                size_t stepCount = box.Count[0];
+                auto handle = m_Remote->Get(Req.VarName, stepStart, stepCount, Req.BlockID, count,
+                                            start, VB->m_AccuracyRequested, ReqInfo.Data);
                 handles.push_back(handle);
                 remoteRequestsInfo.push_back(ReqInfo);
             }
@@ -622,9 +628,12 @@ void BP5Reader::PerformRemoteGetsWithKVCache()
             void *data = malloc(ReqInfo.ReqBox.size() * ReqInfo.TypeSize);
             m_KVCache.ExecuteBatch(ReqInfo.CacheKey.c_str(), 1,
                                    ReqInfo.ReqBox.size() * ReqInfo.TypeSize, data);
+            // cache result includes steps, need to adjust output Start/Count for N+1 dim copy
+            adios2::Dims outStart = helper::DimsWithStep(Req.RelStep, Req.Start);
+            adios2::Dims outCount = helper::DimsWithStep(Req.StepCount, Req.Count);
             helper::NdCopy(reinterpret_cast<char *>(data), ReqInfo.ReqBox.Start,
                            ReqInfo.ReqBox.Count, true, false, reinterpret_cast<char *>(Req.Data),
-                           Req.Start, Req.Count, true, false, static_cast<int>(ReqInfo.TypeSize));
+                           outStart, outCount, true, false, static_cast<int>(ReqInfo.TypeSize));
             free(data);
         }
     }
@@ -635,9 +644,12 @@ void BP5Reader::PerformRemoteGetsWithKVCache()
         m_Remote->WaitForGet(handle);
         auto &ReqInfo = remoteRequestsInfo[handle_seq];
         auto &Req = GetRequests[ReqInfo.ReqSeq];
+        // cache result includes steps, need to adjust output Start/Count for N+1 dim copy
+        adios2::Dims outStart = helper::DimsWithStep(Req.RelStep, Req.Start);
+        adios2::Dims outCount = helper::DimsWithStep(Req.StepCount, Req.Count);
         helper::NdCopy(reinterpret_cast<char *>(ReqInfo.Data), ReqInfo.ReqBox.Start,
                        ReqInfo.ReqBox.Count, true, false, reinterpret_cast<char *>(Req.Data),
-                       Req.Start, Req.Count, true, false, static_cast<int>(ReqInfo.TypeSize));
+                       outStart, outCount, true, false, static_cast<int>(ReqInfo.TypeSize));
 
         m_KVCache.AppendCommandInBatch(ReqInfo.CacheKey.c_str(), 0,
                                        ReqInfo.ReqSize * ReqInfo.TypeSize, ReqInfo.Data);
@@ -660,8 +672,8 @@ void BP5Reader::PerformRemoteGets()
     for (auto &Req : GetRequests)
     {
         VariableBase *VB = m_BP5Deserializer->GetVariableBaseFromBP5VarRec(Req.VarRec);
-        auto handle = m_Remote->Get(Req.VarName, Req.RelStep, Req.BlockID, Req.Count, Req.Start,
-                                    VB->m_AccuracyRequested, Req.Data);
+        auto handle = m_Remote->Get(Req.VarName, Req.RelStep, Req.StepCount, Req.BlockID, Req.Count,
+                                    Req.Start, VB->m_AccuracyRequested, Req.Data);
         handles.push_back(handle);
     }
 

source/adios2/engine/bp5/BP5Reader.tcc

@@ -24,14 +24,14 @@ namespace engine
 
 inline void BP5Reader::GetSyncCommon(VariableBase &variable, void *data)
 {
-    bool need_sync = m_BP5Deserializer->QueueGet(variable, data);
+    bool need_sync = m_BP5Deserializer->QueueGet(variable, data, m_dataIsRemote);
     if (need_sync)
         PerformGets();
 }
 
 void BP5Reader::GetDeferredCommon(VariableBase &variable, void *data)
 {
-    (void)m_BP5Deserializer->QueueGet(variable, data);
+    (void)m_BP5Deserializer->QueueGet(variable, data, m_dataIsRemote);
 }
 
 } // end namespace engine

source/adios2/helper/adiosType.cpp

@@ -290,5 +290,20 @@ std::string OpenModeToString(const Mode openMode, const bool oneLetter) noexcept
     return openModeString;
 }
 
+adios2::Dims DimsWithStep(size_t firstElement, adios2::Dims &dimsWithoutSteps) noexcept
+{
+    // Start/Count in cache includes steps as first dimension
+    adios2::Dims d = {firstElement};
+    d.insert(d.end(), dimsWithoutSteps.begin(), dimsWithoutSteps.end());
+    return d;
+}
+
+std::tuple<size_t, adios2::Dims> DimsWithoutStep(adios2::Dims &dimsWithSteps) noexcept
+{
+    size_t step = dimsWithSteps[0];
+    adios2::Dims d = {dimsWithSteps.begin() + 1, dimsWithSteps.end()};
+    return std::tuple<size_t, adios2::Dims>(step, d);
+}
+
 } // end namespace helper
 } // end namespace adios2

source/adios2/helper/adiosType.h

@@ -187,6 +187,19 @@ class DimsArray : public CoreDims
     }
 };
 
+/**
+ * Make an adios2::Dims vector from steps + another Dims vector
+ * @return adios2::Dims vector, first element is 'firstElement', rest is input vector
+ */
+adios2::Dims DimsWithStep(size_t firstElement, adios2::Dims &dimsWithoutSteps) noexcept;
+
+/**
+ * Separate adios2::Dims vector with step
+ * @return tuple of first element, and an adios2::Dims vector without first element of input
+ * vector.
+ */
+std::tuple<size_t, adios2::Dims> DimsWithoutStep(adios2::Dims &dimsWithSteps) noexcept;
+
 /**
  * Gets type from template parameter T
  * @return DataType enumeration value

source/adios2/toolkit/format/bp5/BP5Deserializer.cpp

@@ -1271,7 +1271,7 @@ void BP5Deserializer::InstallAttributesV2(FFSTypeHandle FFSformat, void *BaseDat
     }
 }
 
-bool BP5Deserializer::QueueGet(core::VariableBase &variable, void *DestData)
+bool BP5Deserializer::QueueGet(core::VariableBase &variable, void *DestData, bool dataIsRemote)
 {
     if (!m_RandomAccessMode)
     {
@@ -1298,20 +1298,29 @@ bool BP5Deserializer::QueueGet(core::VariableBase &variable, void *DestData)
                     "(random access), or "
                     "number of BeginStep calls (streaming)");
         }
-        for (size_t RelStep = variable.m_StepsStart;
-             RelStep < variable.m_StepsStart + variable.m_StepsCount; RelStep++)
+        if (dataIsRemote && VarRec->OrigShapeID != ShapeID::LocalValue &&
+            VarRec->OrigShapeID != ShapeID::GlobalValue)
         {
-            const size_t AbsStep = VarRec->AbsStepFromRel[RelStep];
-            const size_t writerCohortSize = WriterCohortSize(AbsStep);
-            for (size_t WriterRank = 0; WriterRank < writerCohortSize; WriterRank++)
+            ret = QueueGetSingleRemote(variable, DestData, variable.m_StepsStart,
+                                       variable.m_StepsCount);
+        }
+        else
+        {
+            for (size_t RelStep = variable.m_StepsStart;
+                 RelStep < variable.m_StepsStart + variable.m_StepsCount; RelStep++)
             {
-                if (GetMetadataBase(VarRec, AbsStep, WriterRank))
+                const size_t AbsStep = VarRec->AbsStepFromRel[RelStep];
+                const size_t writerCohortSize = WriterCohortSize(AbsStep);
+                for (size_t WriterRank = 0; WriterRank < writerCohortSize; WriterRank++)
                 {
-                    // This writer wrote on this timestep
-                    ret = QueueGetSingle(variable, DestData, AbsStep, RelStep);
-                    size_t increment = variable.TotalSize() * variable.m_ElementSize;
-                    DestData = (void *)((char *)DestData + increment);
-                    break;
+                    if (GetMetadataBase(VarRec, AbsStep, WriterRank))
+                    {
+                        // This writer wrote on this timestep
+                        ret = QueueGetSingle(variable, DestData, AbsStep, RelStep);
+                        size_t increment = variable.TotalSize() * variable.m_ElementSize;
+                        DestData = (void *)((char *)DestData + increment);
+                        break;
+                    }
                 }
             }
         }
@@ -1429,6 +1438,7 @@ bool BP5Deserializer::QueueGetSingle(core::VariableBase &variable, void *DestDat
         Req.Start = variable.m_Start;
         Req.Step = AbsStep;
         Req.RelStep = RelStep;
+        Req.StepCount = 1;
         Req.MemSpace = MemSpace;
         Req.Data = DestData;
         PendingGetRequests.push_back(Req);
@@ -1450,6 +1460,64 @@ bool BP5Deserializer::QueueGetSingle(core::VariableBase &variable, void *DestDat
         Req.MemSpace = MemSpace;
         Req.Step = AbsStep;
         Req.RelStep = RelStep;
+        Req.StepCount = 1;
+        PendingGetRequests.push_back(Req);
+    }
+    else
+    {
+        std::cout << "Missed get type " << variable.m_SelectionType << " shape "
+                  << variable.m_ShapeID << std::endl;
+    }
+    return true;
+}
+
+bool BP5Deserializer::QueueGetSingleRemote(core::VariableBase &variable, void *DestData,
+                                           size_t RelStep, size_t StepCount)
+{
+    BP5VarRec *VarRec = VarByKey[&variable];
+    if (variable.m_Type == adios2::DataType::Struct)
+    {
+        // This probably does not work at all. Need to be implemented
+        StructQueueReadChecks(dynamic_cast<core::VariableStruct *>(&variable), VarRec);
+    }
+
+    MemorySpace MemSpace = variable.GetMemorySpace(DestData);
+    if ((variable.m_SelectionType == adios2::SelectionType::BoundingBox) &&
+        ((variable.m_ShapeID == ShapeID::GlobalArray) ||
+         (variable.m_ShapeID == ShapeID::JoinedArray)))
+    {
+        BP5ArrayRequest Req;
+        Req.VarRec = VarRec;
+        Req.VarName = (char *)variable.m_Name.c_str();
+        Req.RequestType = Global;
+        Req.BlockID = (size_t)-1;
+        Req.Count = variable.m_Count;
+        Req.Start = variable.m_Start;
+        Req.Step = RelStep;
+        Req.RelStep = RelStep;
+        Req.StepCount = StepCount;
+        Req.MemSpace = MemSpace;
+        Req.Data = DestData;
+        PendingGetRequests.push_back(Req);
+    }
+    else if ((variable.m_SelectionType == adios2::SelectionType::WriteBlock) ||
+             (variable.m_ShapeID == ShapeID::LocalArray))
+    {
+        BP5ArrayRequest Req;
+        Req.VarRec = VarByKey[&variable];
+        Req.VarName = (char *)variable.m_Name.c_str();
+        Req.RequestType = Local;
+        Req.BlockID = variable.m_BlockID;
+        if (variable.m_SelectionType == adios2::SelectionType::BoundingBox)
+        {
+            Req.Start = variable.m_Start;
+            Req.Count = variable.m_Count;
+        }
+        Req.Data = DestData;
+        Req.MemSpace = MemSpace;
+        Req.Step = RelStep;
+        Req.RelStep = RelStep;
+        Req.StepCount = StepCount;
         PendingGetRequests.push_back(Req);
     }
     else

source/adios2/toolkit/format/bp5/BP5Deserializer.h

@@ -61,9 +61,11 @@ class BP5Deserializer : virtual public BP5Base
 
     void SetupForStep(size_t Step, size_t WriterCount);
     // return from QueueGet is true if a sync is needed to fill the data
-    bool QueueGet(core::VariableBase &variable, void *DestData);
+    bool QueueGet(core::VariableBase &variable, void *DestData, bool dataIsRemote = false);
     bool QueueGetSingle(core::VariableBase &variable, void *DestData, size_t AbsStep,
                         size_t RelStep);
+    bool QueueGetSingleRemote(core::VariableBase &variable, void *DestData, size_t RelStep,
+                              size_t StepCount);
 
     /* generate read requests. return vector of requests AND the size of
      * the largest allocation block necessary for reading.
@@ -105,8 +107,9 @@ class BP5Deserializer : virtual public BP5Base
         void *VarRec = NULL;
         char *VarName;
         enum RequestTypeEnum RequestType;
-        size_t Step;    // local operations use absolute steps
-        size_t RelStep; // preserve Relative Step for remote
+        size_t Step;      // local operations use absolute steps
+        size_t RelStep;   // preserve Relative Step for remote
+        size_t StepCount; // =1 for local, can be >1 for remote
         size_t BlockID;
         Dims Start;
         Dims Count;

source/adios2/toolkit/kvcache/QueryBox.h

@@ -54,7 +54,7 @@ class QueryBox
     // ToString
     std::string toString() const
     {
-        std::string str = "|Start_";
+        std::string str = "Start_";
         for (size_t i = 0; i < Start.size(); ++i)
         {
             str += std::to_string(Start[i]);
@@ -101,17 +101,17 @@ class QueryBox
     }
 
     // convert helper::DimsArray to std::vector<size_t>
-    void StartToVector(std::vector<size_t> &vec) const
+    void StartToVector(std::vector<size_t> &vec, size_t startPos = 0) const
     {
-        for (size_t i = 0; i < Start.size(); ++i)
+        for (size_t i = startPos; i < Start.size(); ++i)
         {
             vec.push_back(Start[i]);
         }
     }
 
-    void CountToVector(std::vector<size_t> &vec) const
+    void CountToVector(std::vector<size_t> &vec, size_t startPos = 0) const
     {
-        for (size_t i = 0; i < Count.size(); ++i)
+        for (size_t i = startPos; i < Count.size(); ++i)
         {
             vec.push_back(Count[i]);
         }
@@ -260,7 +260,9 @@ class QueryBox
         for (auto &key : samePrefixKeys)
         {
             // Initialize the box from the key
-            size_t DimCount = std::count(key.begin(), key.end(), '_') / 2;
+            auto sp = key.find("|Start");
+            auto cp = key.find("|Count");
+            size_t DimCount = std::count(key.begin() + sp, key.begin() + cp, '_');
             QueryBox box(DimCount);
             lf_ExtractDimensions(key, "|Start_", box.Start);
             lf_ExtractDimensions(key, "|Count_", box.Count);

source/adios2/toolkit/remote/EVPathRemote.cpp

@@ -184,15 +184,17 @@ void EVPathRemote::OpenSimpleFile(const std::string hostname, const int32_t port
     m_Active = true;
 }
 
-EVPathRemote::GetHandle EVPathRemote::Get(char *VarName, size_t Step, size_t BlockID, Dims &Count,
-                                          Dims &Start, Accuracy &accuracy, void *dest)
+EVPathRemote::GetHandle EVPathRemote::Get(char *VarName, size_t Step, size_t StepCount,
+                                          size_t BlockID, Dims &Count, Dims &Start,
+                                          Accuracy &accuracy, void *dest)
 {
     EVPathRemoteCommon::_GetRequestMsg GetMsg;
     memset(&GetMsg, 0, sizeof(GetMsg));
     GetMsg.GetResponseCondition = CMCondition_get(ev_state.cm, m_conn);
     GetMsg.FileHandle = m_ID;
     GetMsg.VarName = VarName;
     GetMsg.Step = Step;
+    GetMsg.StepCount = StepCount;
     GetMsg.BlockID = BlockID;
     GetMsg.DimCount = (int)Count.size();
     GetMsg.Count = Count.data();

source/adios2/toolkit/remote/EVPathRemote.h

@@ -43,8 +43,8 @@ class EVPathRemote : public Remote
 
     void OpenSimpleFile(const std::string hostname, const int32_t port, const std::string filename);
 
-    GetHandle Get(char *VarName, size_t Step, size_t BlockID, Dims &Count, Dims &Start,
-                  Accuracy &accuracy, void *dest);
+    GetHandle Get(char *VarName, size_t Step, size_t StepCount, size_t BlockID, Dims &Count,
+                  Dims &Start, Accuracy &accuracy, void *dest);
 
     bool WaitForGet(GetHandle handle);
 

source/adios2/toolkit/remote/Remote.cpp

@@ -35,8 +35,8 @@ void Remote::OpenSimpleFile(const std::string hostname, const int32_t port,
     ThrowUp("RemoteSimpleOpen");
 };
 
-Remote::GetHandle Remote::Get(char *VarName, size_t Step, size_t BlockID, Dims &Count, Dims &Start,
-                              Accuracy &accuracy, void *dest)
+Remote::GetHandle Remote::Get(char *VarName, size_t Step, size_t StepCount, size_t BlockID,
+                              Dims &Count, Dims &Start, Accuracy &accuracy, void *dest)
 {
     ThrowUp("RemoteGet");
     return (Remote::GetHandle)(intptr_t)0;

source/adios2/toolkit/remote/Remote.h

@@ -43,8 +43,8 @@ class Remote
 
     typedef void *GetHandle;
 
-    virtual GetHandle Get(char *VarName, size_t Step, size_t BlockID, Dims &Count, Dims &Start,
-                          Accuracy &accuracy, void *dest);
+    virtual GetHandle Get(char *VarName, size_t Step, size_t StepCount, size_t BlockID, Dims &Count,
+                          Dims &Start, Accuracy &accuracy, void *dest);
 
     virtual bool WaitForGet(GetHandle handle);