Enable operator info in bpls

source/adios2/common/ADIOSTypes.h

@@ -359,6 +359,34 @@ std::string ToString(const Dims &dims);
 std::string ToString(const Box<Dims> &box);
 std::string ToString(const MemorySpace value);
 
+// Derived exception class for specific errors
+class PluginLoadFailure : public std::runtime_error
+{
+public:
+    PluginLoadFailure(const std::string &msg, const std::string &Library, const std::string &Name)
+    : runtime_error(msg), m_PluginLibrary(Library), m_PluginName(Name)
+    {
+    }
+
+    std::string m_PluginLibrary;
+    std::string m_PluginName;
+
+private:
+};
+
+class MissingOperatorFailure : public std::invalid_argument
+{
+public:
+    MissingOperatorFailure(const std::string &msg, const std::string &Operator)
+    : invalid_argument(msg), m_Operator(Operator)
+    {
+    }
+
+    std::string m_Operator;
+
+private:
+};
+
 /** UserOptions holds all user options from ~/.config/adios2/adios2.yaml */
 struct UserOptions
 {

source/adios2/core/VariableBase.cpp

@@ -343,7 +343,15 @@ size_t VariableBase::AddOperation(const std::string &type, const Params &paramet
     auto op = MakeOperator(type, parameters);
     if (op->IsDataTypeValid(m_Type))
     {
-        m_Operations.push_back(op);
+        if (!m_Operations.empty() && (m_Operations[0]->m_TypeString == "null"))
+        {
+            // if there's a dummy operation in place, replace it
+            m_Operations[0] = op;
+        }
+        else
+        {
+            m_Operations.push_back(op);
+        }
     }
     else
     {

source/adios2/operator/OperatorFactory.cpp

@@ -182,9 +182,11 @@ std::shared_ptr<Operator> MakeOperator(const std::string &type, const Params &pa
 
     if (ret == nullptr)
     {
-        helper::Throw<std::invalid_argument>("Operator", "OperatorFactory", "MakeOperator",
-                                             "ADIOS2 didn't compile with " + typeLowerCase +
-                                                 " library, operator not added");
+        auto m = MakeMessage("Operator", "OperatorFactory", "MakeOperator",
+                             "ADIOS2 didn't compile with " + typeLowerCase +
+                                 " library, operator not added",
+                             -1, helper::LogMode::EXCEPTION);
+        throw MissingOperatorFailure(m, typeLowerCase);
     }
 
     return ret;

source/adios2/operator/plugin/PluginOperator.cpp

@@ -61,7 +61,11 @@ PluginOperator::PluginOperator(const Params &parameters)
     }
 }
 
-PluginOperator::~PluginOperator() { m_Impl->m_HandleDestroy(m_Impl->m_Plugin); }
+PluginOperator::~PluginOperator()
+{
+    if (m_Impl->m_Plugin)
+        m_Impl->m_HandleDestroy(m_Impl->m_Plugin);
+}
 
 void PluginOperator::PluginInit(const std::string &pluginName, const std::string &pluginLibrary)
 {
@@ -72,11 +76,24 @@ void PluginOperator::PluginInit(const std::string &pluginName, const std::string
 
     auto &pluginManager = PluginManager::GetInstance();
     pluginManager.SetParameters(m_Parameters);
-    pluginManager.LoadPlugin(pluginName, pluginLibrary);
-
+    try
+    {
+        pluginManager.LoadPlugin(pluginName, pluginLibrary);
+    }
+    catch (...)
+    {
+        auto m = MakeMessage("Plugins", "PluginOperator", "PluginInit",
+                             "Failed to load library " + m_PluginLibrary + " looking for plugin " +
+                                 m_PluginName,
+                             -1, helper::LogMode::EXCEPTION);
+        throw PluginLoadFailure(m, pluginLibrary, pluginName);
+    }
     m_Impl->m_HandleCreate = pluginManager.GetOperatorCreateFun(pluginName);
     m_Impl->m_HandleDestroy = pluginManager.GetOperatorDestroyFun(pluginName);
     m_Impl->m_Plugin = m_Impl->m_HandleCreate(m_Parameters);
+    // add for external visibility
+    m_PluginName = pluginName;
+    m_PluginLibrary = pluginLibrary;
 }
 
 size_t PluginOperator::GetEstimatedSize(const size_t ElemCount, const size_t ElemSize,

source/adios2/operator/plugin/PluginOperator.h

@@ -48,6 +48,9 @@ class PluginOperator : public core::Operator
 
     bool IsDataTypeValid(const DataType type) const override;
 
+    std::string m_PluginLibrary;
+    std::string m_PluginName;
+
 protected:
     void PluginInit(const std::string &pluginName, const std::string &pluginLibrary);
 

source/adios2/toolkit/format/bp5/BP5Deserializer.cpp

@@ -477,7 +477,7 @@ BP5Deserializer::ControlInfo *BP5Deserializer::BuildControl(FMFormat Format)
                     VarRec->Def = Def;
                 }
                 if (Operator)
-                    VarRec->Operator = strdup("SomeOperator");
+                    VarRec->Operator = strdup("null");
                 C->ElementSize = ElementSize;
             }
             C->VarRec = VarRec;
@@ -940,6 +940,12 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, size
                     static_cast<VariableBase *>(VarRec->Variable)->m_Engine = m_Engine;
                     VarByKey[VarRec->Variable] = VarRec;
                     VarRec->LastTSAdded = Step; // starts at 1
+                    if (VarRec->Operator)
+                    {
+                        auto VB = static_cast<VariableBase *>(VarRec->Variable);
+                        auto tmpOp = MakeOperator("null", {});
+                        VB->m_Operations.push_back(tmpOp);
+                    }
                     if (!meta_base->Shape)
                     {
                         static_cast<VariableBase *>(VarRec->Variable)->m_ShapeID =
@@ -1974,38 +1980,57 @@ void BP5Deserializer::FinalizeGet(const ReadRequest &Read, const bool freeAddr)
     std::vector<char> decompressBuffer;
     if (((struct BP5VarRec *)Req.VarRec)->Operator != NULL)
     {
-        size_t DestSize = ((struct BP5VarRec *)Req.VarRec)->ElementSize;
-        for (size_t dim = 0; dim < ((struct BP5VarRec *)Req.VarRec)->DimCount; dim++)
+        try
         {
-            DestSize *= writer_meta_base->Count[dim + Read.BlockID * writer_meta_base->Dims];
-        }
-        decompressBuffer.resize(DestSize);
+            size_t DestSize = ((struct BP5VarRec *)Req.VarRec)->ElementSize;
+            for (size_t dim = 0; dim < ((struct BP5VarRec *)Req.VarRec)->DimCount; dim++)
+            {
+                DestSize *= writer_meta_base->Count[dim + Read.BlockID * writer_meta_base->Dims];
+            }
+            decompressBuffer.resize(DestSize);
 
-        // Get the operator of the variable if exists or create one
-        std::shared_ptr<Operator> op = nullptr;
-        VariableBase *VB = static_cast<VariableBase *>(((struct BP5VarRec *)Req.VarRec)->Variable);
-        if (!VB->m_Operations.empty())
-        {
-            op = VB->m_Operations[0];
-        }
-        else
-        {
-            Operator::OperatorType compressorType =
-                static_cast<Operator::OperatorType>(IncomingData[0]);
-            op = MakeOperator(OperatorTypeToString(compressorType), {});
-        }
-        op->SetAccuracy(VB->GetAccuracyRequested());
+            // Get the operator of the variable if exists or create one
+            std::shared_ptr<Operator> op = nullptr;
+            VariableBase *VB =
+                static_cast<VariableBase *>(((struct BP5VarRec *)Req.VarRec)->Variable);
+            if (!VB->m_Operations.empty() && (VB->m_Operations[0]->m_TypeString != "null"))
+            {
+                op = VB->m_Operations[0];
+            }
+            else
+            {
+                Operator::OperatorType compressorType =
+                    static_cast<Operator::OperatorType>(IncomingData[0]);
+                op = MakeOperator(OperatorTypeToString(compressorType), {});
+                VB->m_Operations.resize(1);
+                VB->m_Operations[0] = op;
+            }
+            op->SetAccuracy(VB->GetAccuracyRequested());
 
+            {
+                std::lock_guard<std::mutex> lockGuard(mutexDecompress);
+                core::Decompress(
+                    IncomingData,
+                    ((MetaArrayRecOperator *)writer_meta_base)->DataBlockSize[Read.BlockID],
+                    decompressBuffer.data(), Req.MemSpace, op, m_Engine, VB);
+                VB->m_AccuracyProvided = op->GetAccuracy();
+            }
+            IncomingData = decompressBuffer.data();
+            VirtualIncomingData = IncomingData;
+        }
+        catch (...)
         {
-            std::lock_guard<std::mutex> lockGuard(mutexDecompress);
-            core::Decompress(
-                IncomingData,
-                ((MetaArrayRecOperator *)writer_meta_base)->DataBlockSize[Read.BlockID],
-                decompressBuffer.data(), Req.MemSpace, op, m_Engine, VB);
-            VB->m_AccuracyProvided = op->GetAccuracy();
+            std::exception_ptr ex = std::current_exception();
+            // if MakeOperator or Decompress throws an exception, we can't complete this request. To
+            // make it isn't tried again, we have to clear PendingGetRequests.  Also cleanup
+            // Read.DestinationAddr.
+            PendingGetRequests.clear();
+            if (freeAddr)
+            {
+                free((char *)Read.DestinationAddr);
+            }
+            std::rethrow_exception(ex);
         }
-        IncomingData = decompressBuffer.data();
-        VirtualIncomingData = IncomingData;
     }
     if (Req.Start.size())
     {

source/utils/bpls/bpls.cpp

@@ -32,6 +32,7 @@
 #include <errno.h>
 
 #include "adios2/helper/adiosLog.h"
+#include "adios2/operator/plugin/PluginOperator.h"
 
 #if defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER))
 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40900
@@ -97,6 +98,7 @@ bool attrsonly;          // do list attributes only
 bool longopt;            // -l is turned on
 bool timestep;           // read step by step
 bool ignore_flatten;     // dont flatten steps to one
+bool list_operators;     // list all operators used in the file
 bool filestream = false; // are we using an engine through FileStream?
 bool noindex;            // do no print array indices with data
 bool printByteAsChar;    // print 8 bit integer arrays as string
@@ -149,6 +151,7 @@ void display_help()
            "reading)\n"
            "  --ignore_flatten           Display steps as written (don't flatten, even if writer "
            "said to)\n"
+           "  --list_operators           List all operators used in the file\n"
            "  --dump      | -d           Dump matched variables/attributes\n"
            "                               To match attributes too, add option "
            "-a\n"
@@ -600,6 +603,8 @@ int bplsMain(int argc, char *argv[])
     arg.AddCallback("--help", argT::NO_ARGUMENT, optioncb_help, &arg, "Help");
     arg.AddCallback("-h", argT::NO_ARGUMENT, optioncb_help, &arg, "");
     arg.AddBooleanArgument("--dump", &dump, "Dump matched variables/attributes");
+    arg.AddBooleanArgument("--list_operators", &list_operators,
+                           "List the operators used in the file");
     arg.AddBooleanArgument("-d", &dump, "");
     arg.AddBooleanArgument("--long", &longopt,
                            "Print values of all scalars and attributes and min/max "
@@ -781,6 +786,7 @@ void init_globals()
     listmeshes = false;
     attrsonly = false;
     longopt = false;
+    list_operators = false;
     // timefrom             = 1;
     // timeto               = -1;
     use_regexp = false;
@@ -978,6 +984,68 @@ int printAttributeValue(core::Engine *fp, core::IO *io, core::Attribute<std::str
 
 int nEntriesMatched = 0;
 
+int doList_operators(core::Engine *fp, core::IO *io)
+{
+    const core::VarMap &variables = io->GetVariables();
+    std::set<std::string> OpStrings;
+
+    for (const auto &vpair : variables)
+    {
+        Entry e(vpair.second->m_Type, vpair.second.get());
+        if (e.var->m_Operations.size() > 0)
+        {
+            auto op = e.var->m_Operations[0];
+            if (op->m_TypeString == "null")
+            {
+                try
+                {
+                    if (e.typeName == DataType::Struct)
+                    {
+                        // not supported
+                    }
+#define declare_template_instantiation(T)                                                          \
+    else if (e.typeName == helper::GetDataType<T>())                                               \
+    {                                                                                              \
+        core::Variable<T> *variable = static_cast<core::Variable<T> *>(e.var);                     \
+        variable->SetBlockSelection(0);                                                            \
+        std::vector<T> dataV;                                                                      \
+        dataV.resize(variable->SelectionSize());                                                   \
+        fp->Get(*variable, dataV, adios2::Mode::Sync);                                             \
+    }
+                    ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation)
+#undef declare_template_instantiation
+                    op = e.var->m_Operations[0];
+                    auto plugin = dynamic_cast<plugin::PluginOperator *>(op.get());
+                    if (plugin)
+                    {
+                        OpStrings.insert(plugin->m_PluginLibrary + "(" + plugin->m_PluginName +
+                                         ")");
+                    }
+                    else
+                    {
+                        OpStrings.insert(op->m_TypeString);
+                    }
+                }
+                catch (MissingOperatorFailure const &ex)
+                {
+                    // we didn't compile with the operator used
+                    OpStrings.insert(ex.m_Operator);
+                }
+                catch (PluginLoadFailure const &ex)
+                {
+                    // plugin operator we didn't find
+                    OpStrings.insert(ex.m_PluginLibrary + "(" + ex.m_PluginName + ")");
+                }
+            }
+        }
+    }
+    std::cout << "Operators used in this file:" << std::endl;
+    for (const auto &opname : OpStrings)
+        std::cout << " * " << opname << std::endl;
+
+    return 0;
+}
+
 int doList_vars(core::Engine *fp, core::IO *io)
 {
 
@@ -1595,6 +1663,12 @@ int doList(std::string path)
     if (hidden_attrs)
         strcat(init_params, ";show_hidden_attrs");
 
+    if (list_operators && timestep)
+    {
+        fprintf(stderr,
+                "--list_operators incompatible with --timestep, turning off timestep mode\n");
+        timestep = false;
+    }
     core::ADIOS adios("C++");
     const adios2::UserOptions userOptions = adios.GetUserOptions();
 
@@ -1771,6 +1845,10 @@ int doList(std::string path)
         else
         {
             doList_vars(fp, &io);
+            if (list_operators)
+            {
+                doList_operators(fp, &io);
+            }
         }
 
         fp->Close();

testing/utils/cwriter/TestUtilsCWriter.bplsh.expected.txt

@@ -12,6 +12,7 @@ The time dimension is the first dimension then.
   --meshes    | -m           List meshes
   --timestep  | -t           Read content step by step (stream reading)
   --ignore_flatten           Display steps as written (don't flatten, even if writer said to)
+  --list_operators           List all operators used in the file
   --dump      | -d           Dump matched variables/attributes
                                To match attributes too, add option -a
   --regexp    | -e           Treat masks as extended regular expressions