Add old output files and ctests to see if we can still read them

source/adios2/engine/bp5/BP5Writer.cpp

@@ -1406,7 +1406,7 @@ void BP5Writer::MakeHeader(std::vector<char> &buffer, size_t &position, const st
 
     if (sizeof(BP5IndexTableHeader) != 64)
     {
-        std::cerr << "BP6 Index Table Header must be 64 bytes" << std::endl;
+        std::cerr << "BP5 Index Table Header must be 64 bytes" << std::endl;
         exit(1);
     }
 

source/adios2/toolkit/format/bp5/BP5Helper.cpp

@@ -404,6 +404,8 @@ void BP5Helper::BP5AggregateInformation(helper::Comm &mpiComm,
     if (AttributeEncodeBuffers.size() > 0)
     {
         size_t AlignedSize = ((AttributeEncodeBuffers[0].iov_len + 7) & ~0x7);
+        memset((char *)AttributeEncodeBuffers[0].iov_base + AttributeEncodeBuffers[0].iov_len, 0,
+               AlignedSize - AttributeEncodeBuffers[0].iov_len);
         attrLen = AlignedSize;
     }
     if ((attrLen > 0) && AttributeEncodeBuffers[0].iov_base)

testing/adios2/CMakeLists.txt

@@ -14,6 +14,7 @@ add_subdirectory(performance)
 add_subdirectory(helper)
 add_subdirectory(hierarchy)
 add_subdirectory(backward_compatibility)
+add_subdirectory(output_archive)
 if (ADIOS2_HAVE_Derived_Variable)
 add_subdirectory(derived)
 endif()

testing/adios2/output_archive/CMakeLists.txt

@@ -0,0 +1,93 @@
+#
+# For each test set <S>, we expect to find Write<S>.cpp and Read<S>.cpp as programs to be built.
+# 
+# We also expect in the zipped_output directory that all zip files
+# with the prefix <S> are the compressed contents of bpfiles that can
+# be read with Read<S>
+# 
+# Write<S>.cpp is expected to be use MPI and we'll (eventually) try to
+# create new files with mpirun -n 4 Write<S> ${ENGINE} ${OUTPUT_FILE}
+# 
+# Read<S>.cpp should not require MPI, but should be GTest enabled.  We'll run it as Read<S> ${INPUT_FILE}
+# 
+# We'll generate CTest entries for each zipped file we find in
+# zipped_output. We expect the filename to start with "<S>.", where
+# <S> is one if the TestSet names in the TestSets variable below.
+# Current output files look like <S>.<version info>.<engine>.zip.  But the
+# <engine> isn't provided to the reader, and the <version info> isn't
+# interpreted.  <Version info> could/should also include relevant
+# architecture information, like byte order.
+#
+include(ADIOSFunctions)
+
+set (TestSets Attribute Common)
+
+add_executable(archive_utility archive_utility.c)
+
+foreach (TestSet ${TestSets})
+
+    add_executable(Read${TestSet} Read${TestSet}.cpp)
+    target_link_libraries(Read${TestSet} adios2::cxx11 adios2::c adios2_core adios2::thirdparty::gtest)
+    if(ADIOS2_HAVE_MPI)
+        add_executable(Write${TestSet} Write${TestSet}.cpp)
+        target_link_libraries(Write${TestSet} adios2::cxx11_mpi adios2::c_mpi adios2_core_mpi MPI::MPI_CXX adios2::thirdparty::gtest)
+    endif()
+
+    file(GLOB ${TestSet}S ${CMAKE_CURRENT_SOURCE_DIR}/zipped_output/${TestSet}*.zip)
+    set (OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+    list(REMOVE_DUPLICATES ${TestSet}S)
+
+    foreach (${TestSet}OUT ${${TestSet}S})
+        get_filename_component(NO_ZIP ${${TestSet}OUT} NAME_WLE)
+	get_filename_component(UNIQNAME ${NO_ZIP} NAME)
+	#
+	# Note:  cmake tar extracts to the current working directory.
+	# SetupTestPipeline changes the WD to ${CMAKE_CURRENT_BINARY_DIR}/<pipelinename>
+	# So we have to use that as the source of the read.
+	#
+        add_test(NAME Archive.${UNIQNAME}.Extract
+            COMMAND ${CMAKE_COMMAND} -E tar xv "${${TestSet}OUT}"
+	)
+        add_test(NAME Archive.${UNIQNAME}.Read COMMAND Read${TestSet} "${CMAKE_CURRENT_BINARY_DIR}/Archive.${UNIQNAME}")
+	SetupTestPipeline(Archive.${UNIQNAME} "Extract;Read" TRUE)
+    endforeach()
+
+
+    # The code below is for creating "new" output and failing a test
+    # if it's unique so that it gets added to the test set.  This is not
+    # currently enabled.
+    #
+    set(NewFileEngineList )
+    if(ADIOS2_HAVE_MPI)
+        #  Engines to create new files with  nothing if no MPI  (Not doing this for the moment)
+	#	set(NewFileEngineList bp5)
+    endif()
+
+    foreach	(Engine ${NewFileEngineList})
+        add_test(NAME Archive.${TestSet}.${Engine}.Write
+            COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_EXTRA_FLAGS}
+		${MPIEXEC_NUMPROC_FLAG} 4 $<TARGET_FILE:Write${TestSet}>
+		${Engine} ${OUTPUT_DIRECTORY}/Current${TestSet}.${Engine}
+	)
+	set_tests_properties(Archive.${TestSet}.${Engine}.Write PROPERTIES PROCESSORS 4)
+
+	add_test(NAME Archive.${TestSet}.${Engine}.RemoveVersionInfo
+	    COMMAND archive_utility -make_version_neutral ${OUTPUT_DIRECTORY}/Current${TestSet}.${Engine}
+	)
+
+	add_test(NAME Archive.${TestSet}.${Engine}.Zip
+	    COMMAND ${CMAKE_COMMAND} -DSOURCE_DIR=${OUTPUT_DIRECTORY}/Current${TestSet}.${Engine}
+	  	-DARCHIVE_NAME=${OUTPUT_DIRECTORY}/Current${TestSet}.${Engine}.zip 
+		-P ${CMAKE_CURRENT_SOURCE_DIR}/create_archive.cmake
+	)
+
+	add_test(NAME Archive.${TestSet}.${Engine}.Unique
+	    COMMAND archive_utility -test_unique ${OUTPUT_DIRECTORY}/Current${TestSet}.${Engine}.zip ${${TestSet}S}
+	)
+
+
+	SetupTestPipeline(Archive.${TestSet}.${Engine} "Write;RemoveVersionInfo;Zip;Unique" TRUE)
+    endforeach()
+# 
+endforeach()  # over testsets
+

testing/adios2/output_archive/ReadAttribute.cpp

@@ -0,0 +1,112 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ */
+
+#include <gtest/gtest.h>
+#include <ios>       //std::ios_base::failure
+#include <iostream>  //std::cout
+#include <stdexcept> //std::invalid_argument std::exception
+#include <string>
+#include <vector>
+
+#include <adios2.h>
+
+class AttributeReadTest : public ::testing::Test
+{
+public:
+    AttributeReadTest() = default;
+};
+
+std::string fname;
+
+// ADIOS2 Common read
+TEST_F(AttributeReadTest, ADIOS2AttributeRead)
+{
+    adios2::ADIOS adios;
+
+    /*** IO class object: settings and factory of Settings: Variables,
+     * Parameters, Transports, and Execution: Engines */
+    adios2::IO bpIO = adios.DeclareIO("BPFile_N2N");
+
+    adios2::Engine bpReader = bpIO.Open(fname, adios2::Mode::ReadRandomAccess);
+    EXPECT_TRUE(bpReader);
+
+    std::vector<float> myFloats;
+    adios2::Variable<float> bpFloats = bpIO.InquireVariable<float>("bpFloats");
+    EXPECT_TRUE(bpFloats);
+
+    bpReader.Get<float>(bpFloats, myFloats, adios2::Mode::Sync);
+
+    EXPECT_GT(myFloats.size(), 9); // at least one MPI rank
+
+    for (size_t i = 0; i < myFloats.size(); i++)
+    {
+        EXPECT_FLOAT_EQ(myFloats[i], (float)i);
+    }
+    const auto attributesInfo = bpIO.AvailableAttributes();
+
+    EXPECT_EQ(attributesInfo.size(), 4);
+
+    auto StrAtt = bpIO.InquireAttribute<std::string>("Single_String");
+    EXPECT_TRUE(StrAtt);
+
+    auto StrArray = StrAtt.Data();
+    EXPECT_EQ(StrArray.size(), 1);
+    EXPECT_STREQ(StrArray[0].c_str(), "File generated with ADIOS2");
+
+    auto StrAtt2 = bpIO.InquireAttribute<std::string>("Array_of_Strings");
+    EXPECT_TRUE(StrAtt2);
+
+    auto StrArray2 = StrAtt2.Data();
+    EXPECT_EQ(StrArray2.size(), 3);
+    EXPECT_STREQ(StrArray2[0].c_str(), "one");
+    EXPECT_STREQ(StrArray2[1].c_str(), "two");
+    EXPECT_STREQ(StrArray2[2].c_str(), "three");
+
+    auto DblAtt2 = bpIO.InquireAttribute<double>("Array_of_Doubles");
+    EXPECT_TRUE(DblAtt2);
+
+    auto DblArray2 = DblAtt2.Data();
+    EXPECT_EQ(DblArray2.size(), 10);
+    for (size_t i = 0; i < 10; i++)
+    {
+        EXPECT_DOUBLE_EQ(DblArray2[i], (double)i + 1);
+    }
+
+    auto DblAtt = bpIO.InquireAttribute<double>("Attr_Double");
+    EXPECT_TRUE(DblAtt);
+
+    auto DblArray = DblAtt.Data();
+    EXPECT_EQ(DblArray.size(), 1);
+    EXPECT_DOUBLE_EQ(DblArray[0], 0.0);
+
+#ifdef VERBOSE_OUTPUT
+    for (const auto &attributeInfoPair : attributesInfo)
+    {
+        std::cout << "Attribute: " << attributeInfoPair.first;
+        for (const auto &attributePair : attributeInfoPair.second)
+        {
+            std::cout << "\tKey: " << attributePair.first << "\tValue: " << attributePair.second
+                      << "\n";
+        }
+        std::cout << "\n";
+    }
+#endif
+    bpReader.Close();
+}
+
+int main(int argc, char *argv[])
+{
+    int result;
+    if (argc > 1)
+    {
+        fname = std::string(argv[1]);
+    }
+    ::testing::InitGoogleTest(&argc, argv);
+
+    result = RUN_ALL_TESTS();
+
+    return result;
+}