ci,python: enable python in windows builds

.github/workflows/everything.yml

@@ -363,14 +363,18 @@ jobs:
       matrix:
         os: [win2019, win2022]
         shared: [shared]
-        parallel: [serial, msmpi]
+        parallel: [msmpi]
         include:
           - os: win2019
             image: windows-2019
             compiler: vs2019
           - os: win2022
             image: windows-2022
             compiler: vs2022
+          - os: win2022
+            image: windows-2022
+            compiler: vs2022
+            parallel: serial
           - os: win2022
             image: windows-2022
             shared: static

bindings/Python/CMakeLists.txt

@@ -29,20 +29,31 @@ target_link_libraries(adios2_py PRIVATE
   adios2::thirdparty::pybind11
   ${maybe_mpi4py}
   )
+
 configure_file(
   ${CMAKE_CURRENT_SOURCE_DIR}/__init__.py.in
   ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings/__init__.py
   @ONLY
 )
 
+# usercustomize.py will be read by python before everything else.
+# This is needed for Windows since Windows requires to point Python to the dll
+# locations of ADIOS2 during our unit tests execution.
+configure_file(
+  ${CMAKE_CURRENT_SOURCE_DIR}/usercustomize.py.in
+  ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/usercustomize.py
+  @ONLY
+)
+
+# We need the $<0:> for the Windows python unittests to work
 set_target_properties(adios2_py PROPERTIES
   CXX_VISIBILITY_PRESET hidden
   OUTPUT_NAME adios2_bindings${ADIOS2_LIBRARY_SUFFIX}
-  ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
-  LIBRARY_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
-  RUNTIME_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
-  PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
-  COMPILE_PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
+  ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings$<0:>
+  LIBRARY_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings$<0:>
+  RUNTIME_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings$<0:>
+  PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings$<0:>
+  COMPILE_PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings$<0:>
 )
 
 set(install_location "${CMAKE_INSTALL_PYTHONDIR}/adios2")

bindings/Python/usercustomize.py.in

@@ -0,0 +1,12 @@
+"""This script is used to setup the testing environment for the project."""
+
+import sys
+
+if sys.platform.startswith("win"):
+    import os
+    from pathlib import Path
+    os.add_dll_directory(Path(__file__).parent)
+    for build_type in "@CMAKE_CONFIGURATION_TYPES@".split(";"):
+        runtime_dir = Path("@CMAKE_RUNTIME_OUTPUT_DIRECTORY@") / build_type
+        if runtime_dir.is_dir():
+            os.add_dll_directory(runtime_dir)

cmake/ADIOSFunctions.cmake

@@ -86,12 +86,14 @@ function(python_add_test)
     set_property(TEST ${ARGS_NAME} PROPERTY
       ENVIRONMENT
         "PYTHONPATH=${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR}:$ENV{PYTHONPATH}"
+        "PYTHONUSERBASE=${CMAKE_BINARY_DIR}"
     )
   else()
     set_property(TEST ${ARGS_NAME} PROPERTY
       ENVIRONMENT
-        "PYTHONPATH=${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR};$ENV{PYTHONPATH}"
-        "PATH=$<TARGET_FILE_DIR:adios2_py>;$ENV{PATH}"
+        "PYTHONPATH=${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR};$<SHELL_PATH:$<TARGET_FILE_DIR:adios2_py>>;$ENV{PYTHONPATH}"
+        "PATH=$<SHELL_PATH:$<TARGET_FILE_DIR:adios2_py>>;$<SHELL_PATH:$<TARGET_FILE_DIR:adios2_core>>;$ENV{PATH}"
+        "PYTHONUSERBASE=${CMAKE_BINARY_DIR}"
     )
   endif()
 endfunction()

scripts/ci/cmake/ci-win2019-vs2019-msmpi.cmake

@@ -10,11 +10,17 @@ ADIOS2_BUILD_EXAMPLES:BOOL=ON
 ADIOS2_USE_BZip2:BOOL=OFF
 ADIOS2_USE_Fortran:BOOL=OFF
 ADIOS2_USE_MPI:BOOL=ON
-ADIOS2_USE_Python=OFF
+ADIOS2_USE_Python:BOOL=ON
+
+Python_ROOT_DIR:PATH=$ENV{CMAKE_PREFIX_PATH}
+Python_FIND_STRATEGY:STRING=LOCATION
+Python_FIND_FRAMEWORK:STRING=FIRST
 ")
 
 set(CTEST_TEST_TIMEOUT 500)
 set(CTEST_CMAKE_GENERATOR "Visual Studio 16 2019")
 set(CTEST_CMAKE_GENERATOR_PLATFORM "x64")
 list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
+# https://github.com/ornladios/ADIOS2/issues/4276
+set(CTEST_TEST_ARGS EXCLUDE "Api.Python.FileReader|Api.Python.BPWriteTypesHighLevelAPI_HDF5")
 include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)

scripts/ci/cmake/ci-win2022-vs2022-msmpi.cmake

@@ -12,10 +12,16 @@ ADIOS2_USE_Fortran:BOOL=OFF
 ADIOS2_USE_MPI:BOOL=ON
 ADIOS2_USE_HDF5:STRING=ON
 ADIOS2_USE_HDF5_VOL:STRING=OFF
-ADIOS2_USE_Python=OFF
+ADIOS2_USE_Python:BOOL=ON
+
+Python_ROOT_DIR:PATH=$ENV{CMAKE_PREFIX_PATH}
+Python_FIND_STRATEGY:STRING=LOCATION
+Python_FIND_FRAMEWORK:STRING=FIRST
 ")
 
 set(CTEST_CMAKE_GENERATOR "Visual Studio 17 2022")
 set(CTEST_CMAKE_GENERATOR_PLATFORM "x64")
 list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
+# https://github.com/ornladios/ADIOS2/issues/4276
+set(CTEST_TEST_ARGS EXCLUDE "Api.Python.FileReader|Api.Python.BPWriteTypesHighLevelAPI_HDF5")
 include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)

scripts/ci/cmake/ci-win2022-vs2022-serial.cmake

@@ -14,11 +14,18 @@ ADIOS2_USE_BZip2:BOOL=OFF
 ADIOS2_USE_Fortran:BOOL=OFF
 ADIOS2_USE_MPI:BOOL=OFF
 ADIOS2_USE_HDF5:STRING=ON
-ADIOS2_USE_Python:BOOL=OFF
+ADIOS2_USE_Python:BOOL=ON
 ADIOS2_USE_HDF5_VOL:STRING=OFF
+
+Python_ROOT_DIR:PATH=$ENV{CMAKE_PREFIX_PATH}
+Python_FIND_STRATEGY:STRING=LOCATION
+Python_FIND_FRAMEWORK:STRING=FIRST
 ")
 
 set(CTEST_CMAKE_GENERATOR "Visual Studio 17 2022")
 set(CTEST_CMAKE_GENERATOR_PLATFORM "x64")
 list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
+# https://github.com/ornladios/ADIOS2/issues/4276
+set(CTEST_TEST_ARGS EXCLUDE "Api.Python.FileReader|Api.Python.BPWriteTypesHighLevelAPI_HDF5")
+
 include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)

scripts/ci/gh-actions/conda-env-win.yml

@@ -4,4 +4,5 @@ variables:
 dependencies:
   - c-blosc2
   - numpy>=1.19
+  - python=3.10
   - hdf5

scripts/ci/gh-actions/windows-setup.ps1

@@ -16,4 +16,3 @@ if($Env:GH_YML_MATRIX_PARALLEL -eq "msmpi")
   conda.bat install -n adios2 -c conda-forge -y "msmpi" "mpi4py"
   Write-Host "::endgroup::"
 }
-