Unpin or update many packages (mostly Python) in configs/common/packages.yaml, fix S4 site config (#1384)

In preparation for spack-stack-1.9.0, this PR unpins or updates several packages in configs/common/packages.yaml (mostly Python packages). Most notably, py-shapely and py-numpy are updated.

The py-numpy update may require bug fixes with the Intel classic compiler that @DavidHuber-NOAA worked on and that are currently under review in spack develop (see #1276).

Included is an update of the S4 site config, which had several flaws that prevented building and testing this PR.

Author: climbfuji

configs/common/packages.yaml

@@ -26,7 +26,16 @@ packages:
   cairo:
     require: '+pic'
   cdo:
-    require: '@2.3.0 ~openmp'
+    require:
+    - any_of: ['@2.3.0 ~openmp']
+      when: '%intel@2021:'
+      message: '2.3.0 is the last version to use C++17, use with Intel Classic'
+    - any_of: ['@2.3.0 ~openmp']
+      when: '%apple-clang@:14'
+      message: '2.3.0 is the last version to use C++17, use with Apple Clang 14 and earlier'
+    - any_of: ['@2.0.5 ~openmp']
+      when: '%intel@:2020'
+      message: '2.0.5 is the last version to use C++14, use with Intel Classic 2020 and earlier'
   cmake:
     version: ['3.27.9']
     require: '+ownlibs'
@@ -142,11 +151,6 @@ packages:
   mapl:
     require: '@2.46.3 ~shared ~f2py'
     variants: '+pflogger'
-  # To avoid duplicate packages being built
-  meson:
-    require:
-    - '@1.5.1:'
-  # If making changes here, also check the Discover site configs and the CI workflows
   met:
     require: '@11.1.1 +python +grib2'
   metplus:
@@ -156,7 +160,7 @@ packages:
   mpich:
     require: '~hwloc'
   nco:
-    require: '@5.1.6 ~doc'
+    require: '@5.2.4 ~doc'
   # ncview - when adding information here, also check Orion
   # and Discover site configs
   nemsio:
@@ -202,9 +206,6 @@ packages:
     require: '~tiff'
   python:
     require: '@3.11.7'
-  py-attrs:
-    # https://github.com/JCSDA/spack-stack/issues/740
-    require: '@21.4.0'
   py-cartopy:
     require: '+plotting'
   # Avoid full rust dependency: version 43+ requires py-maturin
@@ -226,42 +227,23 @@ packages:
     require: '@3.8.0'
   py-h5py:
     require: '~mpi'
-  ## To avoid duplicate packages
-  #py-jinja2:
-  #  require: '@3.1.2'
   # https://github.com/JCSDA/spack-stack/issues/1276
   py-matplotlib:
      require: '@3.7.4'
-  py-meson-python:
-     require: '@0.15.0'
-  # Comment out for now until build problems are solved
-  # https://github.com/jcsda/spack-stack/issues/522
-  # see also ewok-env virtual package and container
-  # README.md
-  #py-mysql-connector-python:
-  #  require: '@8.0.32'
+  # Variant ~mpi not working for latest py-netcdf4
+  # https://github.com/spack/spack/issues/47652
   py-netcdf4:
-    require: '@1.5.8 ~mpi'
-  # [email protected] causes many build problems with older Python packages
-  # also check Nautilus site config when making changes here
-  # https://github.com/JCSDA/spack-stack/issues/1276
-  py-numpy:
-    require:
-    - '@:1.23.5'
+    require: '@1.7.1 +mpi'
   py-pandas:
     require: '+excel'
-  py-pybind11:
-    require: '@2.11.0'
   # To avoid duplicate packages
   py-ruamel-yaml:
     require: '@0.17.16'
   # Pin the py-setuptools version to avoid duplicate Python packages
   py-setuptools:
-    require: '@63.4.3'
+    require: '@69'
   py-setuptools-rust:
     require: '+rust_bootstrap'
-  py-shapely:
-    require: '@1.8.0'
   py-torch:
     require: '+custom-protobuf ~mkldnn'
   # To avoid duplicate packages
@@ -297,4 +279,4 @@ packages:
   wrf-io:
     require: '@1.2.0'
   zstd:
-    require: '@1.5.2 +programs'
+    require: '@1.5.6 +programs'

configs/sites/tier1/aws-pcluster/packages_intel.yaml

@@ -31,5 +31,5 @@ packages:
     - '@1.2.1 ~mkl'
   py-numpy:
     require::
-    - '@:1.23 ^openblas'
+    - '^openblas'
   # *DH

configs/sites/tier1/derecho/packages_intel.yaml

@@ -21,4 +21,3 @@ packages:
   # Newer versions of patchelf (tested 0.18.0) don't build with Intel on Derecho
   patchelf:
     version:: ['0.17.2']
-

configs/sites/tier1/gaea-c5/packages.yaml

@@ -35,7 +35,7 @@ packages:
     - '@1.2.1 ~mkl'
   py-numpy:
     require::
-    - '@:1.23 ^openblas'
+    - '^openblas'
   # *DH
 
 ### All other external packages listed alphabetically

configs/sites/tier1/gaea-c6/packages.yaml

@@ -34,7 +34,7 @@ packages:
     - '@1.2.1 ~mkl'
   py-numpy:
     require::
-    - '@:1.23 ^openblas'
+    - '^openblas'
   # *DH
 
 ### All other external packages listed alphabetically

configs/sites/tier1/hera/packages_intel.yaml

@@ -30,7 +30,7 @@ packages:
     - '@1.2.1 ~mkl'
   py-numpy:
     require::
-    - '@:1.23 ^openblas'
+    - '^openblas'
   # *DH
   zlib-ng:
     require:

configs/sites/tier1/hercules/packages_intel.yaml

@@ -30,4 +30,4 @@ packages:
     - '@1.2.1 ~mkl'
   py-numpy:
     require::
-    - '@:1.23 ^openblas'
+    - '^openblas'

configs/sites/tier1/nautilus/packages_gcc.yaml

@@ -31,4 +31,4 @@ packages:
     - '@1.2.1 +mkl'
   py-numpy:
     require::
-    - '@:1.23 ^intel-oneapi-mkl'
+    - '^intel-oneapi-mkl'

configs/sites/tier1/noaa-aws/packages_intel.yaml

@@ -30,5 +30,5 @@ packages:
     - '@1.2.1 ~mkl'
   py-numpy:
     require::
-    - '@:1.23 ^openblas'
+    - '^openblas'
   # *DH

configs/sites/tier1/noaa-azure/packages_intel.yaml

@@ -30,5 +30,5 @@ packages:
     - '@1.2.1 ~mkl'
   py-numpy:
     require::
-    - '@:1.23 ^openblas'
+    - '^openblas'
   # *DH

configs/sites/tier1/noaa-gcloud/packages_intel.yaml

@@ -30,5 +30,5 @@ packages:
     - '@1.2.1 ~mkl'
   py-numpy:
     require::
-    - '@:1.23 ^openblas'
+    - '^openblas'
   # *DH

configs/sites/tier1/orion/packages_intel.yaml

@@ -30,4 +30,4 @@ packages:
     - '@1.2.1 ~mkl'
   py-numpy:
     require::
-    - '@:1.23 ^openblas'
+    - '^openblas'

configs/sites/tier1/s4/compilers.yaml

@@ -13,9 +13,9 @@ compilers:
     - intel/2023.2
     environment:
       prepend_path:
-        PATH: /opt/gcc/13.3/bin
-        LD_LIBRARY_PATH: /opt/intel/oneapi/2023.2/compiler/2023.2.0/linux/compiler/lib/intel64_lin:/opt/gcc/13.3/lib64:/opt/gcc/13.3/lib
-        CPATH: /opt/gcc/13.3/include
+        PATH: /opt/gcc/10.2.0/bin
+        LD_LIBRARY_PATH: /opt/intel/oneapi/2023.2/compiler/2023.2.0/linux/compiler/lib/intel64_lin:/opt/gcc/10.2.0/lib64:/opt/gcc/10.2.0/lib
+        CPATH: /opt/gcc/10.2.0/include
     extra_rpaths: []
 - compiler:
     spec: gcc@=13.3.0
@@ -31,6 +31,20 @@ compilers:
     - gcc/13.3
     environment: {}
     extra_rpaths: []
+- compiler:
+    spec: gcc@=10.2.0
+    paths:
+      cc: /opt/gcc/10.2.0/bin/gcc
+      cxx: /opt/gcc/10.2.0/bin/g++
+      f77: /opt/gcc/10.2.0/bin/gfortran
+      fc: /opt/gcc/10.2.0/bin/gfortran
+    flags: {}
+    operating_system: rocky8
+    target: x86_64
+    modules:
+    - gcc/10.2
+    environment: {}
+    extra_rpaths: []
 - compiler:
     spec: gcc@=8.5.0
     paths:

configs/sites/tier1/s4/packages.yaml

@@ -1,12 +1,15 @@
 packages:
   all:
-    compiler:: [[email protected],gcc@13.3.0]
+    compiler:: [[email protected],gcc@10.2.0]
     providers:
       mpi:: [[email protected]]
       # Remove the next three lines to switch to intel-oneapi-mkl
       blas:: [openblas]
       fftw-api:: [fftw]
       lapack:: [openblas]
+    # spack concretizer gets confused and wants an AVX512 architecture for some packages
+    require:
+      - 'arch=linux-rocky8-skylake'
 
 ### MPI, Python, MKL
   mpi:
@@ -34,7 +37,7 @@ packages:
     - '@1.2.1 ~mkl'
   py-numpy:
     require::
-    - '@:1.23 ^openblas'
+    - '^openblas'
 
 ### All other external packages listed alphabetically
   autoconf:
@@ -77,10 +80,11 @@ packages:
     externals:
     - spec: [email protected]
       prefix: /usr
-  curl:
-    externals:
-    - spec: [email protected]+gssapi+ldap+nghttp2
-      prefix: /usr
+  # OS curl/openssl cause problems 
+  #curl:
+  #  externals:
+  #  - spec: [email protected]+gssapi+ldap+nghttp2
+  #    prefix: /usr
   diffutils:
     externals:
     - spec: [email protected]
@@ -145,10 +149,11 @@ packages:
     externals:
     - spec: [email protected]
       prefix: /usr
-  openssl:
-    externals:
-    - spec: [email protected]
-      prefix: /usr
+  # OS curl/openssl cause problems
+  #openssl:
+  #  externals:
+  #  - spec: [email protected]
+  #    prefix: /usr
   perl:
     externals:
     - spec: [email protected]~cpanm+opcode+open+shared+threads
@@ -157,14 +162,11 @@ packages:
     externals:
     - spec: [email protected]
       prefix: /usr
-  # TODO - installing via spack-stack works, for example for
-  # spack-stack-1.8.0, but would be good if we didn't have to.
-  # https://github.com/JCSDA/spack-stack/issues/1329
-  #qt:
-  #  buildable: False
-  #  externals:
-  #  - spec: [email protected]
-  #    prefix: /usr
+  qt:
+    buildable: False
+    externals:
+    - spec: [email protected]
+      prefix: /usr
   rsync:
     externals:
     - spec: [email protected]