Build python virtual environments.

.github/workflows/build_macOS.yaml

@@ -48,7 +48,7 @@ jobs:
         export HPC_COMPILER="${{ matrix.compiler }}/${gcc_ver}"
         export HPC_MPI="${{ matrix.mpi }}/${mpi_ver}"
         export HPC_PYTHON="python/${python_ver}"
-        ./build_stack.sh -p $prefix -c config/config_mac.sh -y config/stack_mac.yaml
+        ./build_stack.sh -p $prefix -c config/config_mac.sh -y stack/stack_mac.yaml
 
     - name: Upload logs
       if: ${{ always() }}

.github/workflows/build_ubuntu.yaml

@@ -50,7 +50,7 @@ jobs:
         export HPC_COMPILER="gnu/${gnu_ver}"
         export HPC_MPI="${{ matrix.mpi }}/${mpi_ver}"
         export HPC_PYTHON="python/${python_ver}"
-        ./build_stack.sh -p $prefix -c config/config_custom.sh -y config/stack_custom.yaml
+        ./build_stack.sh -p $prefix -c config/config_custom.sh -y stack/stack_custom.yaml
 
     - name: Upload logs
       if: ${{ always() }}

README.md

@@ -102,7 +102,7 @@ some of the parameter settings available.
 - **MAKE_VERBOSE:** Print out extra information to the log files during the build
 
 The next step is to choose what components of the stack you wish to
-build.  This is done by editing the file `config/stack_custom.yaml`
+build.  This is done by editing the file `stack/stack_custom.yaml`
 which defines the software packages to be built along with their
 version, options and compiler flags along with other package specific
 options.
@@ -174,10 +174,13 @@ script.
   - [EMC_post](https://github.com/noaa-emc/EMC_post.git)
 
 * JEDI Dependencies
-  - [ecbuild](https://github.com/jcsda/ecbuild.git)
-  - [eckit](https://github.com/jcsda/eckit.git)
-  - [fckit](https://github.com/jcsda/fckit.git)
-  - [atlas](https://github.com/jcsda/atlas.git)
+  - [ecbuild](https://github.com/ecmwf/ecbuild.git)
+  - [eckit](https://github.com/ecmwf/eckit.git)
+  - [fckit](https://github.com/ecmwf/fckit.git)
+  - [atlas](https://github.com/ecmwf/atlas.git)
+
+* Python Virtual Environments
+  - [r2d2](https://github.com/jcsda-internal/r2d2.git)
 
 **IMPORTANT: Steps 1, 2, and 3 need to be repeated for each
   compiler/MPI combination that you wish to install.** The new
@@ -270,7 +273,7 @@ default is to use `$HOME/opt` and `config/config_custom.sh`
 respectively.  `<yaml>` represents a user configurable yaml file
 containing a list of packages that need to be built in the stack along
 with their versions and package options. The default value of `<yaml>`
-is `config/stack_custom.yaml`.
+is `stack/stack_custom.yaml`.
 
 ## Additional Notes:
 
@@ -303,9 +306,11 @@ template
 2. define a new section in the `yaml` file for that library/package in
 config directory
 
-3. Add a call to the new build script in `build_stack.sh`
+3. if the package is a python virtual environment, add a `requirements.txt` file listing the python packages required to install the package in `pyvenv/package_name.txt`
+
+4. Add a call to the new build script in `build_stack.sh`
 
-4. Create a new module template at the appropriate place in the
+5. Create a new module template at the appropriate place in the
 modulefiles directory, using exising files as a template
 
 ### Configuring for a new HPC

build_stack.sh

@@ -33,7 +33,7 @@ usage() {
 library=""
 export PREFIX="$HOME/opt"
 config="${HPC_STACK_ROOT}/config/config_custom.sh"
-yaml="${HPC_STACK_ROOT}/config/stack_custom.yaml"
+yaml="${HPC_STACK_ROOT}/stack/stack_custom.yaml"
 export MODULES=false
 
 while getopts ":p:c:y:l:mh" opt; do
@@ -151,11 +151,6 @@ build_lib nco
 build_lib cdo
 build_lib pio
 
-# UFS 3rd party dependencies
-
-build_lib esmf
-build_lib fms
-
 # NCEPlibs
 
 build_lib bacio
@@ -183,6 +178,14 @@ build_lib prod_util
 build_lib grib_util
 build_lib ncio
 
+# Other
+
+build_lib madis
+
+# Python virtual environments
+
+build_lib r2d2
+
 # JEDI 3rd party dependencies
 
 build_lib boost
@@ -196,15 +199,17 @@ build_lib json
 build_lib json_schema_validator
 build_lib pybind11
 
-# JCSDA JEDI dependencies
+# JEDI dependencies
 
 build_lib ecbuild
 build_lib eckit
 build_lib fckit
 build_lib atlas
 
-# Other
-build_lib madis
+# UFS 3rd party dependencies
+
+build_lib esmf
+build_lib fms
 
 # ==============================================================================
 # optionally clean up

config/config_mac.sh

@@ -3,7 +3,7 @@
 # Compiler/MPI combination
 export HPC_COMPILER="clang/12.0.5"
 export HPC_MPI="mpich/3.3.2"
-export HPC_PYTHON="python/3.9.4"
+export HPC_PYTHON="python/3.9.5"
 
 # Build options
 export USE_SUDO=N

cron-ci/README.md

@@ -23,12 +23,12 @@ Set these variables in `setup-cron.sh`
 
 * HPC_CONFIG - Custom config so compiler/MPI and other options can be set
 
-* HPC_STACK_FILE - The yaml file that specifies which libraries and versions to build. Default to config/stack_noaa.yaml
+* HPC_STACK_FILE - The yaml file that specifies which libraries and versions to build. Default to stack/stack_noaa.yaml
 
 * LOG_PATH - The path to write logs to. Defaults to HPC_HOMEDIR/logs.
 
 * HPC_MACHINE_ID - Name of machine (hera, orion, gaea, etc).
-This is used to edit the correct modules in ufs-weather-model/machine.compiler 
+This is used to edit the correct modules in ufs-weather-model/machine.compiler
 
 * TEST_UFS - Run ufs-weather-model regression tests. Defaults to true.
 

cron-ci/setup-cron.sh

@@ -15,7 +15,7 @@ export HPC_DOWNLOAD_PATH=
 # I put these in the HPC_HOMEDIR
 export HPC_CONFIG=${HPC_HOMEDIR}/
 # hpc-stack yaml file to build with, use stack_noaa by default
-export HPC_STACK_FILE=config/stack_noaa.yaml
+export HPC_STACK_FILE=stack/stack_noaa.yaml
 
 # set machine name (hera, jet, orion, etc) so script edits correct ufs modulefiles
 export HPC_MACHINE_ID=
@@ -74,7 +74,7 @@ echo ""
 
 ./cron-ci/build-hpc-stack.sh >> ${hpc_log} 2>&1
 
-# check if hpc-stack build succeded 
+# check if hpc-stack build succeded
 if grep -qi "build_stack.sh: SUCCESS!" ${hpc_log}; then
     echo "hpc-stack build: PASS"
 else

libs/build_pyvenv.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+set -eux
+
+name=$1
+
+var="STACK_${name}_version"
+set +u
+stack_version=${!var}
+set -u
+version=${2:-$stack_version}
+
+var="STACK_${name}_requirements"
+set +u
+stack_rqmts=${!var}
+set -u
+rqmts=${stack_rqmts:-"requirements.txt"}
+
+python=$(echo $HPC_PYTHON | sed 's/\//-/g')
+
+if $MODULES; then
+  set +x
+  source $MODULESHOME/init/bash
+  module load hpc-$HPC_PYTHON
+  module list
+  set -x
+  prefix="${PREFIX:-"/opt/modules"}/$python/$name/$version"
+  if [[ -d $prefix ]]; then
+    [[ $OVERWRITE =~ [yYtT] ]] && ( echo "WARNING: $prefix EXISTS: OVERWRITING!";$SUDO rm -rf $prefix ) \
+                               || ( echo "WARNING: $prefix EXISTS, SKIPPING"; exit 1 )
+  fi
+else
+  nameUpper=$(echo $name | tr [a-z] [A-Z])
+  eval prefix="\${${nameUpper}_ROOT:-'/usr/local'}"
+fi
+
+# Determine python version; 3.x
+python_version=$(python3 -c 'import sys;print(sys.version_info[0],".",sys.version_info[1],sep="")')
+
+# Support python version >= 3.6
+min_python_version=3.6
+if (( $(echo "$min_python_version >= $python_version" | bc -l) )); then
+  echo "Must have python version ($python_version) >= ${min_python_version}. ABORT!"
+  exit 1
+fi
+
+# Check for requirements file
+rqmts_file=${HPC_STACK_ROOT}/pyvenv/$rqmts
+[[ ! -f $rqmts_file ]] && ( echo "Unable to find requirements file: $rqmts \nABORT!"; exit 1 )
+
+# for python >= v3.9, upgrade dependencies (pip, setuptools) in place with '--upgrade-deps'
+upgrade_deps=""
+if (( $(echo "$python_version >= 3.9" | bc -l) )); then
+  upgrade_deps="--upgrade-deps"
+fi
+
+# Create a new virtual env.
+python3 -m venv --prompt $name $upgrade_deps $prefix
+
+# Activate newly created virtual env.
+set +x
+source $prefix/bin/activate
+set -x
+
+# Upgrade pip
+pip install --upgrade pip
+
+# Install packages from $rqmt_file
+pip install --no-cache-dir -r $rqmts_file
+
+# List installed packages
+pip list
+
+# generate modulefile from template
+$MODULES && update_modules python $name $version $python_version
+echo $name $version $rqmts_file >> ${HPC_STACK_ROOT}/hpc-stack-contents.log

modulefiles/core/hpc-cray-python/hpc-cray-python.lua

@@ -17,7 +17,7 @@ load(python)
 prereq(python)
 
 local opt = os.getenv("HPC_OPT") or os.getenv("OPT") or "/opt/modules" or "/opt"
-local mpath = pathJoin(opt,"modulefiles","cray-python",pkgVersion)
+local mpath = pathJoin(opt,"modulefiles/python","cray-python",pkgVersion)
 prepend_path("MODULEPATH", mpath)
 
 whatis("Name: ".. pkgName)

modulefiles/core/hpc-miniconda3/hpc-miniconda3.lua

@@ -17,7 +17,7 @@ load(python)
 prereq(python)
 
 local opt = os.getenv("HPC_OPT") or os.getenv("OPT") or "/opt/modules"
-local mpath = pathJoin(opt,"modulefiles/core","miniconda3",pkgVersion)
+local mpath = pathJoin(opt,"modulefiles/python","miniconda3",pkgVersion)
 prepend_path("MODULEPATH", mpath)
 
 whatis("Name: ".. pkgName)

modulefiles/core/hpc-python/hpc-python.lua

@@ -17,7 +17,7 @@ load(python)
 prereq(python)
 
 local opt = os.getenv("HPC_OPT") or os.getenv("OPT") or "/opt/modules"
-local mpath = pathJoin(opt,"modulefiles/core","python",pkgVersion)
+local mpath = pathJoin(opt,"modulefiles/python","python",pkgVersion)
 prepend_path("MODULEPATH", mpath)
 
 whatis("Name: ".. pkgName)

modulefiles/python/pythonName/pythonVersion/r2d2/r2d2.lua

@@ -0,0 +1,36 @@
+help([[
+]])
+
+local pkgName    = myModuleName()
+local pkgVersion = myModuleVersion()
+local pkgNameVer = myModuleFullName()
+
+local hierA          = hierarchyA(pkgNameVer,1)
+local pythonNameVer  = hierA[1]
+local pythonNameVerD = pythonNameVer:gsub("/","-")
+
+conflict(pkgName)
+
+local opt = os.getenv("HPC_OPT") or os.getenv("OPT") or "/opt/modules"
+
+local base = pathJoin(opt,pythonNameVerD,pkgName,pkgVersion)
+
+-- activate on load, deactivate on unload
+if (mode() == "load") then
+  local source_cmd = "source " .. pathJoin(base, "bin/activate")
+  execute{cmd=source_cmd, modeA={"load"}}
+else
+  if (mode() == "unload") then
+    execute{cmd="deactivate", modeA={"unload"}}
+  end
+end
+
+-- The next 2 lines are automatically added (removed) by activate (deactivate)
+-- Hence, there is no need to add here.
+--prepend_path("PATH", pathJoin(base,"bin"))
+--prepend_path("PYTHONPATH", pathJoin(base,"lib/python@PYTHON_VERSION@/site-packages"))
+
+whatis("Name: ".. pkgName)
+whatis("Version: " .. pkgVersion)
+whatis("Category: Software")
+whatis("Description: R2D2 Python Virtual Environment")

pyvenv/r2d2.txt

@@ -0,0 +1,24 @@
+boto3 >= 1.16.38
+botocore >= 1.19.38
+cftime >= 1.3.0
+click >= 7.1.2
+Cython >= 0.29.21
+jmespath >= 0.10.0
+numpy >= 1.19.4
+pandas >= 1.1.5
+pip >= 20.3.3
+python-dateutil >= 2.8.1
+pytz >= 2020.4
+PyYAML >= 5.3.1
+s3transfer >= 0.3.3
+sdist >= 0.0.0
+six >= 1.15.0
+urllib3 >= 1.26.2
+wheel >= 0.36.2
+xarray >= 0.16.2
+
+netcdf4 >= 1.5.5.1 --prefer-binary
+h5netcdf >= 0.8.1 --prefer-binary
+h5py >= 3.1.0 --prefer-binary
+git+https://github.com/jcsda-internal/solo.git
+git+https://github.com/jcsda-internal/r2d2.git

setup_modules.sh

@@ -86,6 +86,7 @@ pythonVersion=$(echo $HPC_PYTHON | cut -d/ -f2)
 #===============================================================================
 # Deploy directory structure for modulefiles
 
+$SUDO mkdir -p $PREFIX/modulefiles/python
 $SUDO mkdir -p $PREFIX/modulefiles/core
 $SUDO mkdir -p $PREFIX/modulefiles/compiler/$compilerName/$compilerVersion
 $SUDO mkdir -p $PREFIX/modulefiles/mpi/$compilerName/$compilerVersion/$mpiName/$mpiVersion