Cron test script

cron-ci/README.md

@@ -0,0 +1,36 @@
+## Cron CI
+
+Provides a script that can be run using CRON on HPC systems to build
+and test hpc-stack.
+
+Set the variables in `setup_cron.sh` and then have cron run that
+script. It will then checkout hpc-stack and build and test hpc-stack
+using the ufs-weather-model regression tests.
+
+To save resources, a hash of the last build, `prev_hash.txt` is saved
+in `HPC_HOMEDIR` each time and if it doesn't change
+between runs the script will exit.
+
+## Variables
+
+Set these variables in `setup-cron.sh`
+
+* HPC_HOMEDIR - Path to store logs and save temporary data
+
+* HPC_INSTALL_PATH - Path to install hpc-stack to
+
+* HPC_DOWNLOAD_PATH - Path to download and build from
+
+* HPC_CONFIG - Custom config so compiler/MPI and other options can be set
+
+* HPC_STACK_FILE - The yaml file that specifies which libraries and versions to build. Default to config/stack_noaa.yaml
+
+* LOG_PATH - The path to write logs to. Defaults to HPC_HOMEDIR/logs.
+
+* HPC_MACHINE_ID - Name of machine (hera, orion, gaea, etc).
+This is used to edit the correct modules in ufs-weather-model/machine.compiler 
+
+* TEST_UFS - Run ufs-weather-model regression tests. Defaults to true.
+
+
+

cron-ci/build-hpc-stack.sh

@@ -0,0 +1,31 @@
+#!/bin/bash -l
+# pass -l (treat script as login shell) so lmod module environment is picked up
+set -eux
+
+rm -rf ${HPC_INSTALL_PATH}/*
+
+# Don't build these libs because NOAA RDHPCS systems can't use wget or connect to gitlab
+# finds the name of the library and changes the next line to "build: NO"
+sed -i '/^udunits/{n;s/.*/  build: NO/}' ${HPC_STACK_FILE}
+sed -i '/^szip/{n;s/.*/  build: NO/}' ${HPC_STACK_FILE}
+sed -i '/^nccmp/{n;s/.*/  build: NO/}' ${HPC_STACK_FILE}
+sed -i '/^cdo/{n;s/.*/  build: NO/}' ${HPC_STACK_FILE}
+sed -i '/^boost/{n;s/.*/  build: NO/}' ${HPC_STACK_FILE}
+sed -i '/^eigen/{n;s/.*/  build: NO/}' ${HPC_STACK_FILE}
+
+# pass no to setup_modules when asked for native compiler/mpi. Using modules.
+yes no | ./setup_modules.sh -c ${HPC_CONFIG} -p ${HPC_INSTALL_PATH}
+
+./build_stack.sh -c ${HPC_CONFIG} -p ${HPC_INSTALL_PATH} -y ${HPC_STACK_FILE} -m
+
+# Edit stack file to build debug version of ESMF
+# Turn everything to build: NO, then enable just ESMF, then set debug: YES
+sed -i 's/build: YES/build: NO/' ${HPC_STACK_FILE}
+sed -i '/^esmf/{n;s/.*/  build: YES/}' ${HPC_STACK_FILE}
+sed -i 's/debug: NO/debug: YES/' ${HPC_STACK_FILE}
+
+# Build debug version of ESMF
+./build_stack.sh -c ${HPC_CONFIG} -p ${HPC_INSTALL_PATH} -y ${HPC_STACK_FILE} -m
+
+
+

cron-ci/setup-cron.sh

@@ -0,0 +1,84 @@
+#!/bin/bash -l
+# pass -l (treat script as login shell) so lmod module environment is picked up
+
+# Set variables below and have cron run this script to download, build, and test hpc-stack
+set -eu
+
+# store logs and save previous hash here
+export HPC_HOMEDIR=~/.hpc-stack
+# path to install hpc-stack
+export HPC_INSTALL_PATH=
+# path to download hpc-stack and other components
+export HPC_DOWNLOAD_PATH=
+
+# hpc-stack config file to build with (set compiler and other options)
+# I put these in the HPC_HOMEDIR
+export HPC_CONFIG=${HPC_HOMEDIR}/
+# hpc-stack yaml file to build with, use stack_noaa by default
+export HPC_STACK_FILE=config/stack_noaa.yaml
+
+# set machine name (hera, jet, orion, etc) so script edits correct ufs modulefiles
+export HPC_MACHINE_ID=
+export LOG_PATH=${HPC_HOMEDIR}/logs
+
+# Run ufs-weather-model regression tests?
+export TEST_UFS=true
+
+# Create directories
+mkdir -p ${HPC_DOWNLOAD_PATH}
+mkdir -p ${HPC_HOMEDIR}
+mkdir -p ${LOG_PATH}
+
+cd $HPC_DOWNLOAD_PATH
+rm -rf hpc-stack
+
+# mm-dd-yyy-hh:mm
+hpc_logdate=$(date +'%m-%d-%Y-%R')
+hpc_logname=hpc-stack_${hpc_logdate}.log
+hpc_log=${LOG_PATH}/${hpc_logname}
+
+git clone https://github.com/NOAA-EMC/hpc-stack.git > /dev/null 2>&1
+cd hpc-stack
+
+# Parse hpc-stack yaml file to get version variables
+source stack_helpers.sh
+eval $(parse_yaml ${HPC_STACK_FILE} "STACK_")
+
+# get the current hpc-stack hash and compare it to the previously built hash (if it exists)
+# if it exists and matches current hash, don't build
+current_hash=$(git rev-parse HEAD)
+
+if [[ -f "${HPC_HOMEDIR}/prev_hash.txt" ]]; then
+    prev_hash=$(cat ${HPC_HOMEDIR}/prev_hash.txt)
+    if [[ "$current_hash" == "$prev_hash" ]]; then
+        echo `date`
+        echo ""
+        echo "hpc-stack version has not changed since last time. Not building."
+        echo "hpc-stack hash: ${current_hash}"
+        exit 0
+    fi
+fi
+
+echo `date`
+echo ""
+echo "building hpc-stack..."
+./cron-ci/build-hpc-stack.sh >> ${hpc_log} 2>&1
+
+# check if hpc-stack build succeded 
+if grep -qi "build_stack.sh: SUCCESS!" ${hpc_log}; then
+    echo "hpc-stack build: PASS"
+    echo "hpc-stack hash: ${current_hash}"
+    echo "hpc-stack log: ${hpc_log}"
+    echo ""
+else
+    echo "hpc-stack build: FAIL"
+    echo "hpc-stack hash: ${current_hash}"
+    echo "hpc-stack log: ${hpc_log}"
+    echo ""
+    exit 1
+fi
+
+# save the current hash as the previous hash to compare to next time
+echo $current_hash > $HPC_HOMEDIR/prev_hash.txt
+
+./cron-ci/test-applications.sh

cron-ci/test-applications.sh

@@ -0,0 +1,28 @@
+#!/bin/bash -l
+set -eu
+
+cron_dir=${HPC_DOWNLOAD_PATH}/hpc-stack/cron-ci
+
+if [[ "$TEST_UFS" == true ]]; then
+    # mm-dd-yyy-hh:mm
+    ufs_logdate=$(date +'%m-%d-%Y-%R')
+    ufs_logname=ufs_${ufs_logdate}.log
+    ufs_log=${LOG_PATH}/${ufs_logname}
+
+    echo ""
+    echo "testing ufs-weather-model..."
+    ${cron_dir}/test-ufs.sh >> ${ufs_log} 2>&1
+
+    ufs-hash=$(git -C ${HPC_DOWNLOAD_PATH}/ufs-weather-model rev-parse HEAD)
+    # check if ufs regression tests were successful
+    if grep -qi "REGRESSION TEST WAS SUCCESSFUL" ${ufs_log}; then
+        echo "UFS regression tests: PASS"
+    else
+        echo "UFS regression tests: FAIL"
+    fi
+    # Output data
+    echo ""
+    echo "ufs hash: ${ufs_hash}"
+    echo "UFS log: ${ufs_log}"
+fi
+

cron-ci/test-ufs.sh

@@ -0,0 +1,24 @@
+#!/bin/bash -l
+set -eux
+
+cd ${HPC_DOWNLOAD_PATH}
+rm -rf ufs-weather-model
+
+git clone https://github.com/ufs-community/ufs-weather-model.git
+cd ufs-weather-model
+git submodule update --init --recursive
+
+# change module use to new hpc-stack location
+sed -i "s:module use /.*/stack:module use ${HPC_INSTALL_PATH}/modulefiles/stack:" modulefiles/${HPC_MACHINE_ID}.intel/*
+
+# remove version from module load
+sed -i "s:module load \(.*\)/.*:module load \1:" modulefiles/${HPC_MACHINE_ID}.intel/*
+
+# give version to ESMF because two versions are built
+sed -i "s:module load esmf:module load esmf/${STACK_esmf_version}:" modulefiles/${HPC_MACHINE_ID}.intel/fv3
+sed -i "s:module load esmf:module load esmf/${STACK_esmf_version}-debug:" modulefiles/${HPC_MACHINE_ID}.intel/fv3_debug
+
+cd tests
+./rt.sh -f -e
+
+