Support Arnold 7 and 6.2 in a single build

.github/workflows/main.yml

@@ -139,9 +139,13 @@ jobs:
       # below.
       run: |
         echo GAFFER_DEPENDENCIES_HASH=`python .github/workflows/main/installDependencies.py --archiveURL ${{ matrix.dependenciesURL }} --dependenciesDir ${{ env.GAFFER_BUILD_DIR }} --outputFormat "{archiveDigest}"` >> $GITHUB_ENV
-        ./config/installArnold.sh
-        ./config/installDelight.sh
-        echo ARNOLD_ROOT=$GITHUB_WORKSPACE/arnoldRoot >> $GITHUB_ENV
+        # Remove old IECoreArnold components from Cortex build,
+        # as we will be replacing them with our own.
+        ## \todo Remove when we update to Cortex 10.3.
+        rm ${{ env.GAFFER_BUILD_DIR }}/lib/libIECoreArnold.*
+        rm -r ${{ env.GAFFER_BUILD_DIR }}/python/IECoreArnold
+        rm -r ${{ env.GAFFER_BUILD_DIR }}/include/IECoreArnold
+        ./.github/workflows/main/installDelight.sh
         echo DELIGHT=$GITHUB_WORKSPACE/3delight >> $GITHUB_ENV
 
     - name: Cache
@@ -166,6 +170,24 @@ jobs:
         ${{ matrix.testRunner }} "${{ env.GAFFER_BUILD_DIR }}/bin/gaffer test"
         echo "::remove-matcher owner=unittest::"
 
+    - name: Build and test Arnold extension
+      run: |
+        for arnoldVersion in 6.2.0.1 7.0.0.0
+        do
+          # Install Arnold
+          ./.github/workflows/main/installArnold.sh $arnoldVersion
+          export ARNOLD_ROOT=$GITHUB_WORKSPACE/arnoldRoot/$arnoldVersion
+          # Build Arnold extension
+          scons -j 2 build BUILD_TYPE=${{ matrix.buildType }} OPTIONS=.github/workflows/main/sconsOptions
+          # Test Arnold extension
+          echo "::add-matcher::./.github/workflows/main/problemMatchers/unittest.json"
+          ${{ matrix.testRunner }} "${{ env.GAFFER_BUILD_DIR }}/bin/gaffer test IECoreArnoldTest GafferArnoldTest GafferArnoldUITest"
+          echo "::remove-matcher owner=unittest::"
+          # Publish ARNOLD_ROOT to the environment for subsequent steps,
+          # so we can build the docs for GafferArnold.
+          echo ARNOLD_ROOT=$ARNOLD_ROOT >> $GITHUB_ENV
+        done
+
     - name: Build Docs and Package
       # We currently experience sporadic hangs in the docs builds (mac), this
       # aborts them in a more timely fashion than the default 6hr timeout.

config/installArnold.sh → .github/workflows/main/installArnold.sh

@@ -37,7 +37,12 @@
 
 set -e
 
-arnoldVersion=6.2.0.1
+if [ -z $1 ] ; then
+	echo "Usage : installArnold.sh arnoldVersion" >&2
+	exit 1
+fi
+
+arnoldVersion=$1
 
 if [[ `uname` = "Linux" ]] ; then
 	arnoldPlatform=linux
@@ -49,12 +54,11 @@ url=forgithubci.solidangle.com/arnold/Arnold-${arnoldVersion}-${arnoldPlatform}.
 
 # Configure the login information, if this has been supplied.
 login=""
-# TODO: Remove the extra var checks (!= $) once we no longer need Azure support.
-if [ ! -z "${ARNOLD_LOGIN}" ] && [ "${ARNOLD_LOGIN:0:1}" != "$" ] && [ -z "${ARNOLD_PASSWORD}" ] && [ "${ARNOLD_PASSWORD:0:1}" != "$" ]; then
+if [ ! -z "${ARNOLD_LOGIN}" ] && [ ! -z "${ARNOLD_PASSWORD}" ] ; then
 	login="${ARNOLD_LOGIN}:${ARNOLD_PASSWORD}@"
 fi
 
-mkdir -p arnoldRoot && cd arnoldRoot
+mkdir -p arnoldRoot/$arnoldVersion && cd arnoldRoot/$arnoldVersion
 
 echo Downloading Arnold "https://${url}"
 curl -L https://${login}${url} -o Arnold-${arnoldVersion}-${arnoldPlatform}.tgz

.github/workflows/main/sconsOptions

@@ -40,7 +40,7 @@ ENV_VARS_TO_IMPORT = "PATH"
 
 BUILD_CACHEDIR = os.environ["GAFFER_CACHE_DIR"]
 
-ARNOLD_ROOT = os.environ["ARNOLD_ROOT"]
+ARNOLD_ROOT = os.environ.get( "ARNOLD_ROOT", "" )
 DELIGHT_ROOT = os.environ["DELIGHT"]
 
 BUILD_DIR = os.environ["GAFFER_BUILD_DIR"]

Changes.md

@@ -4,6 +4,7 @@
 Features
 --------
 
+- Arnold : Added support for Arnold 7, including multiple simultaneous interactive renders. This is in addition to continuing support for Arnold 6.2 - the appropriate version of GafferArnold is chosen automatically on startup based on the `$ARNOLD_ROOT` environment variable.
 - GraphEditor :
   - Added new "Focus Node" concept. Clicking on the top right of a node tags it as the focus node, and editors and viewers can be set to follow the focus node. This is useful when you have several viewers that you want to view the same node, but you don't want them to follow selection.
   - Nodes and connections which don't contribute to computing the Focus Node are now dimmed. This helps see what part of the graph is active, taking into account the current context.

SConstruct

@@ -681,6 +681,29 @@ else :
 		CPPPATH = [ "$BUILD_DIR/include/python$PYTHON_ABI_VERSION" ]
 	)
 
+###############################################################################################
+# Arnold configuration
+###############################################################################################
+
+arnoldInstallRoot = ""
+if env["ARNOLD_ROOT"] :
+	arnoldHeader = env.subst( "$ARNOLD_ROOT/include/ai_version.h" )
+	if not os.path.exists( arnoldHeader ) :
+		sys.stderr.write( "ERROR : unable to find \"{}\".\n".format( arnoldHeader ) )
+		Exit( 1 )
+
+	arnoldVersions = {}
+	for line in open( arnoldHeader ) :
+		m = re.match( "^#define AI_VERSION_(ARCH|MAJOR)_NUM\s*([0-9]+)", line )
+		if m :
+			arnoldVersions[m.group(1)] = m.group( 2 )
+
+	if set( arnoldVersions.keys() ) != { "ARCH", "MAJOR" } :
+		sys.stderr.write( "ERROR : unable to parse \"{}\".\n".format( arnoldHeader ) )
+		Exit( 1 )
+
+	arnoldInstallRoot = "${{BUILD_DIR}}/arnold/{ARCH}.{MAJOR}".format( **arnoldVersions )
+
 ###############################################################################################
 # Definitions for the libraries we wish to build
 ###############################################################################################
@@ -843,41 +866,88 @@ libraries = {
 		},
 	},
 
+	"IECoreArnold" : {
+		"envAppends" : {
+			"LIBPATH" : [ "$ARNOLD_ROOT/bin" ],
+			## \todo Remove GafferScene. We need it at present to get access to `IECoreScenePreview::Renderer`,
+			# but IECoreArnold must never depend on Gaffer code; logically it is in the layer below Gaffer.
+			"LIBS" : [ "GafferScene", "ai", "IECoreScene$CORTEX_LIB_SUFFIX", "IECoreVDB$CORTEX_LIB_SUFFIX", "openvdb$VDB_LIB_SUFFIX" ],
+			"CXXFLAGS" : [ "-DAI_ENABLE_DEPRECATION_WARNINGS" ],
+			"CPPPATH" : [ "$ARNOLD_ROOT/include" ],
+		},
+		"pythonEnvAppends" : {
+			"LIBPATH" : [ "$ARNOLD_ROOT/bin" ],
+			"LIBS" : [ "IECoreScene$CORTEX_LIB_SUFFIX", "IECoreArnold" ],
+			"CXXFLAGS" : [ "-DAI_ENABLE_DEPRECATION_WARNINGS" ],
+			"CPPPATH" : [ "$ARNOLD_ROOT/include" ],
+		},
+		"requiredOptions" : [ "ARNOLD_ROOT" ],
+		"installRoot" : arnoldInstallRoot,
+	},
+
+	"IECoreArnoldTest" : {
+		"additionalFiles" : [ "python/IECoreArnoldTest/metadata", "python/IECoreArnoldTest/assFiles" ],
+		"requiredOptions" : [ "ARNOLD_ROOT" ],
+		"installRoot" : arnoldInstallRoot,
+	},
+
 	"GafferArnold" : {
 		"envAppends" : {
 			"LIBPATH" : [ "$ARNOLD_ROOT/bin" ],
-			"LIBS" : [ "Gaffer", "GafferScene", "GafferDispatch", "ai", "GafferVDB", "openvdb$VDB_LIB_SUFFIX",  "IECoreScene$CORTEX_LIB_SUFFIX", "IECoreArnold$CORTEX_LIB_SUFFIX", "IECoreVDB$CORTEX_LIB_SUFFIX", "GafferOSL" ],
-			"CXXFLAGS" : [ "-isystem", "$ARNOLD_ROOT/include", "-DAI_ENABLE_DEPRECATION_WARNINGS" ],
+			"LIBS" : [ "Gaffer", "GafferScene", "GafferDispatch", "ai", "GafferVDB", "openvdb$VDB_LIB_SUFFIX",  "IECoreScene$CORTEX_LIB_SUFFIX", "IECoreVDB$CORTEX_LIB_SUFFIX", "IECoreArnold", "GafferOSL" ],
+			"CXXFLAGS" : [ "-DAI_ENABLE_DEPRECATION_WARNINGS" ],
+			"CPPPATH" : [ "$ARNOLD_ROOT/include" ],
 		},
 		"pythonEnvAppends" : {
 			"LIBPATH" : [ "$ARNOLD_ROOT/bin" ],
-			"LIBS" : [ "Gaffer", "GafferScene", "GafferBindings", "GafferVDB", "GafferDispatch", "GafferArnold", "GafferOSL", "IECoreScene$CORTEX_LIB_SUFFIX" ],
-			"CXXFLAGS" : [ "-isystem", "$ARNOLD_ROOT/include", "-DAI_ENABLE_DEPRECATION_WARNINGS" ],
+			"LIBS" : [ "Gaffer", "GafferScene", "GafferBindings", "GafferVDB", "GafferDispatch", "GafferArnold", "GafferOSL", "IECoreScene$CORTEX_LIB_SUFFIX", "IECoreArnold" ],
+			"CXXFLAGS" : [ "-DAI_ENABLE_DEPRECATION_WARNINGS" ],
+			"CPPPATH" : [ "$ARNOLD_ROOT/include" ],
 		},
 		"requiredOptions" : [ "ARNOLD_ROOT" ],
-		"additionalFiles" : [ "arnold/plugins/gaffer.mtd" ],
+		"additionalFiles" : [ "arnoldPlugins/gaffer.mtd" ],
+		"installRoot" : arnoldInstallRoot,
 	},
 
 	"GafferArnoldTest" : {
 		"additionalFiles" : glob.glob( "python/GafferArnoldTest/volumes/*" ) + glob.glob( "python/GafferArnoldTest/metadata/*" ) + glob.glob( "python/GafferArnoldTest/images/*" ),
 		"requiredOptions" : [ "ARNOLD_ROOT" ],
+		"installRoot" : arnoldInstallRoot,
 	},
 
 	"GafferArnoldUI" : {
 		"envAppends" : {
 			"LIBPATH" : [ "$ARNOLD_ROOT/bin" ],
 			"LIBS" : [ "IECoreScene$CORTEX_LIB_SUFFIX", "IECoreGL$CORTEX_LIB_SUFFIX", "OpenImageIO$OIIO_LIB_SUFFIX", "oslquery$OSL_LIB_SUFFIX", "Gaffer", "GafferScene", "GafferOSL", "GafferSceneUI", "ai" ],
-			"CXXFLAGS" : [ "-isystem", "$ARNOLD_ROOT/include", "-DAI_ENABLE_DEPRECATION_WARNINGS" ],
+			"CXXFLAGS" : [ "-DAI_ENABLE_DEPRECATION_WARNINGS" ],
+			"CPPPATH" : [ "$ARNOLD_ROOT/include" ],
 		},
 		"pythonEnvAppends" : {
 			"LIBS" : [ "GafferArnoldUI", "GafferSceneUI", "IECoreScene$CORTEX_LIB_SUFFIX" ],
 		},
 		"requiredOptions" : [ "ARNOLD_ROOT" ],
+		"installRoot" : arnoldInstallRoot,
 	},
 
 	"GafferArnoldUITest" : {
 		"additionalFiles" : glob.glob( "python/GafferArnoldUITest/metadata/*" ),
 		"requiredOptions" : [ "ARNOLD_ROOT" ],
+		"installRoot" : arnoldInstallRoot,
+	},
+
+	"GafferArnoldPlugin" : {
+		"envAppends" : {
+			"LIBPATH" : [ "$ARNOLD_ROOT/bin" ],
+			"LIBS" : [ "IECoreArnold", "ai", "IECoreImage$CORTEX_LIB_SUFFIX" ],
+			"CXXFLAGS" : [ "-DAI_ENABLE_DEPRECATION_WARNINGS" ],
+			"CPPPATH" : [ "$ARNOLD_ROOT/include" ],
+		},
+		"envReplacements" : {
+			"SHLIBPREFIX" : "",
+		},
+		"installName" : "arnoldPlugins/Gaffer",
+		"requiredOptions" : [ "ARNOLD_ROOT" ],
+		"installRoot" : arnoldInstallRoot,
 	},
 
 	"GafferOSL" : {
@@ -1070,16 +1140,23 @@ for libraryName, libraryDef in libraries.items() :
 	libEnv = baseLibEnv.Clone()
 	libEnv.Append( CXXFLAGS = "-D{0}_EXPORTS".format( libraryName ) )
 	libEnv.Append( **(libraryDef.get( "envAppends", {} )) )
+	libEnv.Replace( **(libraryDef.get( "envReplacements", {} )) )
+
+	installRoot = libraryDef.get( "installRoot", "$BUILD_DIR" )
 
 	# library
 
 	librarySource = sorted( glob.glob( "src/" + libraryName + "/*.cpp" ) + glob.glob( "src/" + libraryName + "/*/*.cpp" ) )
 	if librarySource :
 
-		library = libEnv.SharedLibrary( "lib/" + libraryName, librarySource )
+		libraryInstallName = libraryDef.get( "installName", "lib/" + libraryName )
+		library = libEnv.SharedLibrary( libraryInstallName, librarySource )
 		libEnv.Default( library )
 
-		libraryInstall = libEnv.Install( "$BUILD_DIR/lib", library )
+		libraryInstall = libEnv.Install(
+			os.path.join( installRoot, os.path.dirname( libraryInstallName ) ),
+			library
+		)
 		libEnv.Alias( "build", libraryInstall )
 
 	# header install
@@ -1097,7 +1174,7 @@ for libraryName, libraryDef in libraries.items() :
 	)
 
 	for header in headers :
-		headerInstall = env.Command( "$BUILD_DIR/" + header, header, "sed \"" + sedSubstitutions + "\" $SOURCE > $TARGET" )
+		headerInstall = env.Command( os.path.join( installRoot, header ), header, "sed \"" + sedSubstitutions + "\" $SOURCE > $TARGET" )
 		libEnv.Alias( "build", headerInstall )
 
 	# bindings library
@@ -1114,7 +1191,7 @@ for libraryName, libraryDef in libraries.items() :
 		bindingsLibrary = bindingsEnv.SharedLibrary( "lib/" + libraryName + "Bindings", bindingsSource )
 		bindingsEnv.Default( bindingsLibrary )
 
-		bindingsLibraryInstall = bindingsEnv.Install( "$BUILD_DIR/lib", bindingsLibrary )
+		bindingsLibraryInstall = bindingsEnv.Install( os.path.join( installRoot, "lib" ), bindingsLibrary )
 		env.Alias( "build", bindingsLibraryInstall )
 
 	# bindings header install
@@ -1125,7 +1202,7 @@ for libraryName, libraryDef in libraries.items() :
 	)
 
 	for header in bindingsHeaders :
-		headerInstall = env.Command( "$BUILD_DIR/" + header, header, "sed \"" + sedSubstitutions + "\" $SOURCE > $TARGET" )
+		headerInstall = env.Command( os.path.join( installRoot, header ), header, "sed \"" + sedSubstitutions + "\" $SOURCE > $TARGET" )
 		bindingsEnv.Alias( "build", headerInstall )
 
 	# python module binary component
@@ -1154,7 +1231,7 @@ for libraryName, libraryDef in libraries.items() :
 		pythonModule = pythonModuleEnv.SharedLibrary( "python/" + libraryName + "/_" + libraryName, pythonModuleSource )
 		pythonModuleEnv.Default( pythonModule )
 
-		moduleInstall = pythonModuleEnv.Install( "$BUILD_DIR/python/" + libraryName, pythonModule )
+		moduleInstall = pythonModuleEnv.Install( os.path.join( installRoot, "python", libraryName ), pythonModule )
 		pythonModuleEnv.Alias( "build", moduleInstall )
 
 	# Moc preprocessing, for QObject derived classes. SCons does include a "qt" tool that
@@ -1170,36 +1247,36 @@ for libraryName, libraryDef in libraries.items() :
 
 	# python component of python module
 
-	pythonFiles = glob.glob( "python/" + libraryName + "/*.py" ) + glob.glob( "python/" + libraryName + "/*/*.py" )
+	pythonFiles = glob.glob( "python/" + libraryName + "/*.py" ) + glob.glob( "python/" + libraryName + "/*/*.py" ) + glob.glob( "python/" + libraryName + "/*/*/*.py" )
 	for pythonFile in pythonFiles :
-		pythonFileInstall = env.Command( "$BUILD_DIR/" + pythonFile, pythonFile, "sed \"" + sedSubstitutions + "\" $SOURCE > $TARGET" )
+		pythonFileInstall = env.Command( os.path.join( installRoot, pythonFile ), pythonFile, "sed \"" + sedSubstitutions + "\" $SOURCE > $TARGET" )
 		env.Alias( "build", pythonFileInstall )
 
 	# apps
 
 	for app in libraryDef.get( "apps", [] ) :
-		appInstall = env.InstallAs("$BUILD_DIR/apps/{app}/{app}-1.py".format( app=app ), "apps/{app}/{app}-1.py".format( app=app ) )
+		appInstall = env.InstallAs( os.path.join( installRoot, "apps", app, "{app}-1.py".format( app=app ) ), "apps/{app}/{app}-1.py".format( app=app ) )
 		env.Alias( "build", appInstall )
 
 	# startup files
 
 	for startupDir in libraryDef.get( "apps", [] ) + [ libraryName ] :
 		for startupFile in glob.glob( "startup/{startupDir}/*.py".format( startupDir=startupDir ) ) :
-			startupFileInstall = env.InstallAs( "$BUILD_DIR/" + startupFile, startupFile )
+			startupFileInstall = env.InstallAs( os.path.join( installRoot, startupFile ), startupFile )
 			env.Alias( "build", startupFileInstall )
 
 	# additional files
 
 	for additionalFile in libraryDef.get( "additionalFiles", [] ) :
 		if additionalFile in pythonFiles :
 			continue
-		additionalFileInstall = env.InstallAs( "$BUILD_DIR/" + additionalFile, additionalFile )
+		additionalFileInstall = env.InstallAs( os.path.join( installRoot, additionalFile ), additionalFile )
 		env.Alias( "build", additionalFileInstall )
 
 	# osl headers
 
 	for oslHeader in libraryDef.get( "oslHeaders", [] ) :
-		oslHeaderInstall = env.InstallAs( "$BUILD_DIR/" + oslHeader, oslHeader )
+		oslHeaderInstall = env.InstallAs( os.path.join( installRoot, oslHeader ), oslHeader )
 		env.Alias( "oslHeaders", oslHeaderInstall )
 		env.Alias( "build", oslHeaderInstall )
 
@@ -1211,7 +1288,7 @@ for libraryName, libraryDef in libraries.items() :
 
 	for oslShader in libraryDef.get( "oslShaders", [] ) :
 		env.Alias( "build", oslShader )
-		compiledFile = commandEnv.Command( "$BUILD_DIR/" + os.path.splitext( oslShader )[0] + ".oso", oslShader, buildOSL )
+		compiledFile = commandEnv.Command( os.path.join( installRoot, os.path.splitext( oslShader )[0] + ".oso" ), oslShader, buildOSL )
 		env.Depends( compiledFile, "oslHeaders" )
 		env.Alias( "build", compiledFile )
 
@@ -1230,7 +1307,7 @@ for libraryName, libraryDef in libraries.items() :
 		f.write( env.subst( "from $GAFFER_STUB_MODULE import $GAFFER_STUB_CLASS as %s" % classLoadableName ) )
 
 	for classStub in libraryDef.get( "classStubs", [] ) :
-		stubFileName = "$BUILD_DIR/" + classStub[1] + "/" + classStub[1].rpartition( "/" )[2] + "-1.py"
+		stubFileName = os.path.join( installRoot, classStub[1], classStub[1].rpartition( "/" )[2] + "-1.py" )
 		stubEnv = env.Clone(
 			GAFFER_STUB_MODULE = libraryName,
 			GAFFER_STUB_CLASS = classStub[0],

apps/test/test-1.py

@@ -166,7 +166,8 @@ def __allTestModules() :
 
 		result = set()
 		for path in sys.path :
-			for m in glob.glob( os.path.join( path, "Gaffer*Test" ) ) :
+			modules = glob.glob( os.path.join( path, "Gaffer*Test" ) ) + glob.glob( os.path.join( path, "IECore*Test" ) )
+			for m in modules :
 				result.add( os.path.basename( m ) )
 
 		return sorted( result )

bin/gaffer

@@ -233,10 +233,10 @@ fi
 # Set up Arnold
 ##########################################################################
 
-prependToPath "$GAFFER_ROOT/arnold/plugins" ARNOLD_PLUGIN_PATH
-
 if [[ $ARNOLD_ROOT ]] ; then
 
+	# Put Arnold's own libs and binaries on the appropriate paths.
+
 	if [[ `uname` = "Linux" ]] ; then
 		appendToPath "$ARNOLD_ROOT/bin" LD_LIBRARY_PATH
 	else
@@ -246,6 +246,25 @@ if [[ $ARNOLD_ROOT ]] ; then
 	appendToPath "$ARNOLD_ROOT/bin" PATH
 	appendToPath "$ARNOLD_ROOT/python" PYTHONPATH
 
+	# Figure out the Arnold version, and set up the matching Gaffer extension.
+
+	if [[ -f "$ARNOLD_ROOT/include/ai_version.h" ]] ; then
+
+		arnoldVersion=`awk \
+			'$1=="#define" && $2=="AI_VERSION_ARCH_NUM"{arch = $3}; \$1=="#define" && $2=="AI_VERSION_MAJOR_NUM"{print arch "." $3}' \
+			"$ARNOLD_ROOT/include/ai_version.h"`
+
+		if [[ -d "$GAFFER_ROOT/arnold/$arnoldVersion" ]] ; then
+			prependToPath "$GAFFER_ROOT/arnold/$arnoldVersion" GAFFER_EXTENSION_PATHS
+			prependToPath "$ARNOLD_ROOT/plugins:$GAFFER_ROOT/arnold/$arnoldVersion/arnoldPlugins" ARNOLD_PLUGIN_PATH
+		else
+			echo "WARNING : GafferArnold extension not available for Arnold $arnoldVersion" >&2
+		fi
+
+	else
+		echo "WARNING : Unable to determine Arnold version" >&2
+	fi
+
 fi
 
 # Set up 3Delight