fix installation of Hadoop v2.10.0 on RHEL8

easybuild/easyconfigs/h/Hadoop/HADOOP-14597.04.patch

@@ -0,0 +1,140 @@
+# Patch for newer OpenSSL (1.1.x)
+# see: https://issues.apache.org/jira/browse/HADOOP-14597
+# download: https://issues.apache.org/jira/secure/attachment/12875105/HADOOP-14597.04.patch
+diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
+index 5cb5bba9ae..c7984a3347 100644
+--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
+@@ -30,6 +30,11 @@ static void (*dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *);
+ static int (*dlsym_EVP_CIPHER_CTX_cleanup)(EVP_CIPHER_CTX *);
+ static void (*dlsym_EVP_CIPHER_CTX_init)(EVP_CIPHER_CTX *);
+ static int (*dlsym_EVP_CIPHER_CTX_set_padding)(EVP_CIPHER_CTX *, int);
++static int (*dlsym_EVP_CIPHER_CTX_test_flags)(const EVP_CIPHER_CTX *, int);
++static int (*dlsym_EVP_CIPHER_CTX_block_size)(const EVP_CIPHER_CTX *);
++#if OPENSSL_VERSION_NUMBER >= 0x10100000L
++static int (*dlsym_EVP_CIPHER_CTX_encrypting)(const EVP_CIPHER_CTX *);
++#endif
+ static int (*dlsym_EVP_CipherInit_ex)(EVP_CIPHER_CTX *, const EVP_CIPHER *,  \
+            ENGINE *, const unsigned char *, const unsigned char *, int);
+ static int (*dlsym_EVP_CipherUpdate)(EVP_CIPHER_CTX *, unsigned char *,  \
+@@ -46,6 +51,11 @@ typedef void (__cdecl *__dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *);
+ typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_cleanup)(EVP_CIPHER_CTX *);
+ typedef void (__cdecl *__dlsym_EVP_CIPHER_CTX_init)(EVP_CIPHER_CTX *);
+ typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_set_padding)(EVP_CIPHER_CTX *, int);
++typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_test_flags)(const EVP_CIPHER_CTX *, int);
++typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_block_size)(const EVP_CIPHER_CTX *);
++#if OPENSSL_VERSION_NUMBER >= 0x10100000L
++typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_encrypting)(const EVP_CIPHER_CTX *);
++#endif
+ typedef int (__cdecl *__dlsym_EVP_CipherInit_ex)(EVP_CIPHER_CTX *,  \
+              const EVP_CIPHER *, ENGINE *, const unsigned char *,  \
+              const unsigned char *, int);
+@@ -60,6 +70,11 @@ static __dlsym_EVP_CIPHER_CTX_free dlsym_EVP_CIPHER_CTX_free;
+ static __dlsym_EVP_CIPHER_CTX_cleanup dlsym_EVP_CIPHER_CTX_cleanup;
+ static __dlsym_EVP_CIPHER_CTX_init dlsym_EVP_CIPHER_CTX_init;
+ static __dlsym_EVP_CIPHER_CTX_set_padding dlsym_EVP_CIPHER_CTX_set_padding;
++static __dlsym_EVP_CIPHER_CTX_test_flags dlsym_EVP_CIPHER_CTX_test_flags;
++static __dlsym_EVP_CIPHER_CTX_block_size dlsym_EVP_CIPHER_CTX_block_size;
++#if OPENSSL_VERSION_NUMBER >= 0x10100000L
++static __dlsym_EVP_CIPHER_CTX_encrypting dlsym_EVP_CIPHER_CTX_encrypting;
++#endif
+ static __dlsym_EVP_CipherInit_ex dlsym_EVP_CipherInit_ex;
+ static __dlsym_EVP_CipherUpdate dlsym_EVP_CipherUpdate;
+ static __dlsym_EVP_CipherFinal_ex dlsym_EVP_CipherFinal_ex;
+@@ -114,6 +129,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs
+                       "EVP_CIPHER_CTX_init");
+   LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_set_padding, env, openssl,  \
+                       "EVP_CIPHER_CTX_set_padding");
++  LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_test_flags, env, openssl,  \
++                      "EVP_CIPHER_CTX_test_flags");
++  LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_block_size, env, openssl,  \
++                      "EVP_CIPHER_CTX_block_size");
++#if OPENSSL_VERSION_NUMBER >= 0x10100000L
++  LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_encrypting, env, openssl,  \
++                      "EVP_CIPHER_CTX_encrypting");
++#endif
+   LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherInit_ex, env, openssl,  \
+                       "EVP_CipherInit_ex");
+   LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherUpdate, env, openssl,  \
+@@ -135,6 +158,17 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs
+   LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_set_padding,  \
+                       dlsym_EVP_CIPHER_CTX_set_padding, env,  \
+                       openssl, "EVP_CIPHER_CTX_set_padding");
++  LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_test_flags,  \
++                      dlsym_EVP_CIPHER_CTX_test_flags, env,  \
++                      openssl, "EVP_CIPHER_CTX_test_flags");
++  LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_block_size,  \
++                      dlsym_EVP_CIPHER_CTX_block_size, env,  \
++                      openssl, "EVP_CIPHER_CTX_block_size");
++#if OPENSSL_VERSION_NUMBER >= 0x10100000L
++  LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_encrypting,  \
++                      dlsym_EVP_CIPHER_CTX_encrypting, env,  \
++                      openssl, "EVP_CIPHER_CTX_encrypting");
++#endif
+   LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherInit_ex, dlsym_EVP_CipherInit_ex,  \
+                       env, openssl, "EVP_CipherInit_ex");
+   LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherUpdate, dlsym_EVP_CipherUpdate,  \
+@@ -253,14 +287,18 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_init
+ static int check_update_max_output_len(EVP_CIPHER_CTX *context, int input_len, 
+     int max_output_len)
+ {
+-  if (context->flags & EVP_CIPH_NO_PADDING) {
++  if (  dlsym_EVP_CIPHER_CTX_test_flags(context, EVP_CIPH_NO_PADDING) ) {
+     if (max_output_len >= input_len) {
+       return 1;
+     }
+     return 0;
+   } else {
+-    int b = context->cipher->block_size;
++    int b = dlsym_EVP_CIPHER_CTX_block_size(context);
++#if OPENSSL_VERSION_NUMBER < 0x10100000L
+     if (context->encrypt) {
++#else
++    if (dlsym_EVP_CIPHER_CTX_encrypting(context)) {
++#endif
+       if (max_output_len >= input_len + b - 1) {
+         return 1;
+       }
+@@ -307,10 +345,10 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_update
+ static int check_doFinal_max_output_len(EVP_CIPHER_CTX *context, 
+     int max_output_len)
+ {
+-  if (context->flags & EVP_CIPH_NO_PADDING) {
++  if (  dlsym_EVP_CIPHER_CTX_test_flags(context, EVP_CIPH_NO_PADDING) ) {
+     return 1;
+   } else {
+-    int b = context->cipher->block_size;
++    int b = dlsym_EVP_CIPHER_CTX_block_size(context);
+     if (max_output_len >= b) {
+       return 1;
+     }
+diff --git a/hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc b/hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc
+index 91fb5a42b2..45cb8c2024 100644
+--- a/hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc
++++ b/hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc
+@@ -420,6 +420,7 @@ namespace HadoopPipes {
+     }
+
+     string createDigest(string &password, string& msg) {
++#if OPENSSL_VERSION_NUMBER < 0x10100000L
+       HMAC_CTX ctx;
+       unsigned char digest[EVP_MAX_MD_SIZE];
+       HMAC_Init(&ctx, (const unsigned char *)password.c_str(), 
+@@ -428,7 +429,16 @@ namespace HadoopPipes {
+       unsigned int digestLen;
+       HMAC_Final(&ctx, digest, &digestLen);
+       HMAC_cleanup(&ctx);
+-
++#else
++      HMAC_CTX *ctx = HMAC_CTX_new();
++      unsigned char digest[EVP_MAX_MD_SIZE];
++      HMAC_Init_ex(ctx, (const unsigned char *)password.c_str(),
++          password.length(), EVP_sha1(), NULL);
++      HMAC_Update(ctx, (const unsigned char *)msg.c_str(), msg.length());
++      unsigned int digestLen;
++      HMAC_Final(ctx, digest, &digestLen);
++      HMAC_CTX_free(ctx);
++#endif
+       //now apply base64 encoding
+       BIO *bmem, *b64;
+       BUF_MEM *bptr;

easybuild/easyconfigs/h/Hadoop/Hadoop-2.10.0-GCCcore-8.3.0-native.eb

@@ -15,11 +15,15 @@ sources = ['hadoop-%(version)s-src.tar.gz']
 patches = [
     'Hadoop-TeraSort-on-local-filesystem.patch',
     'Hadoop-2.9.2_fix-zlib.patch',
+    'HADOOP-14597.04.patch',
+    'Hadoop-2.10.0_tirpc.patch',
 ]
 checksums = [
     'baa9b125359a30eb209fbaa953e1b324eb61fa65ceb9a7cf19b0967188d8b1c0',  # hadoop-2.10.0-src.tar.gz
     'd0a69a6936b4a01505ba2a20911d0cec4f79440dbc8da52b9ddbd7f3a205468b',  # Hadoop-TeraSort-on-local-filesystem.patch
     '1a1d084c7961078bdbaa84716e9639e37587e1d8c0b1f89ce6f12dde8bbbbc5c',  # Hadoop-2.9.2_fix-zlib.patch
+    'ea93c7c2b03d36f1434c2f2921c031cdc385a98f337ed8f4b3103b45b0ad0da3',  # HADOOP-14597.04.patch
+    '9d66f604e6e03923d8fcb290382936fb93511001bb593025b8d63ababdca3a96',  # Hadoop-2.10.0_tirpc.patch
 ]
 
 builddependencies = [
@@ -33,7 +37,10 @@ builddependencies = [
 ]
 
 # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions
-dependencies = [('Java', '1.8', '', True)]
+dependencies = [
+    ('Java', '1.8', '', True),
+    ('libtirpc', '1.2.6'),
+]
 
 build_native_libs = True
 

easybuild/easyconfigs/h/Hadoop/Hadoop-2.10.0_tirpc.patch

@@ -0,0 +1,15 @@
+# RHEL 8 glibc does not include RPC anymore, so external one has to be used. 
+# And SerialUtils.cc has to be linked with the external library, libtirpc
+# SEP 24th 2020 by B. Hajgato (UGent)  
+diff -ru hadoop-2.10.0-src.orig/hadoop-tools/hadoop-pipes/src/CMakeLists.txt hadoop-2.10.0-src/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
+--- hadoop-2.10.0-src.orig/hadoop-tools/hadoop-pipes/src/CMakeLists.txt	2019-10-15 20:12:38.000000000 +0200
++++ hadoop-2.10.0-src/hadoop-tools/hadoop-pipes/src/CMakeLists.txt	2020-09-24 12:02:21.022819284 +0200
+@@ -63,6 +63,8 @@
+     set(LIB_DL "dl")
+ endif()
+
++set(LIB_NET "tirpc")
++
+ if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS")
+     exec_program("uname" ARGS "-r" OUTPUT_VARIABLE OS_VERSION)
+     if(OS_VERSION VERSION_LESS "5.12")