From 59b2cc86a9204831794c24f12bf0c19ce1242882 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 24 Jun 2025 20:08:37 +0800 Subject: [PATCH 1/4] HADOOP-19594. Bump Maven 3.9.10 --- BUILDING.txt | 5 ++--- dev-support/docker/Dockerfile | 4 +++- dev-support/docker/Dockerfile_aarch64 | 4 +++- dev-support/docker/Dockerfile_debian_10 | 4 +++- dev-support/docker/Dockerfile_ubuntu_24 | 4 +++- dev-support/docker/Dockerfile_windows_10 | 6 +++--- dev-support/docker/pkg-resolver/install-maven.sh | 12 ++++++------ dev-support/docker/pkg-resolver/packages.json | 6 ------ .../hadoop-auth/src/site/markdown/BuildingIt.md | 4 ++-- pom.xml | 3 +-- 10 files changed, 26 insertions(+), 26 deletions(-) diff --git a/BUILDING.txt b/BUILDING.txt index 4dfb48fb089eb..28c936062eaf4 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -279,8 +279,7 @@ Maven build goals: OWASP's dependency-check plugin will scan the third party dependencies of this project for known CVEs (security vulnerabilities against them). It will produce a report in target/dependency-check-report.html. To - invoke, run 'mvn dependency-check:aggregate'. Note that this plugin - requires maven 3.1.1 or greater. + invoke, run 'mvn dependency-check:aggregate'. PMDK library build options: @@ -529,7 +528,7 @@ Requirements: * Windows 10 * JDK 1.8 -* Maven 3.0 or later (maven.apache.org) +* Maven 3.3 or later (maven.apache.org) * Boost 1.86.0 (boost.org) * Protocol Buffers 3.21.12 (https://github.com/protocolbuffers/protobuf/tags) * CMake 3.19 or newer (cmake.org) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 6beef860a9b39..14bc9fb3be070 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -61,7 +61,8 @@ ENV PYTHONIOENCODING=utf-8 ###### # Set env vars required to build Hadoop ###### -ENV MAVEN_HOME /usr +ENV MAVEN_HOME /opt/maven +ENV PATH "${PATH}:${MAVEN_HOME}/bin" # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 @@ -88,6 +89,7 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true # Install packages #### RUN pkg-resolver/install-common-pkgs.sh +RUN pkg-resolver/install-maven.sh ubuntu:focal RUN pkg-resolver/install-spotbugs.sh ubuntu:focal RUN pkg-resolver/install-boost.sh ubuntu:focal RUN pkg-resolver/install-protobuf.sh ubuntu:focal diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index b9249f5aec9d3..af0a6957d5311 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -62,7 +62,8 @@ ENV PYTHONIOENCODING=utf-8 ###### # Set env vars required to build Hadoop ###### -ENV MAVEN_HOME /usr +ENV MAVEN_HOME /opt/maven +ENV PATH "${PATH}:${MAVEN_HOME}/bin" # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64 @@ -92,6 +93,7 @@ ENV QT_QPA_PLATFORM offscreen # Install packages #### RUN pkg-resolver/install-common-pkgs.sh +RUN pkg-resolver/install-maven.sh ubuntu:focal::arch64 RUN pkg-resolver/install-spotbugs.sh ubuntu:focal::arch64 RUN pkg-resolver/install-boost.sh ubuntu:focal::arch64 RUN pkg-resolver/install-protobuf.sh ubuntu:focal::arch64 diff --git a/dev-support/docker/Dockerfile_debian_10 b/dev-support/docker/Dockerfile_debian_10 index 71446b27f686b..c6ea3b9e4f1c0 100644 --- a/dev-support/docker/Dockerfile_debian_10 +++ b/dev-support/docker/Dockerfile_debian_10 @@ -56,7 +56,8 @@ RUN apt-get -q update \ ###### # Set env vars required to build Hadoop ###### -ENV MAVEN_HOME /usr +ENV MAVEN_HOME /opt/maven +ENV PATH "${PATH}:${MAVEN_HOME}/bin" # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 @@ -82,6 +83,7 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true #### # Install packages #### +RUN pkg-resolver/install-maven.sh debian:10 RUN pkg-resolver/install-cmake.sh debian:10 RUN pkg-resolver/install-spotbugs.sh debian:10 RUN pkg-resolver/install-boost.sh debian:10 diff --git a/dev-support/docker/Dockerfile_ubuntu_24 b/dev-support/docker/Dockerfile_ubuntu_24 index 28ab04c3e09fe..f379b857d83c6 100644 --- a/dev-support/docker/Dockerfile_ubuntu_24 +++ b/dev-support/docker/Dockerfile_ubuntu_24 @@ -61,7 +61,8 @@ ENV PYTHONIOENCODING=utf-8 ###### # Set env vars required to build Hadoop ###### -ENV MAVEN_HOME /usr +ENV MAVEN_HOME /opt/maven +ENV PATH "${PATH}:${MAVEN_HOME}/bin" # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-amd64 @@ -87,6 +88,7 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true #### # Install packages #### +RUN pkg-resolver/install-maven.sh ubuntu:noble RUN pkg-resolver/install-spotbugs.sh ubuntu:noble RUN pkg-resolver/install-boost.sh ubuntu:noble RUN pkg-resolver/install-protobuf.sh ubuntu:noble diff --git a/dev-support/docker/Dockerfile_windows_10 b/dev-support/docker/Dockerfile_windows_10 index 70ff3e92c2964..eed86a13511dd 100644 --- a/dev-support/docker/Dockerfile_windows_10 +++ b/dev-support/docker/Dockerfile_windows_10 @@ -62,8 +62,8 @@ RUN powershell Invoke-WebRequest -URI https://cdn.azul.com/zulu/bin/zulu8.62.0.1 RUN powershell Expand-Archive -Path $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -DestinationPath "C:\Java" # Install Apache Maven. -RUN powershell Invoke-WebRequest -URI https://archive.apache.org/dist/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.8-bin.zip -RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.8.8-bin.zip -DestinationPath "C:\Maven" +RUN powershell Invoke-WebRequest -URI https://archive.apache.org/dist/maven/maven-3/3.9.10/binaries/apache-maven-3.9.10-bin.zip -OutFile $Env:TEMP\apache-maven-3.9.10-bin.zip +RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.9.10-bin.zip -DestinationPath "C:\Maven" # Install CMake 3.19.0. RUN powershell Invoke-WebRequest -URI https://cmake.org/files/v3.19/cmake-3.19.0-win64-x64.zip -OutFile $Env:TEMP\cmake-3.19.0-win64-x64.zip @@ -141,7 +141,7 @@ ENV MAVEN_OPTS '-Xmx2048M -Xss128M' ENV IS_WINDOWS 1 RUN setx PATH "%PATH%;%ALLUSERSPROFILE%\chocolatey\bin" RUN setx PATH "%PATH%;%JAVA_HOME%\bin" -RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.8.8\bin" +RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.9.10\bin" RUN setx PATH "%PATH%;C:\CMake\cmake-3.19.0-win64-x64\bin" RUN setx PATH "%PATH%;C:\ZStd" RUN setx PATH "%PATH%;C:\Program Files\Git\usr\bin" diff --git a/dev-support/docker/pkg-resolver/install-maven.sh b/dev-support/docker/pkg-resolver/install-maven.sh index fb7d4a5be77dc..7b06e5080ce64 100644 --- a/dev-support/docker/pkg-resolver/install-maven.sh +++ b/dev-support/docker/pkg-resolver/install-maven.sh @@ -27,22 +27,22 @@ if [ $? -eq 1 ]; then exit 1 fi -default_version="3.6.3" +default_version="3.9.10" version_to_install=$default_version if [ -n "$2" ]; then version_to_install="$2" fi -if [ "$version_to_install" != "3.6.3" ]; then +if [ "$version_to_install" != "3.9.10" ]; then echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" version_to_install=$default_version fi -if [ "$version_to_install" == "3.6.3" ]; then +if [ "$version_to_install" == "3.9.10" ]; then mkdir -p /opt/maven /tmp/maven && - curl -L -s -S https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.tar.gz \ - -o /tmp/maven/apache-maven-3.6.3-bin.tar.gz && - tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven + curl -L -s -S https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.10/apache-maven-3.9.10-bin.tar.gz \ + -o /tmp/maven/apache-maven-3.9.10-bin.tar.gz && + tar xzf /tmp/maven/apache-maven-3.9.10-bin.tar.gz --strip-components 1 -C /opt/maven else echo "ERROR: Don't know how to install version $version_to_install" exit 1 diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json index 4b37ea62b42fc..e1e835b344f29 100644 --- a/dev-support/docker/pkg-resolver/packages.json +++ b/dev-support/docker/pkg-resolver/packages.json @@ -295,12 +295,6 @@ "centos:7": "make", "centos:8": "make" }, - "maven": { - "debian:10": "maven", - "ubuntu:focal": "maven", - "ubuntu:noble": "maven", - "ubuntu:focal::arch64": "maven" - }, "java": { "debian:10": "openjdk-11-jdk", "ubuntu:focal": [ diff --git a/hadoop-common-project/hadoop-auth/src/site/markdown/BuildingIt.md b/hadoop-common-project/hadoop-auth/src/site/markdown/BuildingIt.md index 52d9ddc1558d2..85f7c9f230cf2 100644 --- a/hadoop-common-project/hadoop-auth/src/site/markdown/BuildingIt.md +++ b/hadoop-common-project/hadoop-auth/src/site/markdown/BuildingIt.md @@ -18,8 +18,8 @@ Hadoop Auth, Java HTTP SPNEGO - Building It Requirements ------------ -* Java 7+ -* Maven 3+ +* JDK 1.8 +* Maven 3.3 or later * Kerberos KDC (for running Kerberos test cases) Building diff --git a/pom.xml b/pom.xml index 0511ffd58a8e9..630876a812eb3 100644 --- a/pom.xml +++ b/pom.xml @@ -617,8 +617,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x dependencies of this project for known CVEs (security vulnerabilities against them). It will produce a report in target/dependency-check-report.html. To invoke, run - 'mvn dependency-check:aggregate'. Note that this plugin - requires maven 3.1.1 or greater. + 'mvn dependency-check:aggregate'. --> org.owasp dependency-check-maven From f3a6f24c694c0c6e462090193bc927ed22250924 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 7 Jul 2025 02:53:33 +0800 Subject: [PATCH 2/4] tackle dirty cache --- dev-support/docker/Dockerfile_centos_7 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/docker/Dockerfile_centos_7 b/dev-support/docker/Dockerfile_centos_7 index b97e59969a760..4816eb26a7251 100644 --- a/dev-support/docker/Dockerfile_centos_7 +++ b/dev-support/docker/Dockerfile_centos_7 @@ -17,7 +17,7 @@ # Dockerfile for installing the necessary dependencies for building Hadoop. # See BUILDING.txt. -FROM centos:7 +FROM centos:7.9 WORKDIR /root From 067b22e26be1c0c1eb2559b614e5245afa2dc956 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 7 Jul 2025 02:58:32 +0800 Subject: [PATCH 3/4] fix --- dev-support/docker/Dockerfile_centos_7 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/docker/Dockerfile_centos_7 b/dev-support/docker/Dockerfile_centos_7 index 4816eb26a7251..d36ac5a957ba0 100644 --- a/dev-support/docker/Dockerfile_centos_7 +++ b/dev-support/docker/Dockerfile_centos_7 @@ -17,7 +17,7 @@ # Dockerfile for installing the necessary dependencies for building Hadoop. # See BUILDING.txt. -FROM centos:7.9 +FROM centos:7.9.2009 WORKDIR /root From 2d2d88d2f84fca82ee46de1322153b9ad779e163 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 7 Jul 2025 03:15:02 +0800 Subject: [PATCH 4/4] changeb back --- dev-support/docker/Dockerfile_centos_7 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/docker/Dockerfile_centos_7 b/dev-support/docker/Dockerfile_centos_7 index d36ac5a957ba0..b97e59969a760 100644 --- a/dev-support/docker/Dockerfile_centos_7 +++ b/dev-support/docker/Dockerfile_centos_7 @@ -17,7 +17,7 @@ # Dockerfile for installing the necessary dependencies for building Hadoop. # See BUILDING.txt. -FROM centos:7.9.2009 +FROM centos:7 WORKDIR /root