Skip to content

Upgrade pip to latest released 22.1.0 version #23665

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 12, 2022
Merged

Upgrade pip to latest released 22.1.0 version #23665

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 20 additions & 20 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ ARG AIRFLOW_VERSION="2.3.0"

ARG PYTHON_BASE_IMAGE="python:3.7-slim-bullseye"

ARG AIRFLOW_PIP_VERSION=22.0.4
ARG AIRFLOW_PIP_VERSION=22.1.0
ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
ARG AIRFLOW_IMAGE_README_URL="https://raw.githubusercontent.com/apache/airflow/main/docs/docker-stack/README.md"

Expand Down Expand Up @@ -316,11 +316,11 @@ function install_airflow_dependencies_from_branch_tip() {
# Install latest set of dependencies using constraints. In case constraints were upgraded and there
# are conflicts, this might fail, but it should be fixed in the following installation steps
set -x
pip install \
pip install --root-user-action ignore \
"https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true
set +x
echo
Expand Down Expand Up @@ -367,7 +367,7 @@ function common::get_airflow_version_specification() {
function common::override_pip_version_if_needed() {
if [[ -n ${AIRFLOW_VERSION} ]]; then
if [[ ${AIRFLOW_VERSION} =~ ^2\.0.* || ${AIRFLOW_VERSION} =~ ^1\.* ]]; then
export AIRFLOW_PIP_VERSION="20.2.4"
export AIRFLOW_PIP_VERSION="22.1.0"
fi
fi
}
Expand Down Expand Up @@ -570,13 +570,13 @@ function install_airflow_and_providers_from_docker_context_files(){
echo
# force reinstall all airflow + provider package local files with eager upgrade
set -x
pip install "${pip_flags[@]}" --upgrade --upgrade-strategy eager \
pip install "${pip_flags[@]}" --root-user-action ignore --upgrade --upgrade-strategy eager \
${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} \
${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
set +x

# make sure correct PIP version is left installed
pip install "pip==${AIRFLOW_PIP_VERSION}"
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
pip check
}

Expand All @@ -591,9 +591,9 @@ function install_all_other_packages_from_docker_context_files() {
grep -v apache_airflow | grep -v apache-airflow || true)
if [[ -n "${reinstalling_other_packages}" ]]; then
set -x
pip install --force-reinstall --no-deps --no-index ${reinstalling_other_packages}
pip install --root-user-action ignore --force-reinstall --no-deps --no-index ${reinstalling_other_packages}
# make sure correct PIP version is used
pip install "pip==${AIRFLOW_PIP_VERSION}"
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
set -x
fi
}
Expand Down Expand Up @@ -641,21 +641,21 @@ function install_airflow() {
echo "${COLOR_BLUE}Installing all packages with eager upgrade${COLOR_RESET}"
echo
# eager upgrade
pip install --upgrade --upgrade-strategy eager \
pip install --root-user-action ignore --upgrade --upgrade-strategy eager \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
if [[ -n "${AIRFLOW_INSTALL_EDITABLE_FLAG}" ]]; then
# Remove airflow and reinstall it using editable flag
# We can only do it when we install airflow from sources
set -x
pip uninstall apache-airflow --yes
pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
pip install --root-user-action ignore ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
set +x
fi

# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
Expand All @@ -665,17 +665,17 @@ function install_airflow() {
echo "${COLOR_BLUE}Installing all packages with constraints and upgrade if needed${COLOR_RESET}"
echo
set -x
pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
pip install --root-user-action ignore ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
# then upgrade if needed without using constraints to account for new limits in setup.py
pip install --upgrade --upgrade-strategy only-if-needed \
pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
set +x
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand Down Expand Up @@ -711,10 +711,10 @@ function install_additional_dependencies() {
echo "${COLOR_BLUE}Installing additional dependencies while upgrading to newer dependencies${COLOR_RESET}"
echo
set -x
pip install --upgrade --upgrade-strategy eager \
pip install --root-user-action ignore --upgrade --upgrade-strategy eager \
${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
set +x
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand All @@ -725,10 +725,10 @@ function install_additional_dependencies() {
echo "${COLOR_BLUE}Installing additional dependencies upgrading only if needed${COLOR_RESET}"
echo
set -x
pip install --upgrade --upgrade-strategy only-if-needed \
pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
${ADDITIONAL_PYTHON_DEPS}
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
set +x
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand Down Expand Up @@ -1023,7 +1023,7 @@ if [[ -n "${_PIP_ADDITIONAL_REQUIREMENTS=}" ]] ; then
>&2 echo " the container starts, so it is onlny useful for testing and trying out"
>&2 echo " of adding dependencies."
>&2 echo
pip install --no-cache-dir ${_PIP_ADDITIONAL_REQUIREMENTS}
pip install --root-user-action ignore --no-cache-dir ${_PIP_ADDITIONAL_REQUIREMENTS}
fi


Expand Down
38 changes: 18 additions & 20 deletions Dockerfile.ci
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,11 +276,11 @@ function install_airflow_dependencies_from_branch_tip() {
# Install latest set of dependencies using constraints. In case constraints were upgraded and there
# are conflicts, this might fail, but it should be fixed in the following installation steps
set -x
pip install \
pip install --root-user-action ignore \
"https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true
set +x
echo
Expand Down Expand Up @@ -327,7 +327,7 @@ function common::get_airflow_version_specification() {
function common::override_pip_version_if_needed() {
if [[ -n ${AIRFLOW_VERSION} ]]; then
if [[ ${AIRFLOW_VERSION} =~ ^2\.0.* || ${AIRFLOW_VERSION} =~ ^1\.* ]]; then
export AIRFLOW_PIP_VERSION="20.2.4"
export AIRFLOW_PIP_VERSION="22.1.0"
fi
fi
}
Expand Down Expand Up @@ -366,7 +366,7 @@ function install_pipx_tools() {
echo "${COLOR_BLUE}Installing pipx tools${COLOR_RESET}"
echo
# Make sure PIPX is installed in latest version
pip install --upgrade pipx
pip install --root-user-action ignore --upgrade pipx
if [[ $(uname -m) != "aarch64" ]]; then
# Do not install mssql-cli for ARM
# Install all the tools we need available in command line but without impacting the current environment
Expand Down Expand Up @@ -510,21 +510,21 @@ function install_airflow() {
echo "${COLOR_BLUE}Installing all packages with eager upgrade${COLOR_RESET}"
echo
# eager upgrade
pip install --upgrade --upgrade-strategy eager \
pip install --root-user-action ignore --upgrade --upgrade-strategy eager \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
if [[ -n "${AIRFLOW_INSTALL_EDITABLE_FLAG}" ]]; then
# Remove airflow and reinstall it using editable flag
# We can only do it when we install airflow from sources
set -x
pip uninstall apache-airflow --yes
pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
pip install --root-user-action ignore ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
set +x
fi

# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
Expand All @@ -534,17 +534,17 @@ function install_airflow() {
echo "${COLOR_BLUE}Installing all packages with constraints and upgrade if needed${COLOR_RESET}"
echo
set -x
pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
pip install --root-user-action ignore ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
# then upgrade if needed without using constraints to account for new limits in setup.py
pip install --upgrade --upgrade-strategy only-if-needed \
pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
set +x
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand Down Expand Up @@ -580,10 +580,10 @@ function install_additional_dependencies() {
echo "${COLOR_BLUE}Installing additional dependencies while upgrading to newer dependencies${COLOR_RESET}"
echo
set -x
pip install --upgrade --upgrade-strategy eager \
pip install --root-user-action ignore --upgrade --upgrade-strategy eager \
${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
set +x
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand All @@ -594,10 +594,10 @@ function install_additional_dependencies() {
echo "${COLOR_BLUE}Installing additional dependencies upgrading only if needed${COLOR_RESET}"
echo
set -x
pip install --upgrade --upgrade-strategy only-if-needed \
pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
${ADDITIONAL_PYTHON_DEPS}
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
set +x
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand Down Expand Up @@ -741,7 +741,7 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
fi
done
if (( ${#installable_files[@]} )); then
pip install "${installable_files[@]}"
pip install --root-user-action ignore "${installable_files[@]}"
fi
fi

Expand Down Expand Up @@ -1195,7 +1195,7 @@ ARG AIRFLOW_CI_BUILD_EPOCH="3"
ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true"
# By default in the image, we are installing all providers when installing from sources
ARG INSTALL_PROVIDERS_FROM_SOURCES="true"
ARG AIRFLOW_PIP_VERSION=22.0.4
ARG AIRFLOW_PIP_VERSION=22.1.0
# Setup PIP
# By default PIP install run without cache to make image smaller
ARG PIP_NO_CACHE_DIR="true"
Expand Down Expand Up @@ -1265,9 +1265,7 @@ COPY --from=scripts install_pip_version.sh install_airflow_dependencies_from_bra
# are uninstalled, only dependencies remain.
# the cache is only used when "upgrade to newer dependencies" is not set to automatically
# account for removed dependencies (we do not install them in the first place)
RUN echo -e "\n\e[32mThe 'Running pip as the root user' warnings below are not valid but we can't disable them :(\e[0m\n"; \
echo -e "\n\e[34mSee https://github.com/pypa/pip/issues/10556 for details.\e[0m\n" ; \
bash /scripts/docker/install_pip_version.sh; \
RUN bash /scripts/docker/install_pip_version.sh; \
if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \
${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \
bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \
Expand Down
2 changes: 1 addition & 1 deletion IMAGES.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -475,7 +475,7 @@ The following build arguments (``--build-arg`` in docker build command) can be u
| ``ADDITIONAL_RUNTIME_APT_ENV`` | | Additional env variables defined |
| | | when installing runtime deps |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``AIRFLOW_PIP_VERSION`` | ``22.0.4`` | PIP version used. |
| ``AIRFLOW_PIP_VERSION`` | ``22.1.0`` | PIP version used. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``PIP_PROGRESS_BAR`` | ``on`` | Progress bar for PIP installation |
+------------------------------------------+------------------------------------------+------------------------------------------+
Expand Down
2 changes: 1 addition & 1 deletion dev/TRACKING_BACKTRACKING_ISSUES.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ image build jobs in CI.
An example of such issue is described [here](https://github.com/pypa/pip/issues/10924).

Unfortunately the problem is that in such cases, it is not possible to figure out what caused the
problem from `pip` output (state as of `pip` 22.0.4).
problem from `pip` output (state as of `pip` 22.1.0).

There are a number of issues in `pip` that describe the issue, and some backtracking reasons have been already
tracked down and fixed by `pip` maintainers, but this is a difficult problem to solve and it is likely it
Expand Down
2 changes: 1 addition & 1 deletion docs/docker-stack/build-arg-ref.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ Those are the most common arguments that you use when you want to build a custom
+------------------------------------------+------------------------------------------+---------------------------------------------+
| ``AIRFLOW_USER_HOME_DIR`` | ``/home/airflow`` | Home directory of the Airflow user. |
+------------------------------------------+------------------------------------------+---------------------------------------------+
| ``AIRFLOW_PIP_VERSION`` | ``22.0.4`` | PIP version used. |
| ``AIRFLOW_PIP_VERSION`` | ``22.1.0`` | PIP version used. |
+------------------------------------------+------------------------------------------+---------------------------------------------+
| ``PIP_PROGRESS_BAR`` | ``on`` | Progress bar for PIP installation |
+------------------------------------------+------------------------------------------+---------------------------------------------+
Expand Down
2 changes: 1 addition & 1 deletion scripts/ci/libraries/_initialization.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -418,7 +418,7 @@ function initialization::initialize_image_build_variables() {

export INSTALLED_EXTRAS="async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,imap,ldap,google,microsoft.azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv"

AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="22.0.4"}
AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="22.1.0"}
export AIRFLOW_PIP_VERSION

# We also pin version of wheel used to get consistent builds
Expand Down
2 changes: 1 addition & 1 deletion scripts/docker/common.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ function common::get_airflow_version_specification() {
function common::override_pip_version_if_needed() {
if [[ -n ${AIRFLOW_VERSION} ]]; then
if [[ ${AIRFLOW_VERSION} =~ ^2\.0.* || ${AIRFLOW_VERSION} =~ ^1\.* ]]; then
export AIRFLOW_PIP_VERSION="20.2.4"
export AIRFLOW_PIP_VERSION="22.1.0"
fi
fi
}
Expand Down
2 changes: 1 addition & 1 deletion scripts/docker/entrypoint_ci.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
fi
done
if (( ${#installable_files[@]} )); then
pip install "${installable_files[@]}"
pip install --root-user-action ignore "${installable_files[@]}"
fi
fi

Expand Down
2 changes: 1 addition & 1 deletion scripts/docker/entrypoint_prod.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,7 @@ if [[ -n "${_PIP_ADDITIONAL_REQUIREMENTS=}" ]] ; then
>&2 echo " the container starts, so it is onlny useful for testing and trying out"
>&2 echo " of adding dependencies."
>&2 echo
pip install --no-cache-dir ${_PIP_ADDITIONAL_REQUIREMENTS}
pip install --root-user-action ignore --no-cache-dir ${_PIP_ADDITIONAL_REQUIREMENTS}
fi


Expand Down
Loading