ARG AIRFLOW_EXTRAS="aiobotocore,amazon,async,celery,cncf-kubernetes,common-io,docker,elasticsearch,fab,ftp,google,google-auth,graphviz,grpc,hashicorp,http,ldap,microsoft-azure,mysql,odbc,openlineage,pandas,postgres,redis,sendgrid,sftp,slack,snowflake,ssh,statsd,uv"

ARG ADDITIONAL_AIRFLOW_EXTRAS=""

ARG ADDITIONAL_PYTHON_DEPS=""

ARG AIRFLOW_HOME=/opt/airflow

ARG AIRFLOW_UID="50000"

ARG AIRFLOW_USER_HOME_DIR=/home/airflow

ARG AIRFLOW_VERSION="2.10.3"

ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm"

ARG AIRFLOW_PIP_VERSION=24.3.1

ARG AIRFLOW_UV_VERSION=0.5.2

ARG AIRFLOW_USE_UV="false"

ARG UV_HTTP_TIMEOUT="300"

ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"

ARG AIRFLOW_IMAGE_README_URL="https://raw.githubusercontent.com/apache/airflow/main/docs/docker-stack/README.md"

ARG AIRFLOW_SOURCES_FROM="Dockerfile"

ARG AIRFLOW_SOURCES_TO="/Dockerfile"

ARG AIRFLOW_VERSION_SPECIFICATION=""

ARG PIP_PROGRESS_BAR="on"

FROM scratch as scripts

COPY <<"EOF" /install_os_dependencies.sh

set -euo pipefail

if [[ "$#" != 1 ]]; then

echo "ERROR! There should be 'runtime' or 'dev' parameter passed as argument.".

exit 1

fi

if [[ "${1}" == "runtime" ]]; then

INSTALLATION_TYPE="RUNTIME"

elif [[ "${1}" == "dev" ]]; then

INSTALLATION_TYPE="dev"

else

echo "ERROR! Wrong argument. Passed ${1} and it should be one of 'runtime' or 'dev'.".

exit 1

fi

function get_dev_apt_deps() {

if [[ "${DEV_APT_DEPS=}" == "" ]]; then

DEV_APT_DEPS="apt-transport-https apt-utils build-essential ca-certificates dirmngr \

export DEV_APT_DEPS

fi

}

function get_runtime_apt_deps() {

local debian_version

local debian_version_apt_deps

# Get debian version without installing lsb_release

# shellcheck disable=SC1091

debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME";)

echo

echo "DEBIAN CODENAME: ${debian_version}"

echo

debian_version_apt_deps="libffi8 libldap-2.5-0 libssl3 netcat-openbsd"

echo

echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION: ${debian_version}"

echo

if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then

RUNTIME_APT_DEPS="apt-transport-https apt-utils ca-certificates \

export RUNTIME_APT_DEPS

fi

}

function install_docker_cli() {

apt-get update

apt-get install ca-certificates curl

install -m 0755 -d /etc/apt/keyrings

curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc

chmod a+r /etc/apt/keyrings/docker.asc

# shellcheck disable=SC1091

echo \

"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \

$(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \

tee /etc/apt/sources.list.d/docker.list > /dev/null

apt-get update

apt-get install -y --no-install-recommends docker-ce-cli

}

function install_debian_dev_dependencies() {

apt-get update

apt-get install -yqq --no-install-recommends apt-utils >/dev/null 2>&1

apt-get install -y --no-install-recommends curl gnupg2 lsb-release

# shellcheck disable=SC2086

export ${ADDITIONAL_DEV_APT_ENV?}

if [[ ${DEV_APT_COMMAND} != "" ]]; then

bash -o pipefail -o errexit -o nounset -o nolog -c "${DEV_APT_COMMAND}"

fi

if [[ ${ADDITIONAL_DEV_APT_COMMAND} != "" ]]; then

bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_DEV_APT_COMMAND}"

fi

apt-get update

local debian_version

local debian_version_apt_deps

# Get debian version without installing lsb_release

# shellcheck disable=SC1091

debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME";)

echo

echo "DEBIAN CODENAME: ${debian_version}"

echo

# shellcheck disable=SC2086

apt-get install -y --no-install-recommends ${DEV_APT_DEPS} ${ADDITIONAL_DEV_APT_DEPS}

}

function install_debian_runtime_dependencies() {

apt-get update

apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1

apt-get install -y --no-install-recommends curl gnupg2 lsb-release

# shellcheck disable=SC2086

export ${ADDITIONAL_RUNTIME_APT_ENV?}

if [[ "${RUNTIME_APT_COMMAND}" != "" ]]; then

bash -o pipefail -o errexit -o nounset -o nolog -c "${RUNTIME_APT_COMMAND}"

fi

if [[ "${ADDITIONAL_RUNTIME_APT_COMMAND}" != "" ]]; then

bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_RUNTIME_APT_COMMAND}"

fi

apt-get update

# shellcheck disable=SC2086

apt-get install -y --no-install-recommends ${RUNTIME_APT_DEPS} ${ADDITIONAL_RUNTIME_APT_DEPS}

apt-get autoremove -yqq --purge

apt-get clean

rm -rf /var/lib/apt/lists/* /var/log/*

}

if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then

get_runtime_apt_deps

install_debian_runtime_dependencies

install_docker_cli

else

get_dev_apt_deps

install_debian_dev_dependencies

install_docker_cli

fi

EOF

COPY <<"EOF" /install_mysql.sh

. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"

set -euo pipefail

common::get_colors

declare -a packages

readonly MYSQL_LTS_VERSION="8.0"

readonly MARIADB_LTS_VERSION="10.11"

: "${INSTALL_MYSQL_CLIENT:?Should be true or false}"

: "${INSTALL_MYSQL_CLIENT_TYPE:-mariadb}"

install_mysql_client() {

if [[ "${1}" == "dev" ]]; then

packages=("libmysqlclient-dev" "mysql-client")

elif [[ "${1}" == "prod" ]]; then

# `libmysqlclientXX` where XX is number, and it should be increased every new GA MySQL release, for example

# 18 - MySQL 5.6.48

# 20 - MySQL 5.7.42

# 21 - MySQL 8.0.34

# 22 - MySQL 8.1

packages=("libmysqlclient21" "mysql-client")

else

echo

echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}"

echo

exit 1

fi

common::import_trusted_gpg "B7B3B788A8D3785C" "mysql"

echo

echo "${COLOR_BLUE}Installing Oracle MySQL client version ${MYSQL_LTS_VERSION}: ${1}${COLOR_RESET}"

echo

echo "deb http://repo.mysql.com/apt/debian/ $(lsb_release -cs) mysql-${MYSQL_LTS_VERSION}" > \

/etc/apt/sources.list.d/mysql.list

apt-get update

apt-get install --no-install-recommends -y "${packages[@]}"

apt-get autoremove -yqq --purge

apt-get clean && rm -rf /var/lib/apt/lists/*

# Remove mysql repository from sources.list.d as MySQL repos have a basic flaw that they put expiry

# date on their GPG signing keys and they sign their repo with those keys. This means that after a

# certain date, the GPG key becomes invalid and if you have the repository added in your sources.list

# then you will not be able to install anything from any other repository. This id unlike any other

# repository we have seen (for example Postgres, MariaDB, MsSQL - all have non-expiring signing keys)

rm /etc/apt/sources.list.d/mysql.list

}

install_mariadb_client() {

# List of compatible package Oracle MySQL -> MariaDB:

# `mysql-client` -> `mariadb-client` or `mariadb-client-compat` (11+)

# `libmysqlclientXX` (where XX is a number) -> `libmariadb3-compat`

# `libmysqlclient-dev` -> `libmariadb-dev-compat`

#

# Different naming against Debian repo which we used before

# that some of packages might contains `-compat` suffix, Debian repo -> MariaDB repo:

# `libmariadb-dev` -> `libmariadb-dev-compat`

# `mariadb-client-core` -> `mariadb-client` or `mariadb-client-compat` (11+)

if [[ "${1}" == "dev" ]]; then

packages=("libmariadb-dev-compat" "mariadb-client")

elif [[ "${1}" == "prod" ]]; then

packages=("libmariadb3-compat" "mariadb-client")

else

echo

echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}"

echo

exit 1

fi

common::import_trusted_gpg "0xF1656F24C74CD1D8" "mariadb"

echo

echo "${COLOR_BLUE}Installing MariaDB client version ${MARIADB_LTS_VERSION}: ${1}${COLOR_RESET}"

echo "${COLOR_YELLOW}MariaDB client protocol-compatible with MySQL client.${COLOR_RESET}"

echo

echo "deb [arch=amd64,arm64] https://archive.mariadb.org/mariadb-${MARIADB_LTS_VERSION}/repo/debian/ $(lsb_release -cs) main" > \

/etc/apt/sources.list.d/mariadb.list

# Make sure that dependencies from MariaDB repo are preferred over Debian dependencies

printf "Package: *\nPin: release o=MariaDB\nPin-Priority: 999\n" > /etc/apt/preferences.d/mariadb

apt-get update

apt-get install --no-install-recommends -y "${packages[@]}"

apt-get autoremove -yqq --purge

apt-get clean && rm -rf /var/lib/apt/lists/*

}

if [[ ${INSTALL_MYSQL_CLIENT:="true"} == "true" ]]; then

if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; then

INSTALL_MYSQL_CLIENT_TYPE="mariadb"

echo

echo "${COLOR_YELLOW}Client forced to mariadb for ARM${COLOR_RESET}"

echo

fi

if [[ "${INSTALL_MYSQL_CLIENT_TYPE}" == "mysql" ]]; then

install_mysql_client "${@}"

elif [[ "${INSTALL_MYSQL_CLIENT_TYPE}" == "mariadb" ]]; then

install_mariadb_client "${@}"

else

echo

echo "${COLOR_RED}Specify either mysql or mariadb, got ${INSTALL_MYSQL_CLIENT_TYPE}${COLOR_RESET}"

echo

exit 1

fi

fi

EOF

COPY <<"EOF" /install_mssql.sh

. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"

set -euo pipefail

common::get_colors

declare -a packages

: "${INSTALL_MSSQL_CLIENT:?Should be true or false}"

function install_mssql_client() {

# Install MsSQL client from Microsoft repositories

if [[ ${INSTALL_MSSQL_CLIENT:="true"} != "true" ]]; then

echo

echo "${COLOR_BLUE}Skip installing mssql client${COLOR_RESET}"

echo

return

fi

packages=("msodbcsql18")

common::import_trusted_gpg "EB3E94ADBE1229CF" "microsoft"

echo

echo "${COLOR_BLUE}Installing mssql client${COLOR_RESET}"

echo

echo "deb [arch=amd64,arm64] https://packages.microsoft.com/debian/$(lsb_release -rs)/prod $(lsb_release -cs) main" > \

/etc/apt/sources.list.d/mssql-release.list

apt-get update -yqq

apt-get upgrade -yqq

ACCEPT_EULA=Y apt-get -yqq install --no-install-recommends "${packages[@]}"

rm -rf /var/lib/apt/lists/*

apt-get autoremove -yqq --purge

apt-get clean && rm -rf /var/lib/apt/lists/*

}

install_mssql_client "${@}"

EOF

COPY <<"EOF" /install_postgres.sh

. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"

set -euo pipefail

common::get_colors

declare -a packages

: "${INSTALL_POSTGRES_CLIENT:?Should be true or false}"

install_postgres_client() {

echo

echo "${COLOR_BLUE}Installing postgres client${COLOR_RESET}"

echo

if [[ "${1}" == "dev" ]]; then

packages=("libpq-dev" "postgresql-client")

elif [[ "${1}" == "prod" ]]; then

packages=("postgresql-client")

else

echo

echo "Specify either prod or dev"

echo

exit 1

fi

common::import_trusted_gpg "7FCC7D46ACCC4CF8" "postgres"

echo "deb [arch=amd64,arm64] https://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > \

/etc/apt/sources.list.d/pgdg.list

apt-get update

apt-get install --no-install-recommends -y "${packages[@]}"

apt-get autoremove -yqq --purge

apt-get clean && rm -rf /var/lib/apt/lists/*

}

if [[ ${INSTALL_POSTGRES_CLIENT:="true"} == "true" ]]; then

install_postgres_client "${@}"

fi

EOF

COPY <<"EOF" /install_packaging_tools.sh

. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"

common::get_colors

common::get_packaging_tool

common::show_packaging_tool_version_and_location

common::install_packaging_tools

EOF

COPY <<"EOF" /install_airflow_dependencies_from_branch_tip.sh

. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"

: "${AIRFLOW_REPO:?Should be set}"

: "${AIRFLOW_BRANCH:?Should be set}"

: "${INSTALL_MYSQL_CLIENT:?Should be true or false}"

: "${INSTALL_POSTGRES_CLIENT:?Should be true or false}"

function install_airflow_dependencies_from_branch_tip() {

echo

echo "${COLOR_BLUE}Installing airflow from ${AIRFLOW_BRANCH}. It is used to cache dependencies${COLOR_RESET}"

echo

if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then

AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}

fi

if [[ ${INSTALL_POSTGRES_CLIENT} != "true" ]]; then

AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/postgres,}

fi

local TEMP_AIRFLOW_DIR

TEMP_AIRFLOW_DIR=$(mktemp -d)

# Install latest set of dependencies - without constraints. This is to download a "base" set of

# dependencies that we can cache and reuse when installing airflow using constraints and latest

# pyproject.toml in the next step (when we install regular airflow).

set -x

curl -fsSL "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz" | \

tar xz -C "${TEMP_AIRFLOW_DIR}" --strip 1

# Make sure editable dependencies are calculated when devel-ci dependencies are installed

${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} \

--editable "${TEMP_AIRFLOW_DIR}[${AIRFLOW_EXTRAS}]"

set +x

common::install_packaging_tools

set -x

echo "${COLOR_BLUE}Uninstalling providers. Dependencies remain${COLOR_RESET}"

# Uninstall airflow and providers to keep only the dependencies. In the future when

# planned https://github.com/pypa/pip/issues/11440 is implemented in pip we might be able to use this

# flag and skip the remove step.

pip freeze | grep apache-airflow-providers | xargs ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} || true

set +x

echo

echo "${COLOR_BLUE}Uninstalling just airflow. Dependencies remain. Now target airflow can be reinstalled using mostly cached dependencies${COLOR_RESET}"

echo

set +x

${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} apache-airflow

rm -rf "${TEMP_AIRFLOW_DIR}"

set -x

# If you want to make sure dependency is removed from cache in your PR when you removed it from

# pyproject.toml - please add your dependency here as a list of strings

# for example:

# DEPENDENCIES_TO_REMOVE=("package_a" "package_b")

# Once your PR is merged, you should make a follow-up PR to remove it from this list

# and increase the AIRFLOW_CI_BUILD_EPOCH in Dockerfile.ci to make sure your cache is rebuilt.

local DEPENDENCIES_TO_REMOVE

# IMPORTANT!! Make sure to increase AIRFLOW_CI_BUILD_EPOCH in Dockerfile.ci when you remove a dependency from that list

DEPENDENCIES_TO_REMOVE=()

if [[ "${DEPENDENCIES_TO_REMOVE[*]}" != "" ]]; then

echo

echo "${COLOR_BLUE}Uninstalling just removed dependencies (temporary until cache refreshes)${COLOR_RESET}"

echo "${COLOR_BLUE}Dependencies to uninstall: ${DEPENDENCIES_TO_REMOVE[*]}${COLOR_RESET}"

echo

set +x

${PACKAGING_TOOL_CMD} uninstall "${DEPENDENCIES_TO_REMOVE[@]}" || true

set -x

# make sure that the dependency is not needed by something else

pip check

fi

}

common::get_colors

common::get_packaging_tool

common::get_airflow_version_specification

common::get_constraints_location

common::show_packaging_tool_version_and_location

install_airflow_dependencies_from_branch_tip

EOF

COPY <<"EOF" /common.sh

set -euo pipefail

function common::get_colors() {

COLOR_BLUE=$'\e[34m'

COLOR_GREEN=$'\e[32m'

COLOR_RED=$'\e[31m'

COLOR_RESET=$'\e[0m'

COLOR_YELLOW=$'\e[33m'

export COLOR_BLUE

export COLOR_GREEN

export COLOR_RED

export COLOR_RESET

export COLOR_YELLOW

}

function common::get_packaging_tool() {

: "${AIRFLOW_USE_UV:?Should be set}"

## IMPORTANT: IF YOU MODIFY THIS FUNCTION YOU SHOULD ALSO MODIFY CORRESPONDING FUNCTION IN

## `scripts/in_container/_in_container_utils.sh`

local PYTHON_BIN

PYTHON_BIN=$(which python)

if [[ ${AIRFLOW_USE_UV} == "true" ]]; then

echo

echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}"

echo

export PACKAGING_TOOL="uv"

export PACKAGING_TOOL_CMD="uv pip"

if [[ -z ${VIRTUAL_ENV=} ]]; then

export EXTRA_INSTALL_FLAGS="--python ${PYTHON_BIN}"

export EXTRA_UNINSTALL_FLAGS="--python ${PYTHON_BIN}"

else

export EXTRA_INSTALL_FLAGS=""

export EXTRA_UNINSTALL_FLAGS=""

fi

export UPGRADE_EAGERLY="--upgrade --resolution highest"

export UPGRADE_IF_NEEDED="--upgrade"

UV_CONCURRENT_DOWNLOADS=$(nproc --all)

export UV_CONCURRENT_DOWNLOADS

else

echo

echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}"

echo

export PACKAGING_TOOL="pip"

export PACKAGING_TOOL_CMD="pip"

export EXTRA_INSTALL_FLAGS="--root-user-action ignore"

export EXTRA_UNINSTALL_FLAGS="--yes"

export UPGRADE_EAGERLY="--upgrade --upgrade-strategy eager"

export UPGRADE_IF_NEEDED="--upgrade --upgrade-strategy only-if-needed"

fi

}

function common::get_airflow_version_specification() {

if [[ -z ${AIRFLOW_VERSION_SPECIFICATION=}

&& -n ${AIRFLOW_VERSION}

&& ${AIRFLOW_INSTALLATION_METHOD} != "." ]]; then

AIRFLOW_VERSION_SPECIFICATION="==${AIRFLOW_VERSION}"

fi

}

function common::get_constraints_location() {

# auto-detect Airflow-constraint reference and location

if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE=}" ]]; then

if [[ ${AIRFLOW_VERSION} =~ v?2.* && ! ${AIRFLOW_VERSION} =~ .*dev.* ]]; then

AIRFLOW_CONSTRAINTS_REFERENCE=constraints-${AIRFLOW_VERSION}

else

AIRFLOW_CONSTRAINTS_REFERENCE=${DEFAULT_CONSTRAINTS_BRANCH}

fi

fi

if [[ -z ${AIRFLOW_CONSTRAINTS_LOCATION=} ]]; then

local constraints_base="https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${AIRFLOW_CONSTRAINTS_REFERENCE}"

local python_version

python_version=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')

AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS_MODE}-${python_version}.txt"

fi

if [[ ${AIRFLOW_CONSTRAINTS_LOCATION} =~ http.* ]]; then

echo

echo "${COLOR_BLUE}Downloading constraints from ${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}"

echo

curl -sSf -o "${HOME}/constraints.txt" "${AIRFLOW_CONSTRAINTS_LOCATION}"

else

echo

echo "${COLOR_BLUE}Copying constraints from ${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}"

echo

cp "${AIRFLOW_CONSTRAINTS_LOCATION}" "${HOME}/constraints.txt"

fi

}

function common::show_packaging_tool_version_and_location() {

echo "PATH=${PATH}"

echo "Installed pip: $(pip --version): $(which pip)"

if [[ ${PACKAGING_TOOL} == "pip" ]]; then

echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}"

else

echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}"

echo "Installed uv: $(uv --version 2>/dev/null || echo "Not installed yet"): $(which uv 2>/dev/null)"

fi

}

function common::install_packaging_tools() {

: "${AIRFLOW_USE_UV:?Should be set}"

if [[ "${VIRTUAL_ENV=}" != "" ]]; then

echo

echo "${COLOR_BLUE}Checking packaging tools in venv: ${VIRTUAL_ENV}${COLOR_RESET}"

echo

else

echo

echo "${COLOR_BLUE}Checking packaging tools for system Python installation: $(which python)${COLOR_RESET}"

echo

fi

if [[ ${AIRFLOW_PIP_VERSION=} == "" ]]; then

echo

echo "${COLOR_BLUE}Installing latest pip version${COLOR_RESET}"

echo

pip install --root-user-action ignore --disable-pip-version-check --upgrade pip

elif [[ ! ${AIRFLOW_PIP_VERSION} =~ ^[0-9].* ]]; then

echo

echo "${COLOR_BLUE}Installing pip version from spec ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"

echo

# shellcheck disable=SC2086

pip install --root-user-action ignore --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}"

else

local installed_pip_version

installed_pip_version=$(python -c 'from importlib.metadata import version; print(version("pip"))')

if [[ ${installed_pip_version} != "${AIRFLOW_PIP_VERSION}" ]]; then

echo

echo "${COLOR_BLUE}(Re)Installing pip version: ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"

echo

pip install --root-user-action ignore --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"

fi

fi

if [[ ${AIRFLOW_UV_VERSION=} == "" ]]; then

echo

echo "${COLOR_BLUE}Installing latest uv version${COLOR_RESET}"

echo

pip install --root-user-action ignore --disable-pip-version-check --upgrade uv

elif [[ ! ${AIRFLOW_UV_VERSION} =~ ^[0-9].* ]]; then

echo

echo "${COLOR_BLUE}Installing uv version from spec ${AIRFLOW_UV_VERSION}${COLOR_RESET}"

echo

# shellcheck disable=SC2086

pip install --root-user-action ignore --disable-pip-version-check "uv @ ${AIRFLOW_UV_VERSION}"

else

local installed_uv_version

installed_uv_version=$(python -c 'from importlib.metadata import version; print(version("uv"))' 2>/dev/null || echo "Not installed yet")

if [[ ${installed_uv_version} != "${AIRFLOW_UV_VERSION}" ]]; then

echo

echo "${COLOR_BLUE}(Re)Installing uv version: ${AIRFLOW_UV_VERSION}${COLOR_RESET}"

echo

# shellcheck disable=SC2086

pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}"

fi

fi

if [[ ${AIRFLOW_PRE_COMMIT_VERSION=} == "" ]]; then

echo

echo "${COLOR_BLUE}Installing latest pre-commit with pre-commit-uv uv${COLOR_RESET}"

echo

uv tool install pre-commit --with pre-commit-uv --with uv

# make sure that the venv/user in .local exists

mkdir -p "${HOME}/.local/bin"

else

echo

echo "${COLOR_BLUE}Installing predefined versions of pre-commit with pre-commit-uv and uv:${COLOR_RESET}"

echo "${COLOR_BLUE}pre_commit(${AIRFLOW_PRE_COMMIT_VERSION}) uv(${AIRFLOW_UV_VERSION}) pre_commit-uv(${AIRFLOW_PRE_COMMIT_UV_VERSION})${COLOR_RESET}"

echo

uv tool install "pre-commit==${AIRFLOW_PRE_COMMIT_VERSION}" \

--with "uv==${AIRFLOW_UV_VERSION}" --with "pre-commit-uv==${AIRFLOW_PRE_COMMIT_UV_VERSION}"

# make sure that the venv/user in .local exists

mkdir -p "${HOME}/.local/bin"

fi

}

function common::import_trusted_gpg() {

common::get_colors

local key=${1:?${COLOR_RED}First argument expects OpenPGP Key ID${COLOR_RESET}}

local name=${2:?${COLOR_RED}Second argument expected trust storage name${COLOR_RESET}}

# Please note that not all servers could be used for retrieve keys

# sks-keyservers.net: Unmaintained and DNS taken down due to GDPR requests.

# keys.openpgp.org: User ID Mandatory, not suitable for APT repositories

# keyring.debian.org: Only accept keys in Debian keyring.

# pgp.mit.edu: High response time.

local keyservers=(

"hkps://keyserver.ubuntu.com"

"hkps://pgp.surf.nl"

)

GNUPGHOME="$(mktemp -d)"

export GNUPGHOME

set +e

for keyserver in $(shuf -e "${keyservers[@]}"); do

echo "${COLOR_BLUE}Try to receive GPG public key ${key} from ${keyserver}${COLOR_RESET}"

gpg --keyserver "${keyserver}" --recv-keys "${key}" 2>&1 && break

echo "${COLOR_YELLOW}Unable to receive GPG public key ${key} from ${keyserver}${COLOR_RESET}"

done

set -e

gpg --export "${key}" > "/etc/apt/trusted.gpg.d/${name}.gpg"

gpgconf --kill all

rm -rf "${GNUPGHOME}"

unset GNUPGHOME

}

EOF

COPY <<"EOF" /pip

COLOR_RED=$'\e[31m'

COLOR_RESET=$'\e[0m'

COLOR_YELLOW=$'\e[33m'

if [[ $(id -u) == "0" ]]; then

echo

echo "${COLOR_RED}You are running pip as root. Please use 'airflow' user to run pip!${COLOR_RESET}"

echo

echo "${COLOR_YELLOW}See: https://airflow.apache.org/docs/docker-stack/build.html#adding-a-new-pypi-package${COLOR_RESET}"

echo

exit 1

fi

exec "${HOME}"/.local/bin/pip "${@}"

EOF

COPY <<"EOF" /install_from_docker_context_files.sh

. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"

function install_airflow_and_providers_from_docker_context_files(){

local flags=()

if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then

AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}

fi

if [[ ${INSTALL_POSTGRES_CLIENT} != "true" ]]; then

AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/postgres,}

fi

if [[ ! -d /docker-context-files ]]; then

echo

echo "${COLOR_RED}You must provide a folder via --build-arg DOCKER_CONTEXT_FILES=<FOLDER> and you missed it!${COLOR_RESET}"

echo

exit 1

fi

# This is needed to get package names for local context packages

${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} --constraint ${HOME}/constraints.txt packaging

if [[ -n ${AIRFLOW_EXTRAS=} ]]; then

AIRFLOW_EXTRAS_TO_INSTALL="[${AIRFLOW_EXTRAS}]"

else

AIRFLOW_EXTRAS_TO_INSTALL=""

fi

# Find Apache Airflow package in docker-context files

readarray -t install_airflow_package < <(EXTRAS="${AIRFLOW_EXTRAS_TO_INSTALL}" \

python /scripts/docker/get_package_specs.py /docker-context-files/apache?airflow?[0-9]*.{whl,tar.gz} 2>/dev/null || true)

echo

echo "${COLOR_BLUE}Found airflow packages in docker-context-files folder: ${install_airflow_package[*]}${COLOR_RESET}"

echo

if [[ -z "${install_airflow_package[*]}" && ${AIRFLOW_VERSION=} != "" ]]; then

# When we install only provider packages from docker-context files, we need to still

# install airflow from PyPI when AIRFLOW_VERSION is set. This handles the case where

# pre-release dockerhub image of airflow is built, but we want to install some providers from

# docker-context files

install_airflow_package=("apache-airflow[${AIRFLOW_EXTRAS}]==${AIRFLOW_VERSION}")

fi

# Find Provider/TaskSDK packages in docker-context files

readarray -t airflow_packages< <(python /scripts/docker/get_package_specs.py /docker-context-files/apache?airflow?{providers,task?sdk}*.{whl,tar.gz} 2>/dev/null || true)

echo

echo "${COLOR_BLUE}Found provider packages in docker-context-files folder: ${airflow_packages[*]}${COLOR_RESET}"

echo

if [[ ${USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES=} == "true" ]]; then

local python_version

python_version=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')

local local_constraints_file=/docker-context-files/constraints-"${python_version}"/${AIRFLOW_CONSTRAINTS_MODE}-"${python_version}".txt

if [[ -f "${local_constraints_file}" ]]; then

echo

echo "${COLOR_BLUE}Installing docker-context-files packages with constraints found in ${local_constraints_file}${COLOR_RESET}"

echo

# force reinstall all airflow + provider packages with constraints found in

flags=(--upgrade --constraint "${local_constraints_file}")

echo

echo "${COLOR_BLUE}Copying ${local_constraints_file} to ${HOME}/constraints.txt${COLOR_RESET}"

echo

cp "${local_constraints_file}" "${HOME}/constraints.txt"

else

echo

echo "${COLOR_BLUE}Installing docker-context-files packages with constraints from GitHub${COLOR_RESET}"

echo

flags=(--constraint "${HOME}/constraints.txt")

fi

else

echo

echo "${COLOR_BLUE}Installing docker-context-files packages without constraints${COLOR_RESET}"

echo

flags=()

fi

set -x

${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} \

${ADDITIONAL_PIP_INSTALL_FLAGS} \

"${flags[@]}" \

"${install_airflow_package[@]}" "${airflow_packages[@]}"

set +x

common::install_packaging_tools

pip check

}

function install_all_other_packages_from_docker_context_files() {

echo

echo "${COLOR_BLUE}Force re-installing all other package from local files without dependencies${COLOR_RESET}"

echo

local reinstalling_other_packages

# shellcheck disable=SC2010

reinstalling_other_packages=$(ls /docker-context-files/*.{whl,tar.gz} 2>/dev/null | \

grep -v apache_airflow | grep -v apache-airflow || true)

if [[ -n "${reinstalling_other_packages}" ]]; then

set -x

${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} \

--force-reinstall --no-deps --no-index ${reinstalling_other_packages}

common::install_packaging_tools

set +x

fi

}

common::get_colors

common::get_packaging_tool

common::get_airflow_version_specification

common::get_constraints_location

common::show_packaging_tool_version_and_location

install_airflow_and_providers_from_docker_context_files

install_all_other_packages_from_docker_context_files

EOF

COPY <<"EOF" /get_package_specs.py

from __future__ import annotations

import os

import sys

from pathlib import Path

from packaging.utils import (

InvalidSdistFilename,

InvalidWheelFilename,

parse_sdist_filename,

parse_wheel_filename,

)

def print_package_specs(extras: str = "") -> None:

for package_path in sys.argv[1:]:

try:

package, _, _, _ = parse_wheel_filename(Path(package_path).name)

except InvalidWheelFilename:

try:

package, _ = parse_sdist_filename(Path(package_path).name)

except InvalidSdistFilename:

print(f"Could not parse package name from {package_path}", file=sys.stderr)

continue

print(f"{package}{extras} @ file://{package_path}")

if __name__ == "__main__":

print_package_specs(extras=os.environ.get("EXTRAS", ""))

EOF

COPY <<"EOF" /install_airflow.sh

. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"

function install_airflow() {

# Remove mysql from extras if client is not going to be installed

if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then

AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}

echo "${COLOR_YELLOW}MYSQL client installation is disabled. Extra 'mysql' installations were therefore omitted.${COLOR_RESET}"

fi

# Remove postgres from extras if client is not going to be installed

if [[ ${INSTALL_POSTGRES_CLIENT} != "true" ]]; then

AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/postgres,}

echo "${COLOR_YELLOW}Postgres client installation is disabled. Extra 'postgres' installations were therefore omitted.${COLOR_RESET}"

fi

# Determine the installation_command_flags based on AIRFLOW_INSTALLATION_METHOD method

local installation_command_flags

if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then

# We need _a_ file in there otherwise the editable install doesn't include anything in the .pth file

mkdir -p ./providers/src/airflow/providers/

touch ./providers/src/airflow/providers/__init__.py

# Similarly we need _a_ file for task_sdk too

mkdir -p ./task_sdk/src/airflow/sdk/

echo '__version__ = "0.0.0dev0"' > ./task_sdk/src/airflow/sdk/__init__.py

trap 'rm -f ./providers/src/airflow/providers/__init__.py ./task_sdk/src/airflow/__init__.py 2>/dev/null' EXIT

# When installing from sources - we always use `--editable` mode

installation_command_flags="--editable .[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION} --editable ./providers --editable ./task_sdk"

elif [[ ${AIRFLOW_INSTALLATION_METHOD} == "apache-airflow" ]]; then

installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"

elif [[ ${AIRFLOW_INSTALLATION_METHOD} == apache-airflow\ @\ * ]]; then

installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}] @ ${AIRFLOW_VERSION_SPECIFICATION/apache-airflow @//}"

else

echo

echo "${COLOR_RED}The '${INSTALLATION_METHOD}' installation method is not supported${COLOR_RESET}"

echo

echo "${COLOR_YELLOW}Supported methods are ('.', 'apache-airflow', 'apache-airflow @ URL')${COLOR_RESET}"

echo

exit 1

fi

if [[ "${UPGRADE_INVALIDATION_STRING=}" != "" ]]; then

echo

echo "${COLOR_BLUE}Remove airflow and all provider packages installed before potentially${COLOR_RESET}"

echo

set -x

${PACKAGING_TOOL_CMD} freeze | grep apache-airflow | xargs ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} 2>/dev/null || true

set +x

echo

echo "${COLOR_BLUE}Installing all packages in eager upgrade mode. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"

echo

set -x

${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_EAGERLY} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=}

set +x

common::install_packaging_tools

echo

echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"

echo

pip check

else

echo

echo "${COLOR_BLUE}Installing all packages with constraints. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"

echo

set -x

# Install all packages with constraints

if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then

set +x

echo

echo "${COLOR_YELLOW}Likely pyproject.toml has new dependencies conflicting with constraints.${COLOR_RESET}"

echo

echo "${COLOR_BLUE}Falling back to no-constraints installation.${COLOR_RESET}"

echo

set -x

${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}

fi

set +x

common::install_packaging_tools

echo

echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"

echo

pip check

fi

}

common::get_colors

common::get_packaging_tool

common::get_airflow_version_specification

common::get_constraints_location

common::show_packaging_tool_version_and_location

install_airflow

EOF

COPY <<"EOF" /install_additional_dependencies.sh

set -euo pipefail

: "${ADDITIONAL_PYTHON_DEPS:?Should be set}"

. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"

function install_additional_dependencies() {

if [[ "${UPGRADE_INVALIDATION_STRING=}" != "" ]]; then

echo

echo "${COLOR_BLUE}Installing additional dependencies while upgrading to newer dependencies${COLOR_RESET}"

echo

set -x

${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_EAGERLY} \

${ADDITIONAL_PIP_INSTALL_FLAGS} \

${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=}

set +x

common::install_packaging_tools

echo