# syntax=docker/dockerfile:1.4

# Licensed to the Apache Software Foundation (ASF) under one

# or more contributor license agreements. See the NOTICE file

# distributed with this work for additional information

# regarding copyright ownership. The ASF licenses this file

# to you under the Apache License, Version 2.0 (the

# "License"); you may not use this file except in compliance

# with the License. You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing,

# software distributed under the License is distributed on an

# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

# KIND, either express or implied. See the License for the

# specific language governing permissions and limitations

# under the License.

#

# WARNING: THIS DOCKERFILE IS NOT INTENDED FOR PRODUCTION USE OR DEPLOYMENT.

#

# PYTHON_LTO: Controls whether Python is built with Link-Time Optimization (LTO).

#

# Link-Time Optimization uses MD5 checksums during the compilation process to verify

# object files and intermediate representations. In FIPS-compliant environments, MD5

# is blocked as it's not an approved cryptographic algorithm (see FIPS 140-2/140-3).

# This can cause Python builds with LTO to fail when FIPS mode is enabled.

#

# When building FIPS-compliant images, set this to "false" to disable LTO:

# docker build --build-arg PYTHON_LTO="false" ...

#

# Default: "true" (LTO enabled for better performance)

#

# Related: https://github.com/apache/airflow/issues/58337

ARG PYTHON_LTO="true"

ARG BASE_IMAGE="debian:bookworm-slim"

##############################################################################################

# This is the script image where we keep all inlined bash scripts needed in other segments

# We use BASE_IMAGE to make sure that the scripts are different for different platforms.

##############################################################################################

FROM ${BASE_IMAGE} as scripts

##############################################################################################

# Please DO NOT modify the inlined scripts manually. The content of those files will be

# replaced by prek automatically from the "scripts/docker/" folder.

# This is done in order to avoid problems with caching and file permissions and in order to

# make the PROD Dockerfile standalone

##############################################################################################

# The content below is automatically copied from scripts/docker/install_os_dependencies.sh

COPY <<"EOF" /install_os_dependencies.sh

#!/usr/bin/env bash

set -euo pipefail

if [[ "$#" != 1 ]]; then

echo

echo "ERROR! There should be 'runtime', 'ci' or 'dev' parameter passed as argument.".

echo

exit 1

fi

AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.10.18}

PYTHON_LTO=${PYTHON_LTO:-true}

GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4}

TEMURIN_VERSION=${TEMURIN_VERSION:-11}

NODEJS_VERSION=${NODEJS_VERSION:-22.23.1}

PNPM_VERSION=${PNPM_VERSION:-10.28.1}

RUSTUP_DEFAULT_TOOLCHAIN=${RUSTUP_DEFAULT_TOOLCHAIN:-stable}

RUSTUP_VERSION=${RUSTUP_VERSION:-1.29.0}

COSIGN_VERSION=${COSIGN_VERSION:-3.0.5}

if [[ "${1}" == "runtime" ]]; then

INSTALLATION_TYPE="RUNTIME"

elif [[ "${1}" == "dev" ]]; then

INSTALLATION_TYPE="DEV"

elif [[ "${1}" == "ci" ]]; then

INSTALLATION_TYPE="CI"

else

echo

echo "ERROR! Wrong argument. Passed ${1} and it should be one of 'runtime', 'ci' or 'dev'.".

echo

exit 1

fi

function get_dev_apt_deps() {

if [[ "${DEV_APT_DEPS=}" == "" ]]; then

DEV_APT_DEPS="\

apt-transport-https \

apt-utils \

build-essential \

dirmngr \

freetds-bin \

freetds-dev \

git \

graphviz \

graphviz-dev \

krb5-user \

lcov \

ldap-utils \

libbluetooth-dev \

libbz2-dev \

libc6-dev \

libdb-dev \

libev-dev \

libev4 \

libffi-dev \

libgdbm-compat-dev \

libgdbm-dev \

libgeos-dev \

libkrb5-dev \

libldap2-dev \

libleveldb-dev \

libleveldb1d \

liblzma-dev \

libncurses5-dev \

libreadline6-dev \

libsasl2-2 \

libsasl2-dev \

libsasl2-modules \

libsqlite3-dev \

libssl-dev \

libxmlsec1 \

libxmlsec1-dev \

libzstd-dev \

locales \

lsb-release \

lzma \

lzma-dev \

openssh-client \

openssl \

pkg-config \

pkgconf \

sasl2-bin \

sqlite3 \

sudo \

tdsodbc \

tk-dev \

unixodbc \

unixodbc-dev \

uuid-dev \

wget \

xz-utils \

zlib1g-dev \

"

export DEV_APT_DEPS

fi

}

function get_runtime_apt_deps() {

local debian_version

local debian_version_apt_deps

# Get debian version without installing lsb_release

# shellcheck disable=SC1091

debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME";)

echo

echo "DEBIAN CODENAME: ${debian_version}"

echo

debian_version_apt_deps="\

libffi8 \

libldap-2.5-0 \

libssl3 \

netcat-openbsd\

"

echo

echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION: ${debian_version}"

echo

if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then

RUNTIME_APT_DEPS="\

${debian_version_apt_deps} \

apt-transport-https \

apt-utils \

curl \

dumb-init \

freetds-bin \

git \

gnupg \

iputils-ping \

krb5-user \

ldap-utils \

libev4 \

libgeos-dev \

libsasl2-2 \

libsasl2-modules \

libxmlsec1 \

locales \

lsb-release \

openssh-client \

rsync \

sasl2-bin \

sqlite3 \

sudo \

tdsodbc \

unixodbc \

wget\

"

export RUNTIME_APT_DEPS

fi

}

function install_docker_cli() {

apt-get update

apt-get install ca-certificates curl

install -m 0755 -d /etc/apt/keyrings

curl -fsSL --retry 3 --retry-delay 5 https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc

chmod a+r /etc/apt/keyrings/docker.asc

# shellcheck disable=SC1091

echo \

"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \

$(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \

tee /etc/apt/sources.list.d/docker.list > /dev/null

apt-get update

apt-get install -y --no-install-recommends docker-ce-cli

}

function install_debian_dev_dependencies() {

apt-get update

apt-get install -yqq --no-install-recommends apt-utils >/dev/null 2>&1

apt-get install -y --no-install-recommends wget curl gnupg2 lsb-release ca-certificates

# shellcheck disable=SC2086

export ${ADDITIONAL_DEV_APT_ENV?}

if [[ ${DEV_APT_COMMAND} != "" ]]; then

bash -o pipefail -o errexit -o nounset -o nolog -c "${DEV_APT_COMMAND}"

fi

if [[ ${ADDITIONAL_DEV_APT_COMMAND} != "" ]]; then

bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_DEV_APT_COMMAND}"

fi

apt-get update

local debian_version

local debian_version_apt_deps

# Get debian version without installing lsb_release

# shellcheck disable=SC1091

debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME";)

echo

echo "DEBIAN CODENAME: ${debian_version}"

echo

# shellcheck disable=SC2086

apt-get install -y --no-install-recommends ${DEV_APT_DEPS}

}

function install_additional_dev_dependencies() {

if [[ "${ADDITIONAL_DEV_APT_DEPS=}" != "" ]]; then

# shellcheck disable=SC2086

apt-get install -y --no-install-recommends ${ADDITIONAL_DEV_APT_DEPS}

fi

}

function link_python() {

# link python binaries to /usr/local/bin and /usr/python/bin with and without 3 suffix

# Links in /usr/local/bin are needed for tools that expect python to be there

# Links in /usr/python/bin are needed for tools that are detecting home of python installation including

# lib/site-packages. The /usr/python/bin should be first in PATH in order to help with the last part.

for dst in pip3 python3 python3-config; do

src="$(echo "${dst}" | tr -d 3)"

echo "Linking ${dst} in /usr/local/bin and /usr/python/bin"

ln -sv "/usr/python/bin/${dst}" "/usr/local/bin/${dst}"

for dir in /usr/local/bin /usr/python/bin; do

if [[ ! -e "${dir}/${src}" ]]; then

echo "Creating ${src} - > ${dst} link in ${dir}"

ln -sv "${dir}/${dst}" "${dir}/${src}"

fi

done

done

for dst in /usr/python/lib/*

do

src="/usr/local/lib/$(basename "${dst}")"

if [[ -e "${src}" ]]; then

rm -rf "${src}"

fi

echo "Linking ${dst} to ${src}"

ln -sv "${dst}" "${src}"

done

ldconfig

}

function install_debian_runtime_dependencies() {

apt-get update

apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1

apt-get install -y --no-install-recommends wget curl gnupg2 lsb-release ca-certificates

# shellcheck disable=SC2086

export ${ADDITIONAL_RUNTIME_APT_ENV?}

if [[ "${RUNTIME_APT_COMMAND}" != "" ]]; then

bash -o pipefail -o errexit -o nounset -o nolog -c "${RUNTIME_APT_COMMAND}"

fi

if [[ "${ADDITIONAL_RUNTIME_APT_COMMAND}" != "" ]]; then

bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_RUNTIME_APT_COMMAND}"

fi

apt-get update

# shellcheck disable=SC2086

apt-get install -y --no-install-recommends ${RUNTIME_APT_DEPS} ${ADDITIONAL_RUNTIME_APT_DEPS}

apt-get autoremove -yqq --purge

apt-get clean

link_python

rm -rf /var/lib/apt/lists/* /var/log/*

}

function install_cosign() {

local arch

arch="$(dpkg --print-architecture)"

declare -A cosign_sha256s=(

# https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign_checksums.txt

[amd64]="db15cc99e6e4837daabab023742aaddc3841ce57f193d11b7c3e06c8003642b2"

[arm64]="d098f3168ae4b3aa70b4ca78947329b953272b487727d1722cb3cb098a1a20ab"

)

local cosign_sha256="${cosign_sha256s[${arch}]}"

if [[ -z "${cosign_sha256}" ]]; then

echo "Unsupported architecture for cosign: ${arch}"

exit 1

fi

curl -fsSL --retry 3 --retry-delay 5 \

"https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign-linux-${arch}" \

-o /tmp/cosign

echo "${cosign_sha256} /tmp/cosign" | sha256sum --check

chmod +x /tmp/cosign

}

function install_python() {

# If system python (3.11 in bookworm) is installed (via automatic installation of some dependencies for example), we need

# to fail and make sure that it is not there, because there can be strange interactions if we install

# newer version and system libraries are installed, because

# when you create a virtualenv part of the shared libraries of Python can be taken from the system

# Installation leading to weird errors when you want to install some modules - for example when you install ssl:

# /usr/python/lib/python3.11/lib-dynload/_ssl.cpython-311-aarch64-linux-gnu.so: undefined symbol: _PyModule_Add

if dpkg -l | grep '^ii' | grep '^ii libpython' >/dev/null; then

echo

echo "ERROR! System python is installed by one of the previous steps"

echo

echo "Please make sure that no python packages are installed by default. Displaying the reason why libpython3.11 is installed:"

echo

apt-get install -yqq aptitude >/dev/null

aptitude why libpython3.11

echo

exit 1

else

echo

echo "GOOD! System python is not installed - OK"

echo

fi

wget --tries=3 --waitretry=5 -O python.tar.xz "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz"

local major_minor_version

major_minor_version="${AIRFLOW_PYTHON_VERSION%.*}"

local major minor

major="${major_minor_version%.*}"

minor="${major_minor_version#*.}"

echo "Verifying Python ${AIRFLOW_PYTHON_VERSION} (${major_minor_version})"

if [[ "${major}" -gt 3 ]] || [[ "${major}" -eq 3 && "${minor}" -ge 11 ]]; then

# Sigstore verification for Python >= 3.11 (PEP 761)

declare -A sigstore_identities=(

# https://peps.python.org/pep-0664/#release-manager-and-crew

[3.11]="pablogsal@python.org"

# https://peps.python.org/pep-0693/#release-manager-and-crew

[3.12]="thomas@python.org"

# https://peps.python.org/pep-0719/#release-manager-and-crew

[3.13]="thomas@python.org"

# https://peps.python.org/pep-0745/#release-manager-and-crew

[3.14]="hugo@python.org"

)

declare -A sigstore_issuers=(

[3.11]="https://accounts.google.com"

[3.12]="https://accounts.google.com"

[3.13]="https://accounts.google.com"

[3.14]="https://github.com/login/oauth"

)

wget --tries=3 --waitretry=5 -O python.tar.xz.sigstore \

"https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.sigstore"

install_cosign

local identity="${sigstore_identities[${major_minor_version}]}"

local issuer="${sigstore_issuers[${major_minor_version}]}"

/tmp/cosign verify-blob \

--bundle python.tar.xz.sigstore \

--certificate-identity "${identity}" \

--certificate-oidc-issuer "${issuer}" \

python.tar.xz

rm -f python.tar.xz.sigstore /tmp/cosign

else

# PGP verification for Python 3.10

declare -A keys=(

# gpg: key 64E628F8D684696D: public key "Pablo Galindo Salgado <pablogsal@gmail.com>" imported

# https://peps.python.org/pep-0619/#release-manager-and-crew

[3.10]="A035C8C19219BA821ECEA86B64E628F8D684696D"

)

wget --tries=3 --waitretry=5 -O python.tar.xz.asc \

"https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.asc"

GNUPGHOME="$(mktemp -d)"; export GNUPGHOME

local gpg_key="${keys[${major_minor_version}]}"

echo "Using GPG key ${gpg_key}"

gpg --batch --import "/scripts/docker/keys/python-${major_minor_version}.asc"

gpg --batch --verify python.tar.xz.asc python.tar.xz

gpgconf --kill all

rm -rf "${GNUPGHOME}" python.tar.xz.asc

fi

mkdir -p /usr/src/python

tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz

rm python.tar.xz

cd /usr/src/python

arch="$(dpkg --print-architecture)"; arch="${arch##*-}"

gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"

EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)"

EXTRA_CFLAGS="${EXTRA_CFLAGS:-} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer";

LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"

LDFLAGS="${LDFLAGS:--Wl},--strip-all"

# Link-Time Optimization (LTO) uses MD5 checksums for object file verification during

# compilation. In FIPS mode, MD5 is blocked as a non-approved algorithm, causing builds

# to fail. The PYTHON_LTO variable allows disabling LTO for FIPS-compliant builds.

# See: https://github.com/apache/airflow/issues/58337

local lto_option=""

if [[ "${PYTHON_LTO:-true}" == "true" ]]; then

lto_option="--with-lto"

fi

local build_log

build_log=$(mktemp)

echo "Building Python ${AIRFLOW_PYTHON_VERSION} from source..."

if ! (

./configure --enable-optimizations --prefix=/usr/python/ --with-ensurepip --build="$gnuArch" \

--enable-loadable-sqlite-extensions --enable-option-checking=fatal \

--enable-shared ${lto_option} && \

make -s -j "$(nproc)" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \

"LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" python && \

make -s -j "$(nproc)" install

) > "${build_log}" 2>&1; then

echo

echo "ERROR! Python build failed. Build output:"

echo

cat "${build_log}"

rm -f "${build_log}"

exit 1

fi

rm -f "${build_log}"

cd /

rm -rf /usr/src/python

find /usr/python -depth \

\( \

\( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \

-o \( -type f -a \( -name 'libpython*.a' \) \) \

\) -exec rm -rf '{}' +

link_python

}

function install_golang() {

curl --retry 3 --retry-delay 5 "https://dl.google.com/go/go${GOLANG_MAJOR_MINOR_VERSION}.linux-$(dpkg --print-architecture).tar.gz" -o "go${GOLANG_MAJOR_MINOR_VERSION}.linux.tar.gz"

rm -rf /usr/local/go && tar -C /usr/local -xzf go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz

rm -f go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz

}

function install_jdk() {

# Install Eclipse Temurin JDK from the Adoptium apt repository (https://adoptium.net/installation/linux/).

apt-get update -qq

apt-get install -y --no-install-recommends wget gnupg apt-transport-https ca-certificates

mkdir -p /etc/apt/keyrings

wget -qO - https://packages.adoptium.net/artifactory/api/gpg/key/public \

| tee /etc/apt/keyrings/adoptium.asc > /dev/null

# shellcheck disable=SC1091

DISTRO_CODENAME=$(. /etc/os-release; echo "${VERSION_CODENAME}")

echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] \

https://packages.adoptium.net/artifactory/deb ${DISTRO_CODENAME} main" \

| tee /etc/apt/sources.list.d/adoptium.list > /dev/null

apt-get update -qq

apt-get install -y --no-install-recommends "temurin-${TEMURIN_VERSION}-jdk"

apt-get clean

rm -rf /var/lib/apt/lists/*

}

function install_nodejs() {

local arch

arch="$(dpkg --print-architecture)"

declare -A nodejs_targets=(

[amd64]="linux-x64"

[arm64]="linux-arm64"

)

declare -A nodejs_sha256s=(

# https://nodejs.org/dist/v${NODEJS_VERSION}/SHASUMS256.txt

[amd64]="9749e988f437343b7fa832c69ded82a312e41a03116d766797ac14f6f9eee578"

[arm64]="0294e8b915ab75f92c7513d2fcb830ae06e10684e6c603e99a87dbf8835389c1"

)

local target="${nodejs_targets[${arch}]}"

local nodejs_sha256="${nodejs_sha256s[${arch}]}"

if [[ -z "${target}" ]]; then

echo "Unsupported architecture for nodejs: ${arch}"

exit 1

fi

curl --retry 3 --retry-delay 5 \

"https://nodejs.org/dist/v${NODEJS_VERSION}/node-v${NODEJS_VERSION}-${target}.tar.xz" \

-o /tmp/nodejs.tar.xz

echo "${nodejs_sha256} /tmp/nodejs.tar.xz" | sha256sum --check

tar -xJf /tmp/nodejs.tar.xz --strip-components=1 -C /usr/local --no-same-owner

rm -f /tmp/nodejs.tar.xz

corepack enable --install-directory /usr/local/bin

# corepack enable only writes shims; prepare downloads and caches the pnpm binary into the

# image so it is available offline and matches the version ts-sdk pins.

corepack prepare "pnpm@${PNPM_VERSION}" --activate

}

function install_rustup() {

local arch

arch="$(dpkg --print-architecture)"

declare -A rustup_targets=(

[amd64]="x86_64-unknown-linux-gnu"

[arm64]="aarch64-unknown-linux-gnu"

)

declare -A rustup_sha256s=(

# https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/{target}/rustup-init.sha256

[amd64]="4acc9acc76d5079515b46346a485974457b5a79893cfb01112423c89aeb5aa10"

[arm64]="9732d6c5e2a098d3521fca8145d826ae0aaa067ef2385ead08e6feac88fa5792"

)

local target="${rustup_targets[${arch}]}"

local rustup_sha256="${rustup_sha256s[${arch}]}"

if [[ -z "${target}" ]]; then

echo "Unsupported architecture for rustup: ${arch}"

exit 1

fi

curl --proto '=https' --tlsv1.2 -sSf --retry 3 --retry-delay 5 \

"https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/${target}/rustup-init" \

-o /tmp/rustup-init

echo "${rustup_sha256} /tmp/rustup-init" | sha256sum --check

chmod +x /tmp/rustup-init

# Building wheels from source needs only rustc and cargo. The default profile also adds

# rust-docs, clippy and rustfmt, which add tens of thousands of files to the image.

/tmp/rustup-init -y --profile minimal --default-toolchain "${RUSTUP_DEFAULT_TOOLCHAIN}"

rm -f /tmp/rustup-init

}

function apt_clean() {

apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false

rm -rf /var/lib/apt/lists/* /var/log/*

}

if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then

get_runtime_apt_deps

install_debian_runtime_dependencies

install_docker_cli

apt_clean

else

get_dev_apt_deps

install_debian_dev_dependencies

install_python

install_additional_dev_dependencies

install_rustup

if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then

install_golang

install_jdk

install_nodejs

fi

install_docker_cli

apt_clean

fi

EOF

# The content below is automatically copied from scripts/docker/install_mysql.sh

COPY <<"EOF" /install_mysql.sh

#!/usr/bin/env bash

. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"

set -euo pipefail

common::get_colors

declare -a packages

readonly MARIADB_LTS_VERSION="10.11"

: "${INSTALL_MYSQL_CLIENT:?Should be true or false}"

: "${INSTALL_MYSQL_CLIENT_TYPE:-mariadb}"

if [[ "${INSTALL_MYSQL_CLIENT}" != "true" && "${INSTALL_MYSQL_CLIENT}" != "false" ]]; then

echo

echo "${COLOR_RED}INSTALL_MYSQL_CLIENT must be either true or false${COLOR_RESET}"

echo

exit 1

fi

if [[ "${INSTALL_MYSQL_CLIENT_TYPE}" != "mysql" && "${INSTALL_MYSQL_CLIENT_TYPE}" != "mariadb" ]]; then

echo

echo "${COLOR_RED}INSTALL_MYSQL_CLIENT_TYPE must be either mysql or mariadb${COLOR_RESET}"

echo

exit 1

fi

if [[ "${INSTALL_MYSQL_CLIENT_TYPE}" == "mysql" ]]; then

echo

echo "${COLOR_RED}The 'mysql' client type is not supported any more. Use 'mariadb' instead.${COLOR_RESET}"

echo

echo "The MySQL drivers are wrongly packaged and released by Oracle with an expiration date on their GPG keys,"

echo "which causes builds to fail after the expiration date. MariaDB client is protocol-compatible with MySQL client."

echo ""

echo "Every two years the MySQL packages fail and Oracle team is always surprised and struggling"

echo "with fixes and re-signing the packages which lasts few days"

echo "See https://bugs.mysql.com/bug.php?id=113432 for more details."

echo "As a community we are not able to support this broken packaging practice from Oracle"

echo "Feel free however to install MySQL drivers on your own as extension of the image."

echo

exit 1

fi

retry() {

local retries=3

local count=0

# adding delay of 10 seconds

local delay=10

until "$@"; do

exit_code=$?

count=$((count + 1))

if [[ $count -lt $retries ]]; then

echo "Command failed. Attempt $count/$retries. Retrying in ${delay}s..."

sleep $delay

else

echo "Command failed after $retries attempts."

return $exit_code

fi

done

}

install_mariadb_client() {

# List of compatible package Oracle MySQL -> MariaDB:

# `mysql-client` -> `mariadb-client` or `mariadb-client-compat` (11+)

# `libmysqlclientXX` (where XX is a number) -> `libmariadb3-compat`

# `libmysqlclient-dev` -> `libmariadb-dev-compat`

#

# Different naming against Debian repo which we used before

# that some of packages might contains `-compat` suffix, Debian repo -> MariaDB repo:

# `libmariadb-dev` -> `libmariadb-dev-compat`

# `mariadb-client-core` -> `mariadb-client` or `mariadb-client-compat` (11+)

if [[ "${1}" == "dev" ]]; then

packages=("libmariadb-dev-compat" "mariadb-client")

elif [[ "${1}" == "prod" ]]; then

packages=("libmariadb3-compat" "mariadb-client")

else

echo

echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}"

echo

exit 1

fi

common::import_trusted_gpg "0xF1656F24C74CD1D8" "mariadb"

echo

echo "${COLOR_BLUE}Installing MariaDB client version ${MARIADB_LTS_VERSION}: ${1}${COLOR_RESET}"

echo "${COLOR_YELLOW}MariaDB client protocol-compatible with MySQL client.${COLOR_RESET}"

echo

echo "deb [arch=amd64,arm64] https://archive.mariadb.org/mariadb-${MARIADB_LTS_VERSION}/repo/debian/ $(lsb_release -cs) main" > \

/etc/apt/sources.list.d/mariadb.list

# Make sure that dependencies from MariaDB repo are preferred over Debian dependencies

printf "Package: *\nPin: release o=MariaDB\nPin-Priority: 999\n" > /etc/apt/preferences.d/mariadb

retry apt-get update

retry apt-get install --no-install-recommends -y "${packages[@]}"

apt-get autoremove -yqq --purge

apt-get clean && rm -rf /var/lib/apt/lists/*

}

if [[ ${INSTALL_MYSQL_CLIENT:="true"} == "true" ]]; then

install_mariadb_client "${@}"

fi

EOF

# The content below is automatically copied from scripts/docker/install_mssql.sh

COPY <<"EOF" /install_mssql.sh

#!/usr/bin/env bash

. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"

set -euo pipefail

common::get_colors

declare -a packages

: "${INSTALL_MSSQL_CLIENT:?Should be true or false}"

function install_mssql_client() {

# Install MsSQL client from Microsoft repositories

if [[ ${INSTALL_MSSQL_CLIENT:="true"} != "true" ]]; then

echo

echo "${COLOR_BLUE}Skip installing mssql client${COLOR_RESET}"

echo

return

fi

packages=("msodbcsql18")

common::import_trusted_gpg "EB3E94ADBE1229CF" "microsoft"

echo

echo "${COLOR_BLUE}Installing mssql client${COLOR_RESET}"

echo

echo "deb [arch=amd64,arm64] https://packages.microsoft.com/debian/$(lsb_release -rs)/prod $(lsb_release -cs) main" > \

/etc/apt/sources.list.d/mssql-release.list &&

mkdir -p /opt/microsoft/msodbcsql18 &&

touch /opt/microsoft/msodbcsql18/ACCEPT_EULA &&

apt-get update -yqq &&

apt-get upgrade -yqq &&

apt-get -yqq install --no-install-recommends "${packages[@]}" &&

apt-get autoremove -yqq --purge &&

apt-get clean &&

rm -rf /var/lib/apt/lists/*

}

install_mssql_client "${@}"

EOF

# The content below is automatically copied from scripts/docker/install_postgres.sh

COPY <<"EOF" /install_postgres.sh

#!/usr/bin/env bash

. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"

set -euo pipefail

common::get_colors

declare -a packages

: "${INSTALL_POSTGRES_CLIENT:?Should be true or false}"

install_postgres_client() {

echo

echo "${COLOR_BLUE}Installing postgres client${COLOR_RESET}"

echo

if [[ "${1}" == "dev" ]]; then

packages=("libpq-dev" "postgresql-client")

elif [[ "${1}" == "prod" ]]; then

packages=("postgresql-client")

else

echo

echo "Specify either prod or dev"

echo

exit 1

fi

common::import_trusted_gpg "7FCC7D46ACCC4CF8" "postgres"

echo "deb [arch=amd64,arm64] https://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > \

/etc/apt/sources.list.d/pgdg.list

apt-get update

apt-get install --no-install-recommends -y "${packages[@]}"

apt-get autoremove -yqq --purge

apt-get clean && rm -rf /var/lib/apt/lists/*

}

if [[ ${INSTALL_POSTGRES_CLIENT:="true"} == "true" ]]; then

install_postgres_client "${@}"

fi

EOF

# The content below is automatically copied from scripts/docker/install_packaging_tools.sh

COPY <<"EOF" /install_packaging_tools.sh

#!/usr/bin/env bash

. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"

common::get_colors

common::get_packaging_tool

common::show_packaging_tool_version_and_location

common::install_packaging_tools

EOF

# The content below is automatically copied from scripts/docker/common.sh

COPY <<"EOF" /common.sh

#!/usr/bin/env bash

set -euo pipefail

function common::get_colors() {

COLOR_BLUE=$'\e[34m'

COLOR_GREEN=$'\e[32m'

COLOR_RED=$'\e[31m'

COLOR_RESET=$'\e[0m'

COLOR_YELLOW=$'\e[33m'

export COLOR_BLUE

export COLOR_GREEN

export COLOR_RED

export COLOR_RESET

export COLOR_YELLOW

}

function common::get_packaging_tool() {

: "${AIRFLOW_USE_UV:?Should be set}"

## IMPORTANT: IF YOU MODIFY THIS FUNCTION YOU SHOULD ALSO MODIFY CORRESPONDING FUNCTION IN

## `scripts/in_container/_in_container_utils.sh`

if [[ ${AIRFLOW_USE_UV} == "true" ]]; then

echo

echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}"

echo

export PACKAGING_TOOL="uv"

export PACKAGING_TOOL_CMD="uv pip"

# --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2

# (binary lxml embeds its own libxml2, while xmlsec uses system one).

# See https://bugs.launchpad.net/lxml/+bug/2110068

if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." && -f "./pyproject.toml" ]]; then

# for uv only install dev group when we install from sources

export EXTRA_INSTALL_FLAGS="--group=dev --no-binary lxml --no-binary xmlsec"

else

export EXTRA_INSTALL_FLAGS="--no-binary lxml --no-binary xmlsec"

fi

export EXTRA_UNINSTALL_FLAGS=""

export UPGRADE_TO_HIGHEST_RESOLUTION="--upgrade --resolution highest"

export UPGRADE_IF_NEEDED="--upgrade"

UV_CONCURRENT_DOWNLOADS=$(nproc --all)

export UV_CONCURRENT_DOWNLOADS

if [[ ${INCLUDE_PRE_RELEASE=} == "true" ]]; then

EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --prerelease if-necessary"

fi

else

echo

echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}"

echo

export PACKAGING_TOOL="pip"

export PACKAGING_TOOL_CMD="pip"

# --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2

# (binary lxml embeds its own libxml2, while xmlsec uses system one).

# See https://bugs.launchpad.net/lxml/+bug/2110068

export EXTRA_INSTALL_FLAGS="--root-user-action ignore --no-binary lxml,xmlsec"

export EXTRA_UNINSTALL_FLAGS="--yes"

export UPGRADE_TO_HIGHEST_RESOLUTION="--upgrade --upgrade-strategy eager"

export UPGRADE_IF_NEEDED="--upgrade --upgrade-strategy only-if-needed"

if [[ ${INCLUDE_PRE_RELEASE=} == "true" ]]; then

EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --pre"

fi

fi

}

function common::get_airflow_version_specification() {

if [[ -z ${AIRFLOW_VERSION_SPECIFICATION=}

&& -n ${AIRFLOW_VERSION}

&& ${AIRFLOW_INSTALLATION_METHOD} != "." ]]; then

AIRFLOW_VERSION_SPECIFICATION="==${AIRFLOW_VERSION}"

fi

}

function common::get_constraints_location() {

# When installing from sources without upgrade, generate constraints from uv.lock

if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." && -z "${UPGRADE_RANDOM_INDICATOR_STRING=}" ]]; then

echo

echo "${COLOR_BLUE}Installing from sources with uv.lock - generating constraints from uv.lock${COLOR_RESET}"

echo

uv export --frozen --no-hashes --no-emit-project --no-editable --no-header \

--no-annotate > "${HOME}/constraints.txt" 2>/dev/null || true

return

fi

# auto-detect Airflow-constraint reference and location

if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE=}" ]]; then

if [[ ${AIRFLOW_VERSION} =~ v?2.* || ${AIRFLOW_VERSION} =~ v?3.* ]]; then

AIRFLOW_CONSTRAINTS_REFERENCE=constraints-${AIRFLOW_VERSION}

else

AIRFLOW_CONSTRAINTS_REFERENCE=${DEFAULT_CONSTRAINTS_BRANCH}

fi

fi

if [[ -z ${AIRFLOW_CONSTRAINTS_LOCATION=} ]]; then

local constraints_base="https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${AIRFLOW_CONSTRAINTS_REFERENCE}"

local python_version

python_version=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')

AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS_MODE}-${python_version}.txt"

fi

if [[ ${AIRFLOW_CONSTRAINTS_LOCATION} =~ http.* ]]; then

echo

echo "${COLOR_BLUE}Downloading constraints from ${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}"

echo

if ! curl -sSf -o "${HOME}/constraints.txt" "${AIRFLOW_CONSTRAINTS_LOCATION}"; then

echo

echo "${COLOR_YELLOW}Constraints file not found at ${AIRFLOW_CONSTRAINTS_LOCATION} (new Python version being bootstrapped?).${COLOR_RESET}"

echo "${COLOR_YELLOW}Falling back to no-constraints installation.${COLOR_RESET}"

echo

AIRFLOW_CONSTRAINTS_LOCATION=""

# Create an empty constraints file so --constraint flag still works

touch "${HOME}/constraints.txt"

fi

else

echo

echo "${COLOR_BLUE}Copying constraints from ${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}"

echo

cp "${AIRFLOW_CONSTRAINTS_LOCATION}" "${HOME}/constraints.txt"

fi

}

function common::show_packaging_tool_version_and_location() {

echo "PATH=${PATH}"

echo "Installed pip: $(pip --version): $(which pip)"

if [[ ${PACKAGING_TOOL} == "pip" ]]; then

echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}"

else

echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}"

echo "Installed uv: $(uv --version 2>/dev/null || echo "Not installed yet"): $(which uv 2>/dev/null)"

fi

}

function common::install_packaging_tools() {

: "${AIRFLOW_USE_UV:?Should be set}"

if [[ "${VIRTUAL_ENV=}" != "" ]]; then

echo

echo "${COLOR_BLUE}Checking packaging tools in venv: ${VIRTUAL_ENV}${COLOR_RESET}"

echo

else

echo

echo "${COLOR_BLUE}Checking packaging tools for system Python installation: $(which python)${COLOR_RESET}"

echo

fi

if [[ ${AIRFLOW_PIP_VERSION=} == "" ]]; then

echo

echo "${COLOR_BLUE}Installing latest pip version${COLOR_RESET}"

echo

pip install --root-user-action ignore --disable-pip-version-check --upgrade pip

elif [[ ! ${AIRFLOW_PIP_VERSION} =~ ^[0-9].* ]]; then

echo

echo "${COLOR_BLUE}Installing pip version from spec ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"

echo

# shellcheck disable=SC2086

pip install --root-user-action ignore --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}"

else

local installed_pip_version

installed_pip_version=$(python -c 'from importlib.metadata import version; print(version("pip"))')

if [[ ${installed_pip_version} != "${AIRFLOW_PIP_VERSION}" ]]; then

echo

echo "${COLOR_BLUE}(Re)Installing pip version: ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"

echo

pip install --root-user-action ignore --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"

fi

fi

if [[ ${AIRFLOW_UV_VERSION=} == "" ]]; then

echo

echo "${COLOR_BLUE}Installing latest uv version${COLOR_RESET}"

echo

pip install --root-user-action ignore --disable-pip-version-check --upgrade uv

elif [[ ! ${AIRFLOW_UV_VERSION} =~ ^[0-9].* ]]; then

echo

echo "${COLOR_BLUE}Installing uv version from spec ${AIRFLOW_UV_VERSION}${COLOR_RESET}"

echo

# shellcheck disable=SC2086

pip install --root-user-action ignore --disable-pip-version-check "uv @ ${AIRFLOW_UV_VERSION}"

else

local installed_uv_version

installed_uv_version=$(python -c 'from importlib.metadata import version; print(version("uv"))' 2>/dev/null || echo "Not installed yet")

if [[ ${installed_uv_version} != "${AIRFLOW_UV_VERSION}" ]]; then

echo

echo "${COLOR_BLUE}(Re)Installing uv version: ${AIRFLOW_UV_VERSION}${COLOR_RESET}"

echo

# shellcheck disable=SC2086

pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}"

fi

fi

if [[ ${AIRFLOW_PREK_VERSION=} == "" ]]; then

echo

echo "${COLOR_BLUE}Installing latest prek, uv${COLOR_RESET}"

echo

uv tool install prek --with uv

# make sure that the venv/user in .local exists

mkdir -p "${HOME}/.local/bin"

else

echo

echo "${COLOR_BLUE}Installing predefined versions of prek, uv:${COLOR_RESET}"

echo "${COLOR_BLUE}prek(${AIRFLOW_PREK_VERSION}) uv(${AIRFLOW_UV_VERSION})${COLOR_RESET}"

echo

uv tool install "prek==${AIRFLOW_PREK_VERSION}" --with "uv==${AIRFLOW_UV_VERSION}"

# make sure that the venv/user in .local exists

mkdir -p "${HOME}/.local/bin"

fi

}

function common::import_trusted_gpg() {

common::get_colors

local key=${1:?${COLOR_RED}First argument expects OpenPGP Key ID${COLOR_RESET}}

local name=${2:?${COLOR_RED}Second argument expected trust storage name${COLOR_RESET}}

local key_file="/scripts/docker/keys/${name}.asc"

echo "${COLOR_BLUE}Installing GPG public key ${key} from ${key_file}${COLOR_RESET}"

gpg --dearmor < "${key_file}" > "/etc/apt/trusted.gpg.d/${name}.gpg"

}

EOF

# The content below is automatically copied from scripts/docker/install_airflow_when_building_images.sh

COPY <<"EOF" /install_airflow_when_building_images.sh

#!/usr/bin/env bash

. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"

function install_from_sources() {

local extra_sync_flags

extra_sync_flags=""

if [[ ${VIRTUAL_ENV=} != "" ]]; then

extra_sync_flags="--active"

fi

if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then

if [[ ${PACKAGING_TOOL_CMD} == "pip" ]]; then

set +x

echo

echo "${COLOR_RED}We only support uv not pip installation for upgrading dependencies!.${COLOR_RESET}"

echo

exit 1

fi

set +x

echo

echo "${COLOR_BLUE}Attempting to upgrade all packages to highest versions.${COLOR_RESET}"

echo

# --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2

# (binary lxml embeds its own libxml2, while xmlsec uses system one).

# See https://bugs.launchpad.net/lxml/+bug/2110068

set -x

uv sync --all-packages --resolution highest --group ci-image \

${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \

--no-python-downloads --no-managed-python

else

set +x

echo

echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"