Skip to content

[tools] Update install.sh for Flink Agents 0.3.0 and Python 3.12 #888

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wenjin272 merged 4 commits into from
Jul 14, 2026
Merged

[tools] Update install.sh for Flink Agents 0.3.0 and Python 3.12 #888

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
69d0995
[tools] Update install.sh for Flink Agents 0.3.0 and Python 3.12
Olawoyin007 Jul 9, 2026
c4f81b4
[tools] Re-validate PYTHON_BIN when the Flink Agents version is edited
Olawoyin007 Jul 10, 2026
ece24ba
[tools] Sync test fixture version defaults with install.sh
Olawoyin007 Jul 10, 2026
1839dcd
[tools] Make Python compatibility validation aware of the Flink versi…
Olawoyin007 Jul 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 124 additions & 9 deletions tools/install.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -445,7 +445,7 @@ mark_explicit INSTALL_DIR
mark_explicit VENV_DIR

FLINK_VERSION="${FLINK_VERSION:-2.2.1}"
FLINK_AGENTS_VERSION="${FLINK_AGENTS_VERSION:-0.2.1}"
FLINK_AGENTS_VERSION="${FLINK_AGENTS_VERSION:-0.3.0}"
FLINK_SCALA_VERSION="${FLINK_SCALA_VERSION:-2.12}"
FLINK_BASE_URL="${FLINK_BASE_URL:-https://dlcdn.apache.org/flink}"
# Flink Agents JARs live next to Flink on the ASF mirror network. Override
Expand All @@ -460,10 +460,10 @@ FLINK_RECOMMENDED_VERSION="2.2.1"

# Mirrors https://flink.apache.org/downloads/#apache-flink-agents
# (latest first). Note: 0.1.x only ships JARs for Flink 1.20, while 0.2.x
# ships JARs for Flink 1.20 / 2.0 / 2.1 / 2.2 — see the download page for
# the exact compatibility matrix.
FLINK_AGENTS_SUPPORTED_VERSIONS=("0.2.1" "0.2.0" "0.1.1" "0.1.0")
FLINK_AGENTS_RECOMMENDED_VERSION="0.2.1"
# and 0.3.x ship JARs for Flink 1.20 / 2.0 / 2.1 / 2.2 — see the download
# page for the exact compatibility matrix.
FLINK_AGENTS_SUPPORTED_VERSIONS=("0.3.0" "0.2.1" "0.2.0" "0.1.1" "0.1.0")
FLINK_AGENTS_RECOMMENDED_VERSION="0.3.0"

INSTALL_FLINK="${INSTALL_FLINK:-Ask}"
ENABLE_PYFLINK="${ENABLE_PYFLINK:-Ask}"
Expand All @@ -475,6 +475,9 @@ VERBOSE="${FLINK_AGENTS_VERBOSE:-0}"
DRY_RUN="${FLINK_AGENTS_DRY_RUN:-0}"
HELP=0
PYFLINK_ACTUALLY_ENABLED=0
# Set to 1 when a version edit leaves an existing venv on an incompatible
# interpreter and the user opts to recreate it; setup_python_env honors it.
RECREATE_VENV=0

print_usage() {
cat <<EOF
Expand Down Expand Up @@ -1015,6 +1018,7 @@ edit_plan_dump_state() {
printf 'VENV_DIR=%s\n' "$(edit_plan_quote "$VENV_DIR")"
printf 'PYTHON_BIN=%s\n' "$(edit_plan_quote "${PYTHON_BIN:-}")"
printf 'FLINK_AGENTS_VERSION=%s\n' "$(edit_plan_quote "$FLINK_AGENTS_VERSION")"
printf 'RECREATE_VENV=%s\n' "$(edit_plan_quote "${RECREATE_VENV:-0}")"
} > "$out"
}

Expand Down Expand Up @@ -1111,6 +1115,7 @@ edit_plan_interactive() {
case "$action" in
flink_agents_version)
prompt_flink_agents_version_interactive || true
revalidate_python_constraint
;;
install_flink)
if choose_install_method_interactive "Install Flink?"; then
Expand All @@ -1133,11 +1138,13 @@ edit_plan_interactive() {
detect_flink_version_from_home || ui_warn "Could not auto-detect Flink version; keeping ${FLINK_VERSION}"
fi
FLINK_MAJOR_MINOR="$(flink_major_minor "$FLINK_VERSION")"
revalidate_python_constraint
;;
flink_version)
prompt_flink_version_interactive || true
FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}"
FLINK_MAJOR_MINOR="$(flink_major_minor "$FLINK_VERSION")"
revalidate_python_constraint
;;
install_dir)
INSTALL_DIR="$(prompt_path_choice_interactive \
Expand Down Expand Up @@ -1404,10 +1411,25 @@ pip_install_quiet() {
}

setup_python_env() {
# A venv the user opted to recreate (incompatible interpreter, see
# revalidate_existing_venv) is deleted here, not at plan time, so a later
# cancel never destroys it. Guard the rm to a real venv marker.
if [[ "${RECREATE_VENV:-0}" == "1" && -f "$VENV_DIR/pyvenv.cfg" ]]; then
ui_info "Recreating virtual environment: $VENV_DIR"
rm -rf "$VENV_DIR"
fi

if [[ ! -d "$VENV_DIR" ]]; then
ui_info "Creating virtual environment: $VENV_DIR"
create_venv
else
# Reused venvs keep their own interpreter; make sure it still fits the
# selected versions before pip runs under it (belt-and-suspenders for
# any path that bypassed the plan-time revalidation, e.g. an explicit
# VENV_DIR in non-interactive mode).
if [[ -x "$VENV_DIR/bin/python" ]] && ! validate_python_bin "$VENV_DIR/bin/python"; then
die "Existing venv ${VENV_DIR} uses an interpreter incompatible with Flink Agents ${FLINK_AGENTS_VERSION} + Flink ${FLINK_VERSION} (needs >=3.10 and <3.$(python_minor_ceiling)). Remove it or set VENV_DIR to a different path."
fi
ui_info "Reusing existing virtual environment: $VENV_DIR"
fi

Expand Down Expand Up @@ -1563,6 +1585,38 @@ check_java() {
return 0
}

# Upper bound (exclusive) on the Python minor version, derived from BOTH
# selected versions and returning the stricter (lower) of the two:
# - Flink Agents: 0.1.x / 0.2.x publish requires-python >=3.10,<3.12,
# while 0.3.0+ publishes >=3.10,<3.13 (adds Python 3.12 support).
# - Apache Flink: Python 3.12 requires Flink 2.1+; Flink <2.1 caps at <3.12.
# An unparseable FLINK_VERSION leaves the Flink axis unrestricted (fail open),
# so only the Flink Agents ceiling applies in that case.
python_minor_ceiling() {
local agents_ceiling flink_ceiling
case "$FLINK_AGENTS_VERSION" in
0.1.*|0.2.*) agents_ceiling=12 ;;
*) agents_ceiling=13 ;;
esac

@wenjin272 wenjin272 Jul 13, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Python 3.12 compatibility also depends on the selected Flink version, not only the Flink Agents version. The installation docs require Flink 2.1+ for Python 3.12, but 0.3.0 + Flink 1.20/2.0 + Python 3.12 currently passes this validation and only fails later when apache-flink==$FLINK_VERSION is installed.

Could we derive the Python constraint from both versions and revalidate it whenever the Agents version, Flink version, or detected FLINK_HOME changes?

@Olawoyin007 Olawoyin007 Jul 13, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch - fixed in 1839dcd. python_minor_ceiling now derives the ceiling from both versions and returns the stricter of the two: the Flink Agents axis (0.1.x/0.2.x → <3.12, 0.3.0+ → <3.13) and a new Flink axis (Flink <2.1 → <3.12, since Python 3.12 needs Flink 2.1+). The Flink comparison is major-then-minor numeric (so 1.20.x isn't mistaken for ≥2.1) and reuses flink_major_minor, so snapshots/rc/source builds like 2.1-SNAPSHOT resolve correctly; an unparseable FLINK_VERSION leaves the Flink axis unrestricted (fail open). Because validate_python_bin and resolve_python both consult the ceiling, they pick this up automatically.

For revalidation on change: revalidate_python_for_agents_version is renamed revalidate_python_constraint and now also runs on the flink_version and install_flink edit arms (both mutate FLINK_VERSION, and install_flink's No branch re-detects from FLINK_HOME). Your exact case - 0.3.0 + Flink 2.0.x + Python 3.12 - is covered by new bats cases in validate_python_bin.bats and revalidate_python_constraint.bats.

One scoping note: the standalone flink_home edit arm currently only repoints FLINK_HOME and deliberately does not re-detect the Flink version (unlike the install_flink arm), so I left its semantics unchanged rather than widen it here. Happy to add version re-detection to that arm too if you'd prefer it fully symmetric.


flink_ceiling=13
local mm major minor
mm="$(flink_major_minor "${FLINK_VERSION:-}")"
if [[ -n "$mm" ]]; then
major="${mm%%.*}"
minor="${mm##*.}"
if (( major < 2 || (major == 2 && minor < 1) )); then
flink_ceiling=12
fi
fi

if (( agents_ceiling < flink_ceiling )); then
printf '%s' "$agents_ceiling"
else
printf '%s' "$flink_ceiling"
fi
}

validate_python_bin() {
local bin="$1"
[[ -n "$bin" ]] || return 1
Expand All @@ -1576,7 +1630,7 @@ validate_python_bin() {
py_major="${py_version_output%%.*}"
py_minor="${py_version_output##*.}"

if [[ "$py_major" -ne 3 ]] || [[ "$py_minor" -lt 10 ]] || [[ "$py_minor" -ge 12 ]]; then
if [[ "$py_major" -ne 3 ]] || [[ "$py_minor" -lt 10 ]] || [[ "$py_minor" -ge "$(python_minor_ceiling)" ]]; then

@weiqingy weiqingy Jul 10, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This line is where the interpreter check starts depending on the selected release — and once resolve_python has run, the confirm screen can still change that release out from under it. edit_plan_interactive's flink_agents_version) arm reassigns FLINK_AGENTS_VERSION (install.sh:1112-1114 → 693) without re-checking the already-resolved PYTHON_BIN, unlike the enable_pyflink) arm right below it, which re-runs resolve_python when its dependent changes (install.sh:1153-1160). Concretely: a user on Python 3.12 accepts the new 0.3.0 default (validates here, ceiling 13), then at the confirm screen edits the version down to 0.2.1; setup_python_env then runs pip install flink-agents==0.2.1 (install.sh:1434), whose requires-python is <3.12, and the install fails after the plan looked good. Before this PR the interpreter was independent of the release, so this window didn't exist. Would re-validating PYTHON_BIN after a version edit — mirroring the enable_pyflink) arm — catch the mismatch at plan time rather than mid-install?

@wenjin272 wenjin272 Jul 10, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1

@Olawoyin007 Olawoyin007 Jul 10, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch - fixed in c4f81b4. The flink_agents_version) arm now calls a new revalidate_python_for_agents_version after the version prompt: if PyFlink is enabled and the already-resolved PYTHON_BIN no longer satisfies the new release's ceiling, it warns and re-runs resolve_python (same semantics as the enable_pyflink) arm - and since PYTHON_BIN is part of the edit-state dump, the re-resolved interpreter propagates back to the plan). Your exact scenario (3.12 accepted under 0.3.0, then edited down to 0.2.1) is covered by a new bats test, plus four more for the keep/no-op paths.

🤖 Generated with Claude Code

return 1
fi
return 0
Expand All @@ -1588,7 +1642,7 @@ resolve_python() {
ui_success "Using Python: $PYTHON_BIN"
return 0
fi
die "PYTHON_BIN is invalid or unsupported (need >=3.10 and <3.12): $PYTHON_BIN"
die "PYTHON_BIN is invalid or unsupported (need >=3.10 and <3.$(python_minor_ceiling)): $PYTHON_BIN"
fi

if validate_python_bin python3; then
Expand All @@ -1600,7 +1654,7 @@ resolve_python() {
fi

if command -v python3 >/dev/null 2>&1; then
ui_warn "python3 on PATH is incompatible (Flink Agents requires Python >=3.10 and <3.12)"
ui_warn "python3 on PATH is incompatible (Flink Agents ${FLINK_AGENTS_VERSION} requires Python >=3.10 and <3.$(python_minor_ceiling))"
else
ui_warn "python3 not found on PATH"
fi
Expand All @@ -1615,13 +1669,74 @@ resolve_python() {
die "No Python interpreter provided."
fi
if ! validate_python_bin "$input"; then
die "Provided Python is invalid or unsupported (need >=3.10 and <3.12): $input"
die "Provided Python is invalid or unsupported (need >=3.10 and <3.$(python_minor_ceiling)): $input"
fi
PYTHON_BIN="$input"
ui_success "Using Python: $PYTHON_BIN"
return 0
}

# The Python ceiling depends on BOTH the Flink Agents and the Flink version
# (see python_minor_ceiling), so editing either at the confirm screen can
# invalidate a Python setup that was already accepted — e.g. Python 3.12
# passes for 0.3.0 + Flink 2.2 but not after editing down to 0.2.1 or to
# Flink 2.0. Called after a version edit; mirrors the enable_pyflink arm.
# No-op unless PyFlink is enabled. Two independent checks:
# 1. PYTHON_BIN — the resolved interpreter; re-resolve if it no longer fits.
# 2. VENV_DIR — see revalidate_existing_venv; a reused venv keeps its own
# interpreter, which re-resolving PYTHON_BIN does not change.
revalidate_python_constraint() {
if [[ "$ENABLE_PYFLINK" != "Yes" ]]; then
return 0
fi

if [[ -n "${PYTHON_BIN:-}" ]] && ! validate_python_bin "$PYTHON_BIN"; then
ui_warn "The selected Python (${PYTHON_BIN}) is incompatible with Flink Agents ${FLINK_AGENTS_VERSION} + Flink ${FLINK_VERSION} (requires Python >=3.10 and <3.$(python_minor_ceiling))"
PYTHON_BIN=""
resolve_python
fi

revalidate_existing_venv
}

# setup_python_env reuses an existing venv as-is (it only creates one when
# VENV_DIR is absent), so the venv's own interpreter — not the re-resolved
# PYTHON_BIN — is what pip runs under. After a version edit that lowers the
# Python ceiling, an existing venv built on a now-too-new interpreter would
# still be reused and fail at `pip install` time. Catch it at plan time:
# offer to recreate it (deferred to setup_python_env via RECREATE_VENV, which
# only ever deletes a directory carrying a pyvenv.cfg marker) or point at a
# different path. Non-interactive callers get an actionable error instead of a
# mid-install failure. No-op unless VENV_DIR is an existing venv.
revalidate_existing_venv() {
RECREATE_VENV=0
[[ "$ENABLE_PYFLINK" == "Yes" && -n "${VENV_DIR:-}" ]] || return 0

local kind
kind="$(validate_venv_dir "$VENV_DIR")" || true
[[ "$kind" == "venv" ]] || return 0

if validate_python_bin "$VENV_DIR/bin/python"; then
return 0
fi

local venv_pv
venv_pv="$("$VENV_DIR/bin/python" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || printf 'unknown')"
ui_warn "Existing venv ${VENV_DIR} uses Python ${venv_pv}, incompatible with Flink Agents ${FLINK_AGENTS_VERSION} + Flink ${FLINK_VERSION} (needs >=3.10 and <3.$(python_minor_ceiling))"

if ! is_promptable; then
die "Recreate ${VENV_DIR} or set VENV_DIR to a different path (its interpreter ${venv_pv} is unsupported for this version selection)."
fi

if choose_install_method_interactive "Recreate ${VENV_DIR} with a compatible interpreter?"; then
RECREATE_VENV=1
else
prompt_and_validate_venv_dir "$VENV_DIR"
# The newly chosen path may itself be an incompatible venv.
revalidate_existing_venv
fi
}

show_install_plan() {
ui_section "Environment (read-only)"
ui_kv "OS" "$OS"
Expand Down
7 changes: 4 additions & 3 deletions tools/test/helpers/load.bash
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,9 @@ reset_install_sh_state() {
ENABLE_PYFLINK="Ask"
PYTHON_BIN=""
PYFLINK_ACTUALLY_ENABLED=0
RECREATE_VENV=0
FLINK_VERSION="2.2.0"
FLINK_AGENTS_VERSION="0.2.1"
FLINK_AGENTS_VERSION="0.3.0"
FLINK_SCALA_VERSION="2.12"
FLINK_BASE_URL="https://dlcdn.apache.org/flink"
FLINK_SUPPORTED_VERSIONS=("2.2.0" "2.1.1" "2.0.1" "1.20.3")
Expand All @@ -44,6 +45,6 @@ reset_install_sh_state() {
FLINK_AGENTS_VERSION_EXPLICIT=0
INSTALL_DIR_EXPLICIT=0
VENV_DIR_EXPLICIT=0
FLINK_AGENTS_SUPPORTED_VERSIONS=("0.2.1" "0.2.0" "0.1.1" "0.1.0")
FLINK_AGENTS_RECOMMENDED_VERSION="0.2.1"
FLINK_AGENTS_SUPPORTED_VERSIONS=("0.3.0" "0.2.1" "0.2.0" "0.1.1" "0.1.0")
FLINK_AGENTS_RECOMMENDED_VERSION="0.3.0"
}
2 changes: 1 addition & 1 deletion tools/test/integration/dry_run_extra.bats
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ esac
@test "dry-run: plan shows Flink Agents version (review #1 — JARs are implicit)" {
run bash "${BATS_TEST_DIRNAME}/../../install.sh" --dry-run --non-interactive
[ "$status" -eq 0 ]
case "$output" in *"Flink Agents version"*"0.2.1"*) ;; *) false ;; esac
case "$output" in *"Flink Agents version"*"0.3.0"*) ;; *) false ;; esac
}

@test "dry-run: existing FLINK_HOME — plan shows detected version, not default (review #2)" {
Expand Down
Loading
Loading