diff --git a/tools/install.sh b/tools/install.sh index 4fcd3db04..70fc516f3 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -445,7 +445,7 @@ mark_explicit INSTALL_DIR mark_explicit VENV_DIR FLINK_VERSION="${FLINK_VERSION:-2.2.1}" -FLINK_AGENTS_VERSION="${FLINK_AGENTS_VERSION:-0.2.1}" +FLINK_AGENTS_VERSION="${FLINK_AGENTS_VERSION:-0.3.0}" FLINK_SCALA_VERSION="${FLINK_SCALA_VERSION:-2.12}" FLINK_BASE_URL="${FLINK_BASE_URL:-https://dlcdn.apache.org/flink}" # Flink Agents JARs live next to Flink on the ASF mirror network. Override @@ -460,10 +460,10 @@ FLINK_RECOMMENDED_VERSION="2.2.1" # Mirrors https://flink.apache.org/downloads/#apache-flink-agents # (latest first). Note: 0.1.x only ships JARs for Flink 1.20, while 0.2.x -# ships JARs for Flink 1.20 / 2.0 / 2.1 / 2.2 — see the download page for -# the exact compatibility matrix. -FLINK_AGENTS_SUPPORTED_VERSIONS=("0.2.1" "0.2.0" "0.1.1" "0.1.0") -FLINK_AGENTS_RECOMMENDED_VERSION="0.2.1" +# and 0.3.x ship JARs for Flink 1.20 / 2.0 / 2.1 / 2.2 — see the download +# page for the exact compatibility matrix. +FLINK_AGENTS_SUPPORTED_VERSIONS=("0.3.0" "0.2.1" "0.2.0" "0.1.1" "0.1.0") +FLINK_AGENTS_RECOMMENDED_VERSION="0.3.0" INSTALL_FLINK="${INSTALL_FLINK:-Ask}" ENABLE_PYFLINK="${ENABLE_PYFLINK:-Ask}" @@ -475,6 +475,9 @@ VERBOSE="${FLINK_AGENTS_VERBOSE:-0}" DRY_RUN="${FLINK_AGENTS_DRY_RUN:-0}" HELP=0 PYFLINK_ACTUALLY_ENABLED=0 +# Set to 1 when a version edit leaves an existing venv on an incompatible +# interpreter and the user opts to recreate it; setup_python_env honors it. +RECREATE_VENV=0 print_usage() { cat < "$out" } @@ -1111,6 +1115,7 @@ edit_plan_interactive() { case "$action" in flink_agents_version) prompt_flink_agents_version_interactive || true + revalidate_python_constraint ;; install_flink) if choose_install_method_interactive "Install Flink?"; then @@ -1133,11 +1138,13 @@ edit_plan_interactive() { detect_flink_version_from_home || ui_warn "Could not auto-detect Flink version; keeping ${FLINK_VERSION}" fi FLINK_MAJOR_MINOR="$(flink_major_minor "$FLINK_VERSION")" + revalidate_python_constraint ;; flink_version) prompt_flink_version_interactive || true FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}" FLINK_MAJOR_MINOR="$(flink_major_minor "$FLINK_VERSION")" + revalidate_python_constraint ;; install_dir) INSTALL_DIR="$(prompt_path_choice_interactive \ @@ -1404,10 +1411,25 @@ pip_install_quiet() { } setup_python_env() { + # A venv the user opted to recreate (incompatible interpreter, see + # revalidate_existing_venv) is deleted here, not at plan time, so a later + # cancel never destroys it. Guard the rm to a real venv marker. + if [[ "${RECREATE_VENV:-0}" == "1" && -f "$VENV_DIR/pyvenv.cfg" ]]; then + ui_info "Recreating virtual environment: $VENV_DIR" + rm -rf "$VENV_DIR" + fi + if [[ ! -d "$VENV_DIR" ]]; then ui_info "Creating virtual environment: $VENV_DIR" create_venv else + # Reused venvs keep their own interpreter; make sure it still fits the + # selected versions before pip runs under it (belt-and-suspenders for + # any path that bypassed the plan-time revalidation, e.g. an explicit + # VENV_DIR in non-interactive mode). + if [[ -x "$VENV_DIR/bin/python" ]] && ! validate_python_bin "$VENV_DIR/bin/python"; then + die "Existing venv ${VENV_DIR} uses an interpreter incompatible with Flink Agents ${FLINK_AGENTS_VERSION} + Flink ${FLINK_VERSION} (needs >=3.10 and <3.$(python_minor_ceiling)). Remove it or set VENV_DIR to a different path." + fi ui_info "Reusing existing virtual environment: $VENV_DIR" fi @@ -1563,6 +1585,38 @@ check_java() { return 0 } +# Upper bound (exclusive) on the Python minor version, derived from BOTH +# selected versions and returning the stricter (lower) of the two: +# - Flink Agents: 0.1.x / 0.2.x publish requires-python >=3.10,<3.12, +# while 0.3.0+ publishes >=3.10,<3.13 (adds Python 3.12 support). +# - Apache Flink: Python 3.12 requires Flink 2.1+; Flink <2.1 caps at <3.12. +# An unparseable FLINK_VERSION leaves the Flink axis unrestricted (fail open), +# so only the Flink Agents ceiling applies in that case. +python_minor_ceiling() { + local agents_ceiling flink_ceiling + case "$FLINK_AGENTS_VERSION" in + 0.1.*|0.2.*) agents_ceiling=12 ;; + *) agents_ceiling=13 ;; + esac + + flink_ceiling=13 + local mm major minor + mm="$(flink_major_minor "${FLINK_VERSION:-}")" + if [[ -n "$mm" ]]; then + major="${mm%%.*}" + minor="${mm##*.}" + if (( major < 2 || (major == 2 && minor < 1) )); then + flink_ceiling=12 + fi + fi + + if (( agents_ceiling < flink_ceiling )); then + printf '%s' "$agents_ceiling" + else + printf '%s' "$flink_ceiling" + fi +} + validate_python_bin() { local bin="$1" [[ -n "$bin" ]] || return 1 @@ -1576,7 +1630,7 @@ validate_python_bin() { py_major="${py_version_output%%.*}" py_minor="${py_version_output##*.}" - if [[ "$py_major" -ne 3 ]] || [[ "$py_minor" -lt 10 ]] || [[ "$py_minor" -ge 12 ]]; then + if [[ "$py_major" -ne 3 ]] || [[ "$py_minor" -lt 10 ]] || [[ "$py_minor" -ge "$(python_minor_ceiling)" ]]; then return 1 fi return 0 @@ -1588,7 +1642,7 @@ resolve_python() { ui_success "Using Python: $PYTHON_BIN" return 0 fi - die "PYTHON_BIN is invalid or unsupported (need >=3.10 and <3.12): $PYTHON_BIN" + die "PYTHON_BIN is invalid or unsupported (need >=3.10 and <3.$(python_minor_ceiling)): $PYTHON_BIN" fi if validate_python_bin python3; then @@ -1600,7 +1654,7 @@ resolve_python() { fi if command -v python3 >/dev/null 2>&1; then - ui_warn "python3 on PATH is incompatible (Flink Agents requires Python >=3.10 and <3.12)" + ui_warn "python3 on PATH is incompatible (Flink Agents ${FLINK_AGENTS_VERSION} requires Python >=3.10 and <3.$(python_minor_ceiling))" else ui_warn "python3 not found on PATH" fi @@ -1615,13 +1669,74 @@ resolve_python() { die "No Python interpreter provided." fi if ! validate_python_bin "$input"; then - die "Provided Python is invalid or unsupported (need >=3.10 and <3.12): $input" + die "Provided Python is invalid or unsupported (need >=3.10 and <3.$(python_minor_ceiling)): $input" fi PYTHON_BIN="$input" ui_success "Using Python: $PYTHON_BIN" return 0 } +# The Python ceiling depends on BOTH the Flink Agents and the Flink version +# (see python_minor_ceiling), so editing either at the confirm screen can +# invalidate a Python setup that was already accepted — e.g. Python 3.12 +# passes for 0.3.0 + Flink 2.2 but not after editing down to 0.2.1 or to +# Flink 2.0. Called after a version edit; mirrors the enable_pyflink arm. +# No-op unless PyFlink is enabled. Two independent checks: +# 1. PYTHON_BIN — the resolved interpreter; re-resolve if it no longer fits. +# 2. VENV_DIR — see revalidate_existing_venv; a reused venv keeps its own +# interpreter, which re-resolving PYTHON_BIN does not change. +revalidate_python_constraint() { + if [[ "$ENABLE_PYFLINK" != "Yes" ]]; then + return 0 + fi + + if [[ -n "${PYTHON_BIN:-}" ]] && ! validate_python_bin "$PYTHON_BIN"; then + ui_warn "The selected Python (${PYTHON_BIN}) is incompatible with Flink Agents ${FLINK_AGENTS_VERSION} + Flink ${FLINK_VERSION} (requires Python >=3.10 and <3.$(python_minor_ceiling))" + PYTHON_BIN="" + resolve_python + fi + + revalidate_existing_venv +} + +# setup_python_env reuses an existing venv as-is (it only creates one when +# VENV_DIR is absent), so the venv's own interpreter — not the re-resolved +# PYTHON_BIN — is what pip runs under. After a version edit that lowers the +# Python ceiling, an existing venv built on a now-too-new interpreter would +# still be reused and fail at `pip install` time. Catch it at plan time: +# offer to recreate it (deferred to setup_python_env via RECREATE_VENV, which +# only ever deletes a directory carrying a pyvenv.cfg marker) or point at a +# different path. Non-interactive callers get an actionable error instead of a +# mid-install failure. No-op unless VENV_DIR is an existing venv. +revalidate_existing_venv() { + RECREATE_VENV=0 + [[ "$ENABLE_PYFLINK" == "Yes" && -n "${VENV_DIR:-}" ]] || return 0 + + local kind + kind="$(validate_venv_dir "$VENV_DIR")" || true + [[ "$kind" == "venv" ]] || return 0 + + if validate_python_bin "$VENV_DIR/bin/python"; then + return 0 + fi + + local venv_pv + venv_pv="$("$VENV_DIR/bin/python" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || printf 'unknown')" + ui_warn "Existing venv ${VENV_DIR} uses Python ${venv_pv}, incompatible with Flink Agents ${FLINK_AGENTS_VERSION} + Flink ${FLINK_VERSION} (needs >=3.10 and <3.$(python_minor_ceiling))" + + if ! is_promptable; then + die "Recreate ${VENV_DIR} or set VENV_DIR to a different path (its interpreter ${venv_pv} is unsupported for this version selection)." + fi + + if choose_install_method_interactive "Recreate ${VENV_DIR} with a compatible interpreter?"; then + RECREATE_VENV=1 + else + prompt_and_validate_venv_dir "$VENV_DIR" + # The newly chosen path may itself be an incompatible venv. + revalidate_existing_venv + fi +} + show_install_plan() { ui_section "Environment (read-only)" ui_kv "OS" "$OS" diff --git a/tools/test/helpers/load.bash b/tools/test/helpers/load.bash index 043a4fd67..e2cae738d 100644 --- a/tools/test/helpers/load.bash +++ b/tools/test/helpers/load.bash @@ -28,8 +28,9 @@ reset_install_sh_state() { ENABLE_PYFLINK="Ask" PYTHON_BIN="" PYFLINK_ACTUALLY_ENABLED=0 + RECREATE_VENV=0 FLINK_VERSION="2.2.0" - FLINK_AGENTS_VERSION="0.2.1" + FLINK_AGENTS_VERSION="0.3.0" FLINK_SCALA_VERSION="2.12" FLINK_BASE_URL="https://dlcdn.apache.org/flink" FLINK_SUPPORTED_VERSIONS=("2.2.0" "2.1.1" "2.0.1" "1.20.3") @@ -44,6 +45,6 @@ reset_install_sh_state() { FLINK_AGENTS_VERSION_EXPLICIT=0 INSTALL_DIR_EXPLICIT=0 VENV_DIR_EXPLICIT=0 - FLINK_AGENTS_SUPPORTED_VERSIONS=("0.2.1" "0.2.0" "0.1.1" "0.1.0") - FLINK_AGENTS_RECOMMENDED_VERSION="0.2.1" + FLINK_AGENTS_SUPPORTED_VERSIONS=("0.3.0" "0.2.1" "0.2.0" "0.1.1" "0.1.0") + FLINK_AGENTS_RECOMMENDED_VERSION="0.3.0" } diff --git a/tools/test/integration/dry_run_extra.bats b/tools/test/integration/dry_run_extra.bats index 1e7efac05..105ccf4c0 100644 --- a/tools/test/integration/dry_run_extra.bats +++ b/tools/test/integration/dry_run_extra.bats @@ -19,7 +19,7 @@ esac @test "dry-run: plan shows Flink Agents version (review #1 — JARs are implicit)" { run bash "${BATS_TEST_DIRNAME}/../../install.sh" --dry-run --non-interactive [ "$status" -eq 0 ] - case "$output" in *"Flink Agents version"*"0.2.1"*) ;; *) false ;; esac + case "$output" in *"Flink Agents version"*"0.3.0"*) ;; *) false ;; esac } @test "dry-run: existing FLINK_HOME — plan shows detected version, not default (review #2)" { diff --git a/tools/test/unit/revalidate_python_constraint.bats b/tools/test/unit/revalidate_python_constraint.bats new file mode 100644 index 000000000..354df87a6 --- /dev/null +++ b/tools/test/unit/revalidate_python_constraint.bats @@ -0,0 +1,175 @@ +#!/usr/bin/env bats + +setup() { + load '../helpers/load' + load '../helpers/shim' + load_install_sh + reset_install_sh_state + shim_setup + # Default the venv to an absent path so revalidate_existing_venv is a + # no-op unless a test builds a real venv on purpose. + VENV_DIR="$BATS_TEST_TMPDIR/no-venv" +} + +# Helper: write a fake python that reports a chosen version. +fake_python() { + local name="$1" + local ver="$2" + shim_bin_script "$name" " +case \"\$*\" in + *'sys.version_info.major'*) + echo '$ver' + ;; +esac +" +} + +# Helper: build a fake existing venv whose interpreter reports $ver. +fake_venv() { + local dir="$1" + local ver="$2" + mkdir -p "$dir/bin" + : > "$dir/pyvenv.cfg" + : > "$dir/bin/activate" + cat > "$dir/bin/python" <