test-complete.yml

name: Complete Test Suite

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

env:
  CARGO_TERM_COLOR: always

jobs:
  # Rust tests on multiple platforms
  rust-tests:
    name: Rust Tests (${{ matrix.os }})
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
    runs-on: ${{ matrix.os }}
    
    steps:
    - uses: actions/checkout@v4
    
    - name: Set up Rust
      uses: actions-rs/toolchain@v1
      with:
        toolchain: stable
        override: true
        components: rustfmt, clippy
    
    - name: Cache cargo registry
      uses: actions/cache@v4
      with:
        path: |
          ~/.cargo/registry
          ~/.cargo/git
        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
        restore-keys: |
          ${{ runner.os }}-cargo-
    
    - name: Cache cargo build
      uses: actions/cache@v4
      with:
        path: target
        key: ${{ runner.os }}-cargo-build-${{ hashFiles('**/Cargo.lock') }}
        restore-keys: |
          ${{ runner.os }}-cargo-build-
    
    - name: Build
      run: cargo build --release --verbose
    
    - name: Run Rust tests
      run: cargo test --verbose
    
    - name: Run clippy
      run: cargo clippy -- -W warnings
    
    - name: Check formatting
      run: cargo fmt -- --check

  # Python tests only on Linux (where uv is readily available)
  python-tests:
    name: Python Tests (Linux)
    runs-on: ubuntu-latest
    needs: rust-tests  # Run after Rust tests to ensure binary builds
    
    steps:
    - uses: actions/checkout@v4
    
    - name: Set up Python
      id: setup-python
      uses: actions/setup-python@v5
      with:
        python-version: '3.10'

    - name: Install uv
      run: |
        curl -LsSf https://astral.sh/uv/install.sh | sh
        echo "$HOME/.cargo/bin" >> $GITHUB_PATH

    - name: Set up Rust (for building sql-cli)
      uses: actions-rs/toolchain@v1
      with:
        toolchain: stable
        override: true
    
    - name: Cache cargo build
      uses: actions/cache@v4
      with:
        path: target
        key: ${{ runner.os }}-cargo-build-python-${{ hashFiles('**/Cargo.lock') }}
        restore-keys: |
          ${{ runner.os }}-cargo-build-python-
          ${{ runner.os }}-cargo-build-
    
    - name: Build SQL CLI
      run: cargo build --release
    
    - name: Cache Python dependencies
      uses: actions/cache@v4
      with:
        path: .venv
        # The interpreter version MUST be in the key. A .venv is a set of
        # symlinks into the hosted toolcache, so a venv built against
        # 3.10.<old> is broken the moment the runner image ships 3.10.<new>
        # and deletes the old directory. `python-version: '3.10'` floats across
        # patch releases, so this happens without any change on our side.
        key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml', 'uv.lock') }}
        restore-keys: |
          ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-

    - name: Install Python dependencies
      run: |
        # Recreate unless the restored .venv has a working interpreter. Testing
        # `-d .venv` is not enough: a cache hit against a since-removed patch
        # release leaves the directory present with a dangling
        # .venv/bin/python symlink, and uv then fails with "Broken symlink"
        # rather than repairing it. `-x` follows the symlink, so it is false
        # exactly when the target is gone.
        [ -x .venv/bin/python ] || uv venv --clear
        uv pip install -e .
        uv pip install pytest pytest-cov pandas numpy
    
    - name: Generate test data
      run: |
        if [ ! -f "data/test_simple_math.csv" ] || [ ! -f "data/test_simple_strings.csv" ]; then
          uv run python scripts/generate_simple_test.py
        fi
    
    - name: Run Python tests
      run: |
        uv run pytest tests/python_tests/ -v --tb=short
    
    - name: Upload test results
      if: failure()
      uses: actions/upload-artifact@v4
      with:
        name: python-test-results
        path: |
          .pytest_cache/
          **/*.log

  # Examples test suite (formal and smoke tests)
  examples-tests:
    name: Examples Test Suite (Linux)
    runs-on: ubuntu-latest
    needs: rust-tests  # Run after Rust tests to ensure binary builds

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.10'

    - name: Install uv
      run: |
        curl -LsSf https://astral.sh/uv/install.sh | sh
        echo "$HOME/.cargo/bin" >> $GITHUB_PATH

    - name: Set up Rust (for building sql-cli)
      uses: actions-rs/toolchain@v1
      with:
        toolchain: stable
        override: true

    - name: Cache cargo build
      uses: actions/cache@v4
      with:
        path: target
        key: ${{ runner.os }}-cargo-build-examples-${{ hashFiles('**/Cargo.lock') }}
        restore-keys: |
          ${{ runner.os }}-cargo-build-examples-
          ${{ runner.os }}-cargo-build-

    - name: Build SQL CLI
      run: cargo build --release

    # data/test_simple_*.csv are generated, not committed (.gitignore's
    # `test_*.csv` rule covers them), so several examples that pass locally
    # fail here without this. Mirrors the same step in the Python Tests job.
    - name: Generate test data
      run: |
        if [ ! -f "data/test_simple_math.csv" ] || [ ! -f "data/test_simple_strings.csv" ]; then
          uv run python scripts/generate_simple_test.py
        fi

    - name: Run examples test suite
      run: |
        uv run python tests/integration/test_examples.py

    - name: Upload test results
      if: failure()
      uses: actions/upload-artifact@v4
      with:
        name: examples-test-results
        path: |
          examples/expectations/*.json
          **/*.log

  # SQL parity regression gate (differential test vs DuckDB)
  parity-tests:
    name: SQL Parity (vs DuckDB)
    runs-on: ubuntu-latest
    needs: rust-tests  # Run after Rust tests to ensure binary builds

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.12'  # harness uses stdlib tomllib (3.11+)

    - name: Install uv
      run: |
        curl -LsSf https://astral.sh/uv/install.sh | sh
        echo "$HOME/.cargo/bin" >> $GITHUB_PATH

    - name: Set up Rust (for building sql-cli)
      uses: actions-rs/toolchain@v1
      with:
        toolchain: stable
        override: true

    - name: Cache cargo build
      uses: actions/cache@v4
      with:
        path: target
        key: ${{ runner.os }}-cargo-build-parity-${{ hashFiles('**/Cargo.lock') }}
        restore-keys: |
          ${{ runner.os }}-cargo-build-parity-
          ${{ runner.os }}-cargo-build-

    - name: Build SQL CLI
      run: cargo build --release

    - name: Install Python dependencies
      # Resolves the pinned DuckDB from pyproject.toml [dependency-groups].test
      # rather than installing whatever is latest. The corpus contains cases
      # where the SQL standard leaves behaviour implementation-defined and we
      # follow the reference engine by policy, so those cases encode DuckDB's
      # current choices. An unpinned bump could redden this gate for reasons
      # unrelated to our engine — and point at the wrong place when it did.
      # Bump the pin in pyproject.toml deliberately and review any drift.
      run: uv sync --group test

    - name: Run parity check (sql-cli vs DuckDB)
      # Fails if any case drifts from its expected bucket: a regressed AGREE, a
      # silently-closed gap, or a new un-annotated non-AGREE case. See
      # docs/SQL_PARITY.md for the contract.
      run: uv run --group test python tests/comparison/runner.py --check

    - name: Upload parity report
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: parity-report
        path: tests/comparison/reports/

  # Integration test combining both
  integration-test:
    name: Integration Test (Linux)
    runs-on: ubuntu-latest
    needs: [rust-tests, python-tests, examples-tests]

    steps:
    - uses: actions/checkout@v4

    - name: Set up Rust
      uses: actions-rs/toolchain@v1
      with:
        toolchain: stable
        override: true

    - name: Build SQL CLI
      run: cargo build --release

    - name: Test CLI directly
      run: |
        # Generate test data
        echo "id,value" > test.csv
        echo "1,10" >> test.csv
        echo "2,20" >> test.csv
        echo "3,30" >> test.csv

        # Test basic query
        ./target/release/sql-cli test.csv -q "SELECT * FROM test WHERE value > 15" -o csv

        # Test with string methods
        echo "id,name" > names.csv
        echo "1,  Alice  " >> names.csv
        echo "2,Bob" >> names.csv

        ./target/release/sql-cli names.csv -q "SELECT id, name.Trim() as trimmed FROM names" -o csv

    - name: Verify --query-plan works
      run: |
        ./target/release/sql-cli test.csv -q "SELECT id FROM test WHERE value > 15" --query-plan

  # Performance benchmarking (Linux only, non-blocking)
  performance-benchmark:
    name: Performance Benchmarks (Linux)
    runs-on: ubuntu-latest
    needs: rust-tests  # Run after rust tests to get the binary
    continue-on-error: true  # Don't fail the build if benchmarks have issues

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.10'

    - name: Install uv
      run: |
        curl -LsSf https://astral.sh/uv/install.sh | sh
        echo "$HOME/.cargo/bin" >> $GITHUB_PATH

    - name: Set up Rust (for building sql-cli)
      uses: actions-rs/toolchain@v1
      with:
        toolchain: stable
        override: true

    - name: Cache cargo build
      uses: actions/cache@v4
      with:
        path: target
        key: ${{ runner.os }}-cargo-build-benchmark-${{ hashFiles('**/Cargo.lock') }}
        restore-keys: |
          ${{ runner.os }}-cargo-build-benchmark-
          ${{ runner.os }}-cargo-build-

    - name: Build SQL CLI (Release with optimizations)
      run: cargo build --release

    - name: Install Python dependencies
      run: |
        # Recreate unless an existing .venv has a working interpreter — see the
        # Python Tests job for why `-d` is the wrong test. This job caches no
        # .venv, so it cannot hit the stale-symlink case today; kept identical
        # so the two don't drift into different failure modes.
        [ -x .venv/bin/python ] || uv venv --clear
        uv pip install pandas numpy

    - name: Run performance benchmarks
      run: |
        echo "========================================="
        echo "Performance Benchmark Results"
        echo "========================================="
        uv run python scripts/benchmark_all.py || echo "Benchmark completed with warnings"

    - name: Upload benchmark results
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: benchmark-results
        path: |
          benchmarks/*.txt
          benchmarks/*.csv
        retention-days: 30