run-campaign.sh

#!/bin/bash
#
# Drives the full hop-latency grid: chain sizes x resolutions x repetitions.
#
# Resumable. Every completed unit is recorded in the manifest, and re-running skips units
# already marked done - a campaign spanning many hours will be interrupted at some point,
# and nobody should be reconstructing from memory which ones finished.
#
# Two things about this grid are deliberate and load-bearing for the analysis:
#
#   * The chain sizes include the interior points 30, 60, 90 and 120. Without them 149 hops
#     carries most of the leverage in the fit and the linear model is interpolated rather
#     than tested (discussion/hop-latency-statistical-analysis.md sections 5.4 and 8.1).
#   * Units run in a seeded random order, one try at a time, so anything that drifts over a
#     session is not confounded with chain length (section 8.2). The seed is printed and
#     recorded in the manifest, so the order is reproducible and can be used as a covariate.
#
# See discussion/campaign-v2-plan.md for the runbook and discussion/hop-latency-study-plan.md
# for the design.

set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$SCRIPT_DIR"
. "$SCRIPT_DIR/campaign-lib.sh"

WORKER_SIZES="${WORKER_SIZES:-1 2 3 5 10 20 30 50 60 90 100 120 150}"
RESOLUTIONS="${RESOLUTIONS:-480p 720p 1080p}"
TRIES="${TRIES:-5}"
DURATION="${DURATION:-60}"
PIPE_ADDRESSING="${PIPE_ADDRESSING:-private}"
ORDER="${ORDER:-random}"
SEED="${SEED:-20260828}"
UNIT="${UNIT:-try}"
RESULTS_ROOT="${RESULTS_ROOT:-experiment_results}"
MANIFEST="${MANIFEST:-$RESULTS_ROOT/campaign-manifest.csv}"
DRY_RUN="${DRY_RUN:-0}"

usage() {
    cat <<USAGE
Usage: $0 [options]

  --dry-run                 Enumerate the units without launching anything
  --resolutions "480p 720p" Resolutions to cover (default: 480p 720p 1080p)
  --workers "1 2 3"         Chain sizes (default: $WORKER_SIZES)
  --tries N                 Repetitions per cell (default: 5)
  --duration S              Seconds per try (default: 60)
  --pipe-addressing MODE    public | private. 'private' keeps inter-worker hops inside the
                            VPC; it needs the worker AMI rebuilt from current worker/ code
                            (default: public)
  --order MODE              random | ascending. 'ascending' restores the thesis-era order,
                            cheapest cells first (default: random)
  --seed N                  Seed for the random order; recorded in the manifest so the
                            order is reproducible (default: $SEED)
  --unit MODE               try | cell. 'try' interleaves individual tries across chain
                            sizes, which is what removes the ordering confound. 'cell' runs
                            a cell's tries back to back (default: try)
  -h, --help                This message

Environment overrides: WORKER_SIZES, RESOLUTIONS, TRIES, DURATION, PIPE_ADDRESSING, ORDER,
SEED, UNIT, RESULTS_ROOT, MANIFEST, plus everything launch-experiment.sh understands
(WORKER_INSTANCE_TYPE, REGION, ...).

Units already marked 'done' in \$MANIFEST are skipped, so this can be re-run after an
interruption. To force one, delete its row from the manifest and its try directory.
USAGE
}

while [[ $# -gt 0 ]]; do
    case $1 in
        --dry-run) DRY_RUN=1 ;;
        --resolutions) RESOLUTIONS="$2"; shift ;;
        --workers) WORKER_SIZES="$2"; shift ;;
        --tries) TRIES="$2"; shift ;;
        --duration) DURATION="$2"; shift ;;
        --pipe-addressing) PIPE_ADDRESSING="$2"; shift ;;
        --order) ORDER="$2"; shift ;;
        --seed) SEED="$2"; shift ;;
        --unit) UNIT="$2"; shift ;;
        -h|--help) usage; exit 0 ;;
        *) echo "Unknown parameter: $1"; usage; exit 1 ;;
    esac
    shift
done

case "$PIPE_ADDRESSING" in
    public|private) ;;
    *) echo "Error: --pipe-addressing must be public or private (got '$PIPE_ADDRESSING')"; exit 1 ;;
esac
case "$ORDER" in
    random|ascending) ;;
    *) echo "Error: --order must be random or ascending (got '$ORDER')"; exit 1 ;;
esac
case "$UNIT" in
    try|cell) ;;
    *) echo "Error: --unit must be try or cell (got '$UNIT')"; exit 1 ;;
esac

mkdir -p "$RESULTS_ROOT"
manifest_init "$MANIFEST" || exit 1

# ---------------------------------------------------------------------------------------
# Build the unit list
# ---------------------------------------------------------------------------------------
# A resolution with no media file is dropped here rather than mid-campaign, so the announced
# unit count is the number of units that will actually be attempted.
usable_resolutions=""
for res in $RESOLUTIONS; do
    media="client/media/fakevideo-${res}.y4m"
    if [ ! -f "$media" ]; then
        echo "SKIPPING resolution $res entirely: $media not found"
        continue
    fi
    usable_resolutions="$usable_resolutions $res"
done

units=""
for res in $usable_resolutions; do
    for nw in $WORKER_SIZES; do
        if [ "$UNIT" = "cell" ]; then
            units="$units$res $nw all"$'\n'
        else
            for ((t=1; t<=TRIES; t++)); do
                units="$units$res $nw $t"$'\n'
            done
        fi
    done
done
units=$(printf '%s' "$units" | sed '/^$/d')

if [ -z "$units" ]; then
    echo "Nothing to do: no resolution had media."
    exit 0
fi

if [ "$ORDER" = "random" ]; then
    units=$(printf '%s\n' "$units" | shuffle_lines "$SEED")
fi

total=$(printf '%s\n' "$units" | wc -l)
# One master plus N workers per try, which is the quantity that shows up on the bill.
launches=$(printf '%s\n' "$units" | awk -v tries="$TRIES" '{ n = ($3 == "all") ? tries : 1; sum += n * (1 + $2) } END { print sum }')

echo "Campaign: $total units ($(echo $usable_resolutions | wc -w) resolutions x $(echo $WORKER_SIZES | wc -w) chain sizes x $TRIES tries), ${DURATION}s per try"
echo "Unit:     $UNIT   Order: $ORDER (seed $SEED)   Pipe addressing: $PIPE_ADDRESSING"
echo "Manifest: $MANIFEST"
echo "Instance launches if nothing is skipped: $launches"
[ "$DRY_RUN" = "1" ] && echo "*** DRY RUN - nothing will be launched ***"

index=0
completed=0
skipped=0
failed=0
CAMPAIGN_START=$(date +%s)

while read -r res nw try; do
    index=$((index+1))
    if manifest_done "$MANIFEST" "$res" "$nw" "$try"; then
        echo "[$index/$total] SKIP  res=$res workers=$nw try=$try (already done)"
        skipped=$((skipped+1))
        continue
    fi
    echo "[$index/$total] RUN   res=$res workers=$nw try=$try"
    if [ "$DRY_RUN" = "1" ]; then
        continue
    fi
    if ! preflight_credentials; then
        echo
        echo "ABORTING: AWS credentials are not usable (expired MFA session?)."
        echo "Nothing was launched for res=$res workers=$nw try=$try. Refresh the session and"
        echo "re-run; units already marked done in $MANIFEST are skipped."
        exit 2
    fi
    if [ "$try" = "all" ]; then
        n_tries="$TRIES"; try_index=1
    else
        n_tries=1; try_index="$try"
    fi
    started=$(date -u +%Y-%m-%dT%H:%M:%SZ)
    t0=$(date +%s)
    if ./launch-experiment.sh -res "$res" -nw "$nw" -n "$n_tries" -ti "$try_index" \
         -d "$DURATION" -pa "$PIPE_ADDRESSING" \
         -p "$RESULTS_ROOT/$res/${nw}_workers" \
       && [ -f "$RESULTS_ROOT/$res/${nw}_workers/try_$try_index/recordings/fullscreen.mp4" ]; then
        status=done
        completed=$((completed+1))
    else
        status=failed
        failed=$((failed+1))
        # The recording is the deliverable, so its absence is the definition of a failed unit -
        # an exit code alone has been wrong before. A failed row is retried on the next run;
        # a "done" row is skipped for ever, so guessing wrong in this direction loses a cell.
        echo "[$index/$total] FAILED res=$res workers=$nw try=$try - continuing with the next unit"
    fi
    finished=$(date -u +%Y-%m-%dT%H:%M:%SZ)
    manifest_append "$MANIFEST" "$res" "$nw" "$try" "$n_tries" "$DURATION" "$PIPE_ADDRESSING" \
        "$index" "$SEED" "$status" "$started" "$finished" "$(( $(date +%s) - t0 ))"
done <<< "$units"

echo
echo "Campaign finished in $(( ($(date +%s) - CAMPAIGN_START) / 60 )) min: $completed done, $skipped skipped, $failed failed"
[ "$failed" -gt 0 ] && echo "Re-run this script to retry the failed units (successful ones are skipped)."
exit 0