Skip to content

packer clean ups and better logging #2183

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mmlb wants to merge 4 commits into
base: develop
Choose a base branch
from
Open

packer clean ups and better logging #2183

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0511a8d
nix: Drop treefmt.nixfmt.package config
mmlb Jun 2, 2026
8758076
nix: Enable treefmt.shfmt
mmlb Jun 2, 2026
18b57fb
scripts/cleanup: Turn xtrace on
mmlb Jun 4, 2026
74a49bf
bump pg versions to test AMI builds
mmlb Jun 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Dockerfile-multigres
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ COPY docker/pgctld/postgresql.conf.tmpl /etc/pgctld-custom/postgresql.conf.tmpl
# Wrapper: injects --postgres-config-template on every pgctld call AND bridges
# postgres's JSON log file to container stdout via a /proc/1/fd/1 symlink so
# kubelet + Vector can ship it without a sidecar. See docker/pgctld/pgctld-wrapper.
COPY --chmod=755 docker/pgctld/pgctld-wrapper /usr/local/bin/pgctld
COPY --chmod=755 docker/pgctld/pgctld-wrapper.sh /usr/local/bin/pgctld
ENV POSTGRES_CONFIG_TEMPLATE_PATH=/etc/pgctld-custom/postgresql.conf.tmpl

# Strip extensions absent from pg17 vanilla build
Expand Down Expand Up @@ -288,7 +288,7 @@ COPY docker/pgctld/orioledb-postgresql.conf.tmpl /etc/pgctld-custom/orioledb-pos
# Wrapper: injects --postgres-config-template on every pgctld call AND bridges
# postgres's JSON log file to container stdout via a /proc/1/fd/1 symlink so
# kubelet + Vector can ship it without a sidecar. See docker/pgctld/pgctld-wrapper.
COPY --chmod=755 docker/pgctld/pgctld-wrapper /usr/local/bin/pgctld
COPY --chmod=755 docker/pgctld/pgctld-wrapper.sh /usr/local/bin/pgctld
ENV POSTGRES_CONFIG_TEMPLATE_PATH=/etc/pgctld-custom/orioledb-postgresql.conf.tmpl

# Regenerate manifest after orioledb added 00-pre-init.sql to init-scripts/
Expand Down
76 changes: 38 additions & 38 deletions ansible/files/admin_api_scripts/grow_fs.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,14 @@ VOLUME_TYPE=${1:-data}
UBUNTU_VERSION=$(lsb_release -rs)

if pgrep resizefs; then
echo "resize2fs is already running"
exit 1
echo "resize2fs is already running"
exit 1
fi

# install amazon disk utilities if not present on 24.04
if [ "${UBUNTU_VERSION}" = "24.04" ] && ! /usr/bin/dpkg-query -W amazon-ec2-utils >/dev/null 2>&1; then
apt-get update
apt-get install -y amazon-ec2-utils || true
apt-get update
apt-get install -y amazon-ec2-utils || true
fi

# We currently mount 3 possible disks
Expand All @@ -27,15 +27,15 @@ XVDA_DEVICE="/dev/nvme0n1"
XVDH_DEVICE="/dev/nvme1n1"
# Map AWS devices to NVMe for ubuntu 24.04 and later
if [ "${UBUNTU_VERSION}" = "24.04" ] && /usr/bin/dpkg-query -W amazon-ec2-utils >/dev/null 2>&1; then
for nvme_dev in $(lsblk -dprno name,type | grep disk | awk '{print $1}'); do
if [ -b "$nvme_dev" ]; then
mapping=$(ebsnvme-id -b "$nvme_dev" 2>/dev/null)
case "$mapping" in
"xvda"|"/dev/xvda") XVDA_DEVICE="$nvme_dev" ;;
"xvdh"|"/dev/xvdh") XVDH_DEVICE="$nvme_dev" ;;
esac
fi
done
for nvme_dev in $(lsblk -dprno name,type | grep disk | awk '{print $1}'); do
if [ -b "$nvme_dev" ]; then
mapping=$(ebsnvme-id -b "$nvme_dev" 2>/dev/null)
case "$mapping" in
"xvda" | "/dev/xvda") XVDA_DEVICE="$nvme_dev" ;;
"xvdh" | "/dev/xvdh") XVDH_DEVICE="$nvme_dev" ;;
esac
fi
done
fi

echo "Using devices - Root: $XVDA_DEVICE, Data: $XVDH_DEVICE"
Expand All @@ -45,35 +45,35 @@ ROOT_DEVICE_FULL=$(findmnt -no SOURCE /)
ROOT_DEVICE=$(lsblk -no PKNAME "$ROOT_DEVICE_FULL")
ROOT_PARTITION_NUMBER=$(echo "$ROOT_DEVICE_FULL" | sed "s|.*${ROOT_DEVICE}p||")

if ! [[ "$ROOT_PARTITION_NUMBER" =~ ^[0-9]+$ ]]; then
echo "Error: ROOT_PARTITION_NUMBER is not a valid number: $ROOT_PARTITION_NUMBER"
exit 1
if ! [[ $ROOT_PARTITION_NUMBER =~ ^[0-9]+$ ]]; then
echo "Error: ROOT_PARTITION_NUMBER is not a valid number: $ROOT_PARTITION_NUMBER"
exit 1
fi

if [ -b "${XVDH_DEVICE}" ] ; then
if [[ "${VOLUME_TYPE}" == "data" ]]; then
resize2fs "${XVDH_DEVICE}"
if [ -b "${XVDH_DEVICE}" ]; then
if [[ ${VOLUME_TYPE} == "data" ]]; then
resize2fs "${XVDH_DEVICE}"

# Explicitly reserving 100MiB worth of blocks for the data volume
#
# This is owned in $GIT_DIR/ebssurrogate/scripts/surrogate-bootstrap-nix.sh
RESERVED_DATA_VOLUME_BLOCK_COUNT=$((100 * 1024 * 1024 / 4096))
tune2fs -r $RESERVED_DATA_VOLUME_BLOCK_COUNT "${XVDH_DEVICE}"
# Explicitly reserving 100MiB worth of blocks for the data volume
#
# This is owned in $GIT_DIR/ebssurrogate/scripts/surrogate-bootstrap-nix.sh
RESERVED_DATA_VOLUME_BLOCK_COUNT=$((100 * 1024 * 1024 / 4096))
tune2fs -r $RESERVED_DATA_VOLUME_BLOCK_COUNT "${XVDH_DEVICE}"

elif [[ "${VOLUME_TYPE}" == "root" ]] ; then
PLACEHOLDER_FL=/home/ubuntu/50M_PLACEHOLDER
rm -f "${PLACEHOLDER_FL}" || true
growpart "${XVDA_DEVICE}" "${ROOT_PARTITION_NUMBER}"
resize2fs "${XVDA_DEVICE}p${ROOT_PARTITION_NUMBER}"
if [[ ! -f "${PLACEHOLDER_FL}" ]] ; then
fallocate -l50M "${PLACEHOLDER_FL}"
fi
else
echo "Invalid disk specified: ${VOLUME_TYPE}"
exit 1
fi
elif [[ ${VOLUME_TYPE} == "root" ]]; then
PLACEHOLDER_FL=/home/ubuntu/50M_PLACEHOLDER
rm -f "${PLACEHOLDER_FL}" || true
growpart "${XVDA_DEVICE}" "${ROOT_PARTITION_NUMBER}"
resize2fs "${XVDA_DEVICE}p${ROOT_PARTITION_NUMBER}"
if [[ ! -f ${PLACEHOLDER_FL} ]]; then
fallocate -l50M "${PLACEHOLDER_FL}"
fi
else
echo "Invalid disk specified: ${VOLUME_TYPE}"
exit 1
fi
else
growpart "${XVDA_DEVICE}" "${ROOT_PARTITION_NUMBER}"
resize2fs "${XVDA_DEVICE}p${ROOT_PARTITION_NUMBER}"
growpart "${XVDA_DEVICE}" "${ROOT_PARTITION_NUMBER}"
resize2fs "${XVDA_DEVICE}p${ROOT_PARTITION_NUMBER}"
fi
echo "Done resizing disk"
39 changes: 20 additions & 19 deletions ansible/files/admin_api_scripts/manage_readonly_mode.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@ set -euo pipefail
SUBCOMMAND=$1

function set_mode {
MODE=$1
psql -h localhost -U supabase_admin -d postgres -c "ALTER SYSTEM SET default_transaction_read_only to ${MODE};"
psql -h localhost -U supabase_admin -d postgres -c "SELECT pg_reload_conf();"
MODE=$1
psql -h localhost -U supabase_admin -d postgres -c "ALTER SYSTEM SET default_transaction_read_only to ${MODE};"
psql -h localhost -U supabase_admin -d postgres -c "SELECT pg_reload_conf();"
}

function check_override {
COMMAND=$(cat <<EOF
COMMAND=$(
cat <<EOF
WITH role_comment as (
SELECT pg_catalog.shobj_description(r.oid, 'pg_authid') AS content
FROM pg_catalog.pg_roles r
Expand All @@ -25,21 +26,21 @@ SELECT
END as override_active
FROM role_comment;
EOF
)
RESULT=$(psql -h localhost -U supabase_admin -d postgres -At -c "$COMMAND")
echo -n "$RESULT"
)
RESULT=$(psql -h localhost -U supabase_admin -d postgres -At -c "$COMMAND")
echo -n "$RESULT"
}

case $SUBCOMMAND in
"check_override")
check_override
;;
"set")
shift
set_mode "$@"
;;
*)
echo "Error: '$SUBCOMMAND' is not a known subcommand."
exit 1
;;
esac
"check_override")
check_override
;;
"set")
shift
set_mode "$@"
;;
*)
echo "Error: '$SUBCOMMAND' is not a known subcommand."
exit 1
;;
esac
80 changes: 40 additions & 40 deletions ansible/files/admin_api_scripts/mount-volume.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@ set -euo pipefail
DEVICE=${1:-}
MOUNT_POINT=${2:-}

if [[ -z "$DEVICE" || -z "$MOUNT_POINT" ]]; then
echo "Usage: $0 <device> <mount_point>"
echo "Example: sudo ./mount-volume.sh /dev/nvme1n1 /data/150008"
logger "Usage: $0 <device> <mount_point>"
logger "Example: sudo ./mount-volume.sh /dev/nvme1n1 /data/150008"
exit 1
if [[ -z $DEVICE || -z $MOUNT_POINT ]]; then
echo "Usage: $0 <device> <mount_point>"
echo "Example: sudo ./mount-volume.sh /dev/nvme1n1 /data/150008"
logger "Usage: $0 <device> <mount_point>"
logger "Example: sudo ./mount-volume.sh /dev/nvme1n1 /data/150008"
exit 1
fi

OWNER="postgres:postgres"
Expand All @@ -24,66 +24,66 @@ LOGGER_TAG="mount-volume"

# --- Helper function for echo + logger ---
log() {
echo "$1"
logger -t "$LOGGER_TAG" "$1"
echo "$1"
logger -t "$LOGGER_TAG" "$1"
}

log "Starting mount procedure for device $DEVICE → $MOUNT_POINT"

# --- Wait for block device ---
log "Waiting for block device $DEVICE to become available..."
while true; do
if [ -b "$DEVICE" ]; then
if blkid "$DEVICE" >/dev/null 2>&1 || true; then
log "$DEVICE is ready"
break
fi
fi

ELAPSED=$((ELAPSED + INTERVAL))
if [ $ELAPSED -ge $TIMEOUT ]; then
log "Error: $DEVICE did not become ready after $TIMEOUT seconds"
exit 3
fi

sleep $INTERVAL
if [ -b "$DEVICE" ]; then
if blkid "$DEVICE" >/dev/null 2>&1 || true; then
log "$DEVICE is ready"
break
fi
fi

ELAPSED=$((ELAPSED + INTERVAL))
if [ $ELAPSED -ge $TIMEOUT ]; then
log "Error: $DEVICE did not become ready after $TIMEOUT seconds"
exit 3
fi

sleep $INTERVAL
done

# --- Validate device ---
if [ ! -b "$DEVICE" ]; then
log "Error: Block device '$DEVICE' does not exist."
exit 2
log "Error: Block device '$DEVICE' does not exist."
exit 2
fi

# --- Safety: refuse to mount over non-empty directory ---
mkdir -p "$MOUNT_POINT"
if [ "$(ls -A "$MOUNT_POINT" 2>/dev/null)" ]; then
if ! mountpoint -q "$MOUNT_POINT"; then
log "Error: Mount point $MOUNT_POINT is not empty. Aborting to protect existing data."
exit 4
fi
if ! mountpoint -q "$MOUNT_POINT"; then
log "Error: Mount point $MOUNT_POINT is not empty. Aborting to protect existing data."
exit 4
fi
fi

# --- Format if needed ---
if ! blkid "$DEVICE" >/dev/null 2>&1; then
log "Device $DEVICE appears unformatted. Formatting as $FSTYPE..."
mkfs."$FSTYPE" -F "$DEVICE"
log "Device $DEVICE appears unformatted. Formatting as $FSTYPE..."
mkfs."$FSTYPE" -F "$DEVICE"
else
log "$DEVICE already has a filesystem — skipping format."
log "$DEVICE already has a filesystem — skipping format."
fi

# --- Filesystem check ---
if ! mountpoint -q "$MOUNT_POINT"; then
log "Running e2fsck check on $DEVICE"
e2fsck -pf "$DEVICE" || log "Warning: e2fsck returned non-zero exit code"
log "Running e2fsck check on $DEVICE"
e2fsck -pf "$DEVICE" || log "Warning: e2fsck returned non-zero exit code"
fi

# --- Mount ---
if ! mountpoint -q "$MOUNT_POINT"; then
log "Mounting $DEVICE to $MOUNT_POINT"
mount -t "$FSTYPE" -o "$MOUNT_OPTS" "$DEVICE" "$MOUNT_POINT"
log "Mounting $DEVICE to $MOUNT_POINT"
mount -t "$FSTYPE" -o "$MOUNT_OPTS" "$DEVICE" "$MOUNT_POINT"
else
log "$MOUNT_POINT is already mounted"
log "$MOUNT_POINT is already mounted"
fi

# --- Ownership and permissions ---
Expand All @@ -96,10 +96,10 @@ UUID=$(blkid -s UUID -o value "$DEVICE")
FSTAB_LINE="UUID=$UUID $MOUNT_POINT $FSTYPE $MOUNT_OPTS 0 2"

if ! grep -q "$UUID" "$FSTAB_FILE"; then
log "Adding $FSTAB_LINE to $FSTAB_FILE"
echo "$FSTAB_LINE" >> "$FSTAB_FILE"
log "Adding $FSTAB_LINE to $FSTAB_FILE"
echo "$FSTAB_LINE" >>"$FSTAB_FILE"
else
log "UUID $UUID already in $FSTAB_FILE — skipping"
log "UUID $UUID already in $FSTAB_FILE — skipping"
fi

log "Mounted $DEVICE at $MOUNT_POINT with owner=$OWNER and mode=$PERMISSIONS"
log "Mounted $DEVICE at $MOUNT_POINT with owner=$OWNER and mode=$PERMISSIONS"
7 changes: 3 additions & 4 deletions ansible/files/admin_api_scripts/pg_upgrade_scripts/check.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,8 @@ set -euo pipefail
STATUS_FILE="/tmp/pg-upgrade-status"

if [ -f "${STATUS_FILE}" ]; then
STATUS=$(cat "${STATUS_FILE}")
echo -n "${STATUS}"
STATUS=$(cat "${STATUS_FILE}")
echo -n "${STATUS}"
else
echo -n "unknown"
echo -n "unknown"
fi

Loading
Loading