diff --git a/Cargo.lock b/Cargo.lock index 483df2096f..de787f3d94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7563,7 +7563,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "start-cli" -version = "2.2.0" +version = "2.3.0" dependencies = [ "start-core", "tracing", @@ -7718,7 +7718,7 @@ dependencies = [ [[package]] name = "start-os" -version = "0.4.0-rev.2" +version = "0.4.0-rev.3" dependencies = [ "include_dir", "start-core", diff --git a/package-lock.json b/package-lock.json index 8f85a907fb..b2f4a74303 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "startos-ui", - "version": "0.4.0.2", + "version": "0.4.0.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "startos-ui", - "version": "0.4.0.2", + "version": "0.4.0.3", "license": "MIT", "dependencies": { "@angular/cdk": "^22.2.1", diff --git a/package.json b/package.json index f8b35ac4db..2a6336cfcc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "startos-ui", - "version": "0.4.0.2", + "version": "0.4.0.3", "author": "Start9 Labs, Inc", "homepage": "https://start9.com/", "license": "MIT", diff --git a/projects/start-cli/CHANGELOG.md b/projects/start-cli/CHANGELOG.md index 6ca4986cf9..4490e1de29 100644 --- a/projects/start-cli/CHANGELOG.md +++ b/projects/start-cli/CHANGELOG.md @@ -9,6 +9,12 @@ Because `start-cli` is a thin client over `start-core`, most user-visible CLI ch in `start-core`; record here anything that changes this crate's entrypoint, features, packaging, or the CLI's externally observable behavior. +## [2.3.0] + +### Added + +- **`server restart` and `server shutdown` wait for a running backup to finish by default.** Pass `--force` to interrupt the backup. `server cancel-deferred-power` cancels the pending action. These commands require StartOS 0.4.0.3 or later. + ## [2.2.0] ### Security diff --git a/projects/start-cli/Cargo.toml b/projects/start-cli/Cargo.toml index 5c8bce31ed..da81a9ef83 100644 --- a/projects/start-cli/Cargo.toml +++ b/projects/start-cli/Cargo.toml @@ -3,7 +3,7 @@ edition = "2024" license = "MIT" name = "start-cli" repository = "https://github.com/Start9Labs/start-technologies" -version = "2.2.0" # VERSION_BUMP +version = "2.3.0" # VERSION_BUMP [[bin]] name = "start-cli" diff --git a/projects/start-cli/man/start-cli-server-cancel-deferred-power.1 b/projects/start-cli/man/start-cli-server-cancel-deferred-power.1 new file mode 100644 index 0000000000..e5842dd5e0 --- /dev/null +++ b/projects/start-cli/man/start-cli-server-cancel-deferred-power.1 @@ -0,0 +1,13 @@ +.ie \n(.g .ds Aq \(aq +.el .ds Aq ' +.TH start-cli-server-cancel-deferred-power 1 "cancel-deferred-power " +.SH NAME +start\-cli\-server\-cancel\-deferred\-power \- Cancel a restart or shutdown that is waiting for a backup to finish +.SH SYNOPSIS +\fBstart\-cli server cancel\-deferred\-power\fR [\fB\-h\fR|\fB\-\-help\fR] +.SH DESCRIPTION +Cancel a restart or shutdown that is waiting for a backup to finish +.SH OPTIONS +.TP +\fB\-h\fR, \fB\-\-help\fR +Print help diff --git a/projects/start-cli/man/start-cli-server-restart.1 b/projects/start-cli/man/start-cli-server-restart.1 index d77c70470f..eede323bbb 100644 --- a/projects/start-cli/man/start-cli-server-restart.1 +++ b/projects/start-cli/man/start-cli-server-restart.1 @@ -4,7 +4,7 @@ .SH NAME start\-cli\-server\-restart \- Restart the server .SH SYNOPSIS -\fBstart\-cli server restart\fR [\fB\-\-nowait\fR] [\fB\-h\fR|\fB\-\-help\fR] +\fBstart\-cli server restart\fR [\fB\-\-nowait\fR] [\fB\-\-force\fR] [\fB\-h\fR|\fB\-\-help\fR] .SH DESCRIPTION Restart the server .SH OPTIONS @@ -12,5 +12,8 @@ Restart the server \fB\-\-nowait\fR Return immediately instead of waiting for graceful shutdown to complete .TP +\fB\-\-force\fR +Interrupt a running backup instead of waiting for it to finish +.TP \fB\-h\fR, \fB\-\-help\fR Print help diff --git a/projects/start-cli/man/start-cli-server-shutdown.1 b/projects/start-cli/man/start-cli-server-shutdown.1 index a52d735b9d..5a64eb066b 100644 --- a/projects/start-cli/man/start-cli-server-shutdown.1 +++ b/projects/start-cli/man/start-cli-server-shutdown.1 @@ -4,7 +4,7 @@ .SH NAME start\-cli\-server\-shutdown \- Shutdown the server .SH SYNOPSIS -\fBstart\-cli server shutdown\fR [\fB\-\-nowait\fR] [\fB\-h\fR|\fB\-\-help\fR] +\fBstart\-cli server shutdown\fR [\fB\-\-nowait\fR] [\fB\-\-force\fR] [\fB\-h\fR|\fB\-\-help\fR] .SH DESCRIPTION Shutdown the server .SH OPTIONS @@ -12,5 +12,8 @@ Shutdown the server \fB\-\-nowait\fR Return immediately instead of waiting for graceful shutdown to complete .TP +\fB\-\-force\fR +Interrupt a running backup instead of waiting for it to finish +.TP \fB\-h\fR, \fB\-\-help\fR Print help diff --git a/projects/start-cli/man/start-cli-server.1 b/projects/start-cli/man/start-cli-server.1 index 6d7249f468..fc070fe593 100644 --- a/projects/start-cli/man/start-cli-server.1 +++ b/projects/start-cli/man/start-cli-server.1 @@ -13,6 +13,9 @@ Commands related to the server i.e. restart, update, and shutdown Print help .SH SUBCOMMANDS .TP +start\-cli\-server\-cancel\-deferred\-power(1) +Cancel a restart or shutdown that is waiting for a backup to finish +.TP start\-cli\-server\-clear\-smtp(1) Remove system smtp server and credentials .TP diff --git a/projects/start-cli/man/start-cli.1 b/projects/start-cli/man/start-cli.1 index c74cf8af43..c59c910ba5 100644 --- a/projects/start-cli/man/start-cli.1 +++ b/projects/start-cli/man/start-cli.1 @@ -1,6 +1,6 @@ .ie \n(.g .ds Aq \(aq .el .ds Aq ' -.TH start-cli 1 "start-cli 2.2.0" +.TH start-cli 1 "start-cli 2.3.0" .SH NAME start\-cli .SH SYNOPSIS @@ -123,4 +123,4 @@ Command for calculating the blake3 hash of a file start\-cli\-wifi(1) Commands related to wifi networks i.e. add, connect, delete .SH VERSION -v2.2.0 +v2.3.0 diff --git a/projects/start-cli/release-notes/2.3.0.md b/projects/start-cli/release-notes/2.3.0.md new file mode 100644 index 0000000000..ff10fd2b45 --- /dev/null +++ b/projects/start-cli/release-notes/2.3.0.md @@ -0,0 +1,5 @@ +**start-cli 2.3.0 lets a running backup finish before restarting or shutting down your server.** + +## Highlights + +- **`server restart` and `server shutdown` wait for a running backup to finish by default.** Pass `--force` to interrupt the backup. Cancel a pending action with `server cancel-deferred-power`. Requires StartOS 0.4.0.3 or later. diff --git a/projects/start-os/ARCHITECTURE.md b/projects/start-os/ARCHITECTURE.md index 90f9972480..681b58d489 100644 --- a/projects/start-os/ARCHITECTURE.md +++ b/projects/start-os/ARCHITECTURE.md @@ -81,8 +81,19 @@ erasure-coded FUSE filesystem used for StartOS backups. It builds to the this slice at boot. - `startos-shutdown.service` — graceful teardown on power-off only (ties to `poweroff.target`/`halt.target`, not reboot); its `ExecStop` calls - `start-cli server shutdown`. -- `startos-restart.service` — restart handling. + `start-cli server shutdown --force` to interrupt any backup while still + waiting for graceful teardown. +- `startos-restart.service` — graceful teardown on reboot/kexec; its `ExecStop` + calls `start-cli server restart --force`. Systemd-driven teardown cannot + defer a power action until a backup finishes. +- The physical power key is systemd-logind's (`HandlePowerKey=poweroff`), + except while a backup is running: `startd` then holds a logind + `handle-power-key` block inhibitor and reads the key itself, turning a press + into a shutdown that waits for the backup rather than one that interrupts it. + It is best-effort — when the inhibitor cannot be taken or no `power-switch` + device can be read, the key stays logind's — so treat it as one defence and + not a guarantee. See `start-core/src/power_key.rs` for why it inhibits + `handle-power-key` rather than `shutdown`. ## OS image packaging diff --git a/projects/start-os/CHANGELOG.md b/projects/start-os/CHANGELOG.md index c4a824241e..d88110c9aa 100644 --- a/projects/start-os/CHANGELOG.md +++ b/projects/start-os/CHANGELOG.md @@ -8,6 +8,26 @@ This file tracks notable changes since the move to the monorepo, and is what eac [GitHub release](https://github.com/Start9Labs/start-technologies/releases) links to for the detail behind its highlights. +## [0.4.0.3] + +### Added + +- **Restarting or shutting down during a backup offers to wait for it to + finish.** The prompt defaults to waiting after a countdown. A bar shows the + pending action and lets you cancel it. Pressing the physical power button + during a backup also waits when StartOS can intercept the key. Over the CLI, + `start-cli server restart` and `server shutdown` wait by default; `--force` + interrupts the backup. The command + `start-cli server cancel-deferred-power` cancels the pending action. StartOS + refuses new backups once a restart or shutdown is committed. See + [Creating Backups](https://docs.start9.com/start-os/backup-create.html). + +### Fixed + +- **Restarting the StartOS daemon clears interrupted backup, update, restart, + and shutdown indicators**, including any pending power action. An interrupted + operation no longer appears to be running after the daemon starts again. + ## [0.4.0.2] ### Security diff --git a/projects/start-os/Cargo.toml b/projects/start-os/Cargo.toml index 65cbea9b7d..59a78c0d52 100644 --- a/projects/start-os/Cargo.toml +++ b/projects/start-os/Cargo.toml @@ -3,9 +3,8 @@ edition = "2024" license = "MIT" name = "start-os" repository = "https://github.com/Start9Labs/start-technologies" -# Label only: SemVer has no 4th segment, so the OS version 0.4.0.2 is spelled 0.4.0-rev.2 -# here. Root package.json is the source of truth — see build/env/version.sh. -version = "0.4.0-rev.2" # VERSION_BUMP +# Label only: SemVer has no 4th segment. Root package.json is the source of truth. +version = "0.4.0-rev.3" # VERSION_BUMP [[bin]] name = "startbox" diff --git a/projects/start-os/build.mk b/projects/start-os/build.mk index 29c88d0b66..59ea393d8a 100644 --- a/projects/start-os/build.mk +++ b/projects/start-os/build.mk @@ -29,9 +29,10 @@ backup-fs-test: $(call ls-files, projects/start-os/backup-fs/src) projects/start container-runtime-test: projects/start-os/container-runtime/node_modules/.package-lock.json $(call ls-files, projects/start-os/container-runtime/src) projects/start-os/container-runtime/package.json projects/start-os/container-runtime/tsconfig.json cd projects/start-os/container-runtime && npm test -start-os-scripts-test: projects/start-os/build/lib/scripts/normalize-fstab projects/start-os/build/tests/normalize-fstab-test.sh projects/start-os/build/image-recipe/raspberrypi/img/usr/lib/startos/scripts/init_resize.sh projects/start-os/build/tests/init-resize-test.sh +start-os-scripts-test: projects/start-os/build/lib/scripts/normalize-fstab projects/start-os/build/tests/normalize-fstab-test.sh projects/start-os/build/image-recipe/raspberrypi/img/usr/lib/startos/scripts/init_resize.sh projects/start-os/build/tests/init-resize-test.sh projects/start-os/startos-restart.service projects/start-os/startos-shutdown.service projects/start-os/build/tests/power-units-test.sh ./projects/start-os/build/tests/normalize-fstab-test.sh ./projects/start-os/build/tests/init-resize-test.sh + ./projects/start-os/build/tests/power-units-test.sh projects/start-os/build/lib/migration-images/.done: projects/start-os/build/save-migration-images.sh ARCH=$(ARCH) ./projects/start-os/build/save-migration-images.sh projects/start-os/build/lib/migration-images diff --git a/projects/start-os/build/tests/power-units-test.sh b/projects/start-os/build/tests/power-units-test.sh new file mode 100755 index 0000000000..91b2d7117f --- /dev/null +++ b/projects/start-os/build/tests/power-units-test.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -euo pipefail + +ROOT=$(realpath "$(dirname "${BASH_SOURCE[0]}")/../../../..") +TMP=$(mktemp -d) +trap 'rm -rf -- "$TMP"' EXIT +UNITS="$TMP/usr/lib/systemd/system" +mkdir -p "$UNITS" "$TMP/usr/bin" "$TMP/bin" + +# verify checks executables without running them. +printf '#!/bin/sh\nexit 1\n' > "$TMP/usr/bin/start-cli" +cp "$TMP/usr/bin/start-cli" "$TMP/bin/true" +chmod +x "$TMP/usr/bin/start-cli" "$TMP/bin/true" +printf '[Unit]\nDescription=Test target\nDefaultDependencies=no\n' > "$UNITS/sysinit.target" + +for action in restart shutdown; do + unit="startos-$action.service" + cp "$ROOT/projects/start-os/$unit" "$UNITS/$unit" + if ! SYSTEMD_LOG_LEVEL=debug systemd-analyze verify --man=no --root="$TMP" "$unit" > "$TMP/parsed" 2>&1; then + cat "$TMP/parsed" >&2 + exit 1 + fi + awk '/ExecStop:/ { getline; print }' "$TMP/parsed" | + grep -Eq "^[[:space:]]*Command Line: /usr/bin/start-cli server $action --force$" || { + printf 'FAIL: %s must interrupt backups during systemd teardown\n' "$unit" >&2 + cat "$TMP/parsed" >&2 + exit 1 + } +done + +printf 'power unit tests passed\n' diff --git a/projects/start-os/docs/src/backup-create.md b/projects/start-os/docs/src/backup-create.md index a579877f7e..6a7fa986a4 100644 --- a/projects/start-os/docs/src/backup-create.md +++ b/projects/start-os/docs/src/backup-create.md @@ -21,6 +21,8 @@ Back up your server's data to a physical drive or a network folder. 1. To back up a service, StartOS first stops it (if it was running), performs the backup, then restarts it — but only if it was running beforehand. A service that was already stopped stays stopped. Consequently a service cannot be used while it is backing up, though you may continue to use your server and other services in the meantime. +1. Restarting or shutting down mid-backup can corrupt the backup of whichever service is being written at that moment, so StartOS asks first. Choosing `Restart` or `Shutdown` while a backup is running offers to wait for the backup to finish instead, and takes that option for you if you do not choose within 30 seconds — to power down regardless, choose the "now" option in that prompt. Pressing the server's physical power button during a backup waits without asking when StartOS can intercept the key; otherwise the button powers off through the operating system. Either way StartOS performs the restart or shutdown as soon as the backup completes, and until then a bar along the bottom of the screen says what is coming and lets you cancel it. Once a restart or shutdown is under way, StartOS refuses to start a new backup. + 1. Upon completion, StartOS issues a backup report, indicating which services were backed up, as well as any errors. 1. Wait for the `Backup Complete` notification before unplugging a backup drive. StartOS writes out the last of the backup and unmounts the drive before raising that notification, so the drive is safe to remove once it appears. The `Backup Progress` card reads `Complete` first, while StartOS is still finishing — the notification is the one to wait for. diff --git a/projects/start-os/docs/src/cli-reference.md b/projects/start-os/docs/src/cli-reference.md index fd51fc544b..1bccc07417 100644 --- a/projects/start-os/docs/src/cli-reference.md +++ b/projects/start-os/docs/src/cli-reference.md @@ -54,11 +54,21 @@ Restart, shut down, update, and configure the server. ### `start-cli server restart` -Restart the server. +Restart the server, waiting for a running backup to finish first. + +- `--force` — Interrupt a running backup and restart now +- `--nowait` — Return immediately instead of waiting for graceful shutdown ### `start-cli server shutdown` -Shut down the server. +Shut down the server, waiting for a running backup to finish first. + +- `--force` — Interrupt a running backup and shut down now +- `--nowait` — Return immediately instead of waiting for graceful shutdown + +### `start-cli server cancel-deferred-power` + +Cancel a restart or shutdown that is waiting for a backup to finish. ### `start-cli server update` diff --git a/projects/start-os/docs/src/installing-startos.md b/projects/start-os/docs/src/installing-startos.md index aabb9d8377..305bc3981d 100644 --- a/projects/start-os/docs/src/installing-startos.md +++ b/projects/start-os/docs/src/installing-startos.md @@ -8,7 +8,7 @@ This guide is for flashing StartOS to a USB drive, then installing it onto a des ## Download -1. Visit the [Github release page](https://github.com/Start9Labs/start-technologies/releases/tag/start-os/v0.4.0.2) to find the latest version of StartOS. +1. Visit the [Github release page](https://github.com/Start9Labs/start-technologies/releases/tag/start-os/v0.4.0.3) to find the latest version of StartOS. 1. Under "Image Downloads", select the image for your hardware. StartOS is available in x86_64 (AMD64), aarch64 (ARM64), and RISC-V (RVA23). For x86_64 and aarch64, two variants are available: - **Standard**: Includes proprietary firmware and drivers for broader hardware compatibility, including display and wireless. Recommended for most users. @@ -25,7 +25,7 @@ You do not need to understand what any of this means. Follow the three steps and ### 1. Find your file on the release page -On the [release page](https://github.com/Start9Labs/start-technologies/releases/tag/start-os/v0.4.0.2), scroll down to **OS Images Checksums**, then to the block under **SHA-256**. It holds one line per image: a long code, then the filename it belongs to. +On the [release page](https://github.com/Start9Labs/start-technologies/releases/tag/start-os/v0.4.0.3), scroll down to **OS Images Checksums**, then to the block under **SHA-256**. It holds one line per image: a long code, then the filename it belongs to. ```text 37b63c86197150866809d34b5824ae22c5fc705d4f8dc9e9750b8fa23485441a startos-0.4.0.1-fdb27c7_x86_64-nonfree.iso @@ -194,7 +194,7 @@ Anything else is a failure. **`BAD signature`** means the file is not what Start A Raspberry Pi does not use the USB installer above. Instead, you flash the StartOS image directly to the Pi's microSD card. This is also how a Raspberry Pi is updated to a new major version of StartOS — it cannot update over the air. If you are updating an existing 0.3.5.1 server, complete the [preparation steps in the update guide](update-040.md#prepare-your-server) before flashing. -1. Visit the [Github release page](https://github.com/Start9Labs/start-technologies/releases/tag/start-os/v0.4.0.2) and, from the downloads list, download the **Raspberry Pi `.img.gz`** file. +1. Visit the [Github release page](https://github.com/Start9Labs/start-technologies/releases/tag/start-os/v0.4.0.3) and, from the downloads list, download the **Raspberry Pi `.img.gz`** file. 1. Check it against the release page, exactly as in [Verify your download](#verify-your-download). The release lists a checksum for the `.img` inside the archive as well — the line you want is the one ending in `.img.gz`, because that is the file you downloaded. diff --git a/projects/start-os/docs/src/surge-and-ups.md b/projects/start-os/docs/src/surge-and-ups.md index 001ae66fc2..568fd606c9 100644 --- a/projects/start-os/docs/src/surge-and-ups.md +++ b/projects/start-os/docs/src/surge-and-ups.md @@ -47,3 +47,6 @@ There are three common topologies. For a home server, **line-interactive** is th StartOS does not currently include built-in support for UPS monitoring (USB or network), so it cannot automatically shut down when the battery is low during an extended outage. The server will run until battery exhaustion and then power off uncleanly. This still carries some risk of data corruption, but it is dramatically less risky than facing the original surge, brownout, or sudden outage with no UPS at all. If your area has frequent or long outages, size your UPS to give yourself time to shut down manually from the StartOS UI before the battery runs out. + +> [!NOTE] +> If a backup is running when you do, StartOS offers to wait for the backup to finish and takes that option if you do not choose within 30 seconds — which on battery is rarely what you want. Choose `Shut down now` instead. The server's physical power button waits for the backup when StartOS can intercept the key, so on battery use the web UI rather than the button. diff --git a/projects/start-os/man/start-container.1 b/projects/start-os/man/start-container.1 index 8edb6cbcc1..f92e1e52e2 100644 --- a/projects/start-os/man/start-container.1 +++ b/projects/start-os/man/start-container.1 @@ -1,6 +1,6 @@ .ie \n(.g .ds Aq \(aq .el .ds Aq ' -.TH start-container 1 "start-container 0.4.0.2" +.TH start-container 1 "start-container 0.4.0.3" .SH NAME start\-container .SH SYNOPSIS @@ -56,4 +56,4 @@ start\-container\-shutdown(1) .TP start\-container\-subcontainer(1) .SH VERSION -v0.4.0.2 +v0.4.0.3 diff --git a/projects/start-os/release-notes/0.4.0.3.md b/projects/start-os/release-notes/0.4.0.3.md new file mode 100644 index 0000000000..7e5f850797 --- /dev/null +++ b/projects/start-os/release-notes/0.4.0.3.md @@ -0,0 +1,7 @@ +**0.4.0.3 lets your server finish a backup before restarting or shutting down.** + +## Highlights + +- **Restart or shut down after a backup finishes.** During a backup, the UI offers to wait and selects that option after a countdown. A bar shows the pending action and lets you cancel it. Pressing the physical power button also waits for the backup when StartOS can intercept the key. StartOS refuses new backups once a restart or shutdown is committed. See [Creating Backups](https://docs.start9.com/start-os/backup-create.html). +- **CLI restart and shutdown wait for a running backup by default.** Use `start-cli server restart` or `start-cli server shutdown`; add `--force` to interrupt the backup. Cancel a pending action with `start-cli server cancel-deferred-power`. +- **Interrupted operations stop appearing active after the StartOS daemon restarts.** Backup, update, restart, and shutdown indicators reset, along with any pending power action. diff --git a/projects/start-os/startos-restart.service b/projects/start-os/startos-restart.service index c793832002..c55188bfa0 100644 --- a/projects/start-os/startos-restart.service +++ b/projects/start-os/startos-restart.service @@ -1,17 +1,7 @@ [Unit] Description=StartOS graceful restart -# Reboot/kexec counterpart of startos-shutdown.service (see its comment). DefaultDependencies=no -# Conflicts= selects reboot (not poweroff). The ExecStop graceful teardown -# (start-cli server restart -> startd shutdown_all -> per-container Exit RPC + -# lxc-stop) must run while startd and the containers are still up, exactly like a -# manual `start-cli server restart`. A unit ordered After=X is stopped *before* X -# on shutdown, so these two hold until the teardown has finished: -# startd.service - startd must be alive to service the RPC -# lxc.service / lxc-monitord.service - lxc.service's ExecStop is `lxc-containers -# stop`; unordered, it kills the container -# before the Exit RPC ("ungracefully dropped") -# Ordering only; Conflicts= stays the sole reboot-vs-poweroff selector. +# Stop before startd and LXC to finish graceful container teardown. After=startd.service lxc.service lxc-monitord.service Before=shutdown.target reboot.target kexec.target Conflicts=reboot.target kexec.target @@ -20,7 +10,8 @@ Conflicts=reboot.target kexec.target Type=oneshot RemainAfterExit=yes ExecStart=/bin/true -ExecStop=/usr/bin/start-cli server restart +# A systemd shutdown transaction cannot wait for a backup. +ExecStop=/usr/bin/start-cli server restart --force TimeoutStopSec=120 [Install] diff --git a/projects/start-os/startos-shutdown.service b/projects/start-os/startos-shutdown.service index 31ae3d6f4e..42492b11bf 100644 --- a/projects/start-os/startos-shutdown.service +++ b/projects/start-os/startos-shutdown.service @@ -1,16 +1,7 @@ [Unit] Description=StartOS graceful shutdown -# Poweroff/halt counterpart of startos-restart.service. Conflicts=poweroff/halt -# selects power-off (not reboot). The ExecStop graceful teardown must run while -# startd and the containers are still up, like a manual `start-cli server -# shutdown`. A unit ordered After=X is stopped *before* X on shutdown, so these -# two hold until the teardown has finished: -# startd.service - startd must be alive to service the RPC -# lxc.service / lxc-monitord.service - lxc.service's ExecStop is `lxc-containers -# stop`; unordered, it kills the container -# before the Exit RPC ("ungracefully dropped") -# Ordering only; Conflicts= stays the sole selector. DefaultDependencies=no +# Stop before startd and LXC to finish graceful container teardown. After=startd.service lxc.service lxc-monitord.service Before=shutdown.target poweroff.target halt.target Conflicts=poweroff.target halt.target @@ -19,10 +10,8 @@ Conflicts=poweroff.target halt.target Type=oneshot RemainAfterExit=yes ExecStart=/bin/true -# start-cli waits for graceful teardown by default; authenticates locally via -# /run/startos/rpc.authcookie. startd's Shutdown::execute detects that systemd is -# already stopping and leaves the final poweroff to systemd (see shutdown.rs). -ExecStop=/usr/bin/start-cli server shutdown +# A systemd shutdown transaction cannot wait for a backup. +ExecStop=/usr/bin/start-cli server shutdown --force TimeoutStopSec=120 [Install] diff --git a/projects/start-os/web/ui/src/app/routes/portal/components/header/menu.component.ts b/projects/start-os/web/ui/src/app/routes/portal/components/header/menu.component.ts index 9b895140d6..ab74981378 100644 --- a/projects/start-os/web/ui/src/app/routes/portal/components/header/menu.component.ts +++ b/projects/start-os/web/ui/src/app/routes/portal/components/header/menu.component.ts @@ -5,8 +5,8 @@ import { DocsLinkDirective, i18nPipe, SafeLinksDirective, - TaskService, } from '@start9labs/shared' +import { T } from '@start9labs/start-core' import { TuiButton, TuiDataList, @@ -17,6 +17,7 @@ import { import { filter } from 'rxjs' import { ApiService } from 'src/app/services/api/embassy-api.service' import { AuthService } from 'src/app/services/auth.service' +import { PowerService } from 'src/app/services/power.service' import { STATUS } from 'src/app/services/status.service' import { ABOUT } from './about.component' @@ -138,8 +139,8 @@ import { ABOUT } from './about.component' export class HeaderMenuComponent { private readonly api = inject(ApiService) private readonly auth = inject(AuthService) - private readonly tasks = inject(TaskService) private readonly dialog = inject(DialogService) + private readonly power = inject(PowerService) open = false @@ -149,7 +150,11 @@ export class HeaderMenuComponent { this.dialog.openComponent(ABOUT, { label: 'About this server' }).subscribe() } - async promptPower(action: 'restart' | 'shutdown') { + async promptPower(action: T.PowerAction) { + // During a backup the choice on offer is a different one, and asking it is + // confirmation enough. + if (this.power.backingUp()) return this.power.power(action).subscribe() + this.dialog .openConfirm( action === 'restart' @@ -175,15 +180,7 @@ export class HeaderMenuComponent { }, ) .pipe(filter(Boolean)) - .subscribe(() => - this.tasks.run( - async () => - await this.api[ - action === 'restart' ? 'restartServer' : 'shutdownServer' - ]({}), - `Beginning ${action}`, - ), - ) + .subscribe(() => this.power.power(action).subscribe()) } logout() { diff --git a/projects/start-os/web/ui/src/app/routes/portal/components/header/power.component.ts b/projects/start-os/web/ui/src/app/routes/portal/components/header/power.component.ts new file mode 100644 index 0000000000..5e140c5e30 --- /dev/null +++ b/projects/start-os/web/ui/src/app/routes/portal/components/header/power.component.ts @@ -0,0 +1,70 @@ +import { Component, signal } from '@angular/core' +import { takeUntilDestroyed } from '@angular/core/rxjs-interop' +import { i18nPipe } from '@start9labs/shared' +import { T } from '@start9labs/start-core' +import { TuiButton, TuiDialogContext } from '@taiga-ui/core' +import { injectContext, PolymorpheusComponent } from '@taiga-ui/polymorpheus' +import { take, timer } from 'rxjs' + +const COUNTDOWN = 30 + +@Component({ + template: ` +

+ {{ + 'A backup is currently running. Interrupting it now can corrupt the backup of the service being written.' + | i18n + }} +

+ @if (action === 'shutdown') { +

+ {{ + 'Are you sure you want to power down your server? This can take several minutes, and your server will not come back online automatically. To power on again, You will need to physically unplug your server and plug it back in.' + | i18n + }} +

+ } + + `, + styles: ` + button { + white-space: normal; + block-size: auto; + min-block-size: var(--tui-height-m); + } + `, + imports: [TuiButton, i18nPipe], +}) +export class PowerComponent { + private readonly context = + injectContext>() + + protected readonly action = this.context.data + protected readonly seconds = signal(COUNTDOWN) + + constructor() { + timer(0, 1000) + .pipe(take(COUNTDOWN + 1), takeUntilDestroyed()) + .subscribe(tick => { + this.seconds.set(COUNTDOWN - tick) + if (tick === COUNTDOWN) this.wait() + }) + } + + protected now() { + this.context.completeWith(true) + } + + protected wait() { + this.context.completeWith(false) + } +} + +export const POWER = new PolymorpheusComponent(PowerComponent) diff --git a/projects/start-os/web/ui/src/app/routes/portal/portal.component.ts b/projects/start-os/web/ui/src/app/routes/portal/portal.component.ts index 719622dac9..e039e0ebd9 100644 --- a/projects/start-os/web/ui/src/app/routes/portal/portal.component.ts +++ b/projects/start-os/web/ui/src/app/routes/portal/portal.component.ts @@ -2,7 +2,7 @@ import { Component, inject, signal } from '@angular/core' import { toSignal } from '@angular/core/rxjs-interop' import { RouterOutlet } from '@angular/router' import { WA_IS_MOBILE } from '@ng-web-apis/platform' -import { i18nPipe, LeafProgressPipe, TaskService } from '@start9labs/shared' +import { i18nPipe, LeafProgressPipe } from '@start9labs/shared' import { TuiButton, TuiCell, @@ -14,10 +14,10 @@ import { import { TuiActionBar, TuiProgress } from '@taiga-ui/kit' import { PatchDB } from 'patch-db-client' import { TabsComponent } from 'src/app/routes/portal/components/tabs.component' -import { ApiService } from 'src/app/services/api/embassy-api.service' import { OSService } from 'src/app/services/os.service' import { DataModel } from 'src/app/services/patch-db/data-model' import { PluginsService } from 'src/app/services/plugins.service' +import { PowerService } from 'src/app/services/power.service' import { HeaderComponent } from './components/header/header.component' @Component({ @@ -30,7 +30,32 @@ import { HeaderComponent } from './components/header/header.component' - @if (update(); as update) { + @if (deferredPower(); as action) { + + + + @if (action === 'restart') { + {{ + 'A backup is running. Your server will restart when it finishes.' + | i18n + }} + } @else { + {{ + 'A backup is running. Your server will shut down when it finishes.' + | i18n + }} + } + + + + } @else if (update(); as update) { @let leaf = update.overall | leafProgress; @@ -49,8 +74,7 @@ import { HeaderComponent } from './components/header/header.component' } - } - @if (restartReason(); as reason) { + } @else if (restartReason(); as reason) { @@ -167,9 +191,8 @@ import { HeaderComponent } from './components/header/header.component' ], }) export class PortalComponent { - private readonly tasks = inject(TaskService) private readonly patch = inject>(PatchDB) - private readonly api = inject(ApiService) + protected readonly power = inject(PowerService) readonly mobile = inject(WA_IS_MOBILE) readonly plugins = inject(PluginsService) @@ -177,6 +200,9 @@ export class PortalComponent { readonly restartReason = toSignal( this.patch.watch$('serverInfo', 'statusInfo', 'restart'), ) + readonly deferredPower = toSignal( + this.patch.watch$('serverInfo', 'statusInfo', 'deferredPowerAction'), + ) readonly bar = signal(true) getProgress(size: number, downloaded: number): number { @@ -184,9 +210,10 @@ export class PortalComponent { } restart() { - this.tasks.run(async () => { - this.bar.set(false) - await this.api.restartServer({}) - }, 'Beginning restart') + // Only stop offering the restart once one is actually under way — a + // deferred or dismissed one leaves the reason for this bar in place. + this.power.power('restart').subscribe(deferred => { + if (!deferred) this.bar.set(false) + }) } } diff --git a/projects/start-os/web/ui/src/app/routes/portal/routes/system/routes/general/general.component.ts b/projects/start-os/web/ui/src/app/routes/portal/routes/system/routes/general/general.component.ts index a8c8a0641e..63179b5475 100644 --- a/projects/start-os/web/ui/src/app/routes/portal/routes/system/routes/general/general.component.ts +++ b/projects/start-os/web/ui/src/app/routes/portal/routes/system/routes/general/general.component.ts @@ -44,6 +44,7 @@ import { ApiService } from 'src/app/services/api/embassy-api.service' import { ConfigService } from 'src/app/services/config.service' import { OSService } from 'src/app/services/os.service' import { DataModel } from 'src/app/services/patch-db/data-model' +import { PowerService } from 'src/app/services/power.service' import { TitleDirective } from 'src/app/services/title.service' import { KeyboardSelectComponent } from './keyboard-select.component' import { ServerNameDialog } from './server-name.dialog' @@ -278,6 +279,7 @@ export default class SystemGeneralComponent { private readonly injector = inject(INJECTOR) private readonly win = inject(WA_WINDOW) private readonly config = inject(ConfigService) + private readonly power = inject(PowerService) count = 0 @@ -514,9 +516,6 @@ export default class SystemGeneralComponent { } private async restart() { - this.tasks.run( - async () => await this.api.restartServer({}), - 'Beginning restart', - ) + this.power.power('restart').subscribe() } } diff --git a/projects/start-os/web/ui/src/app/services/api/api.fixures.ts b/projects/start-os/web/ui/src/app/services/api/api.fixures.ts index d6ecfbe06d..ebfdf201a5 100644 --- a/projects/start-os/web/ui/src/app/services/api/api.fixures.ts +++ b/projects/start-os/web/ui/src/app/services/api/api.fixures.ts @@ -27,6 +27,7 @@ export namespace Mock { restarting: false, shuttingDown: false, restart: null, + deferredPowerAction: null, } export const RegistryOSUpdate: T.OsVersionInfoMap = { diff --git a/projects/start-os/web/ui/src/app/services/api/embassy-api.service.ts b/projects/start-os/web/ui/src/app/services/api/embassy-api.service.ts index b4ae1541f0..ff4b241170 100644 --- a/projects/start-os/web/ui/src/app/services/api/embassy-api.service.ts +++ b/projects/start-os/web/ui/src/app/services/api/embassy-api.service.ts @@ -114,9 +114,11 @@ export abstract class ApiService { params: T.UpdateSystemParams, ): Promise - abstract restartServer(params: {}): Promise + abstract restartServer(params: Partial): Promise - abstract shutdownServer(params: {}): Promise + abstract shutdownServer(params: Partial): Promise + + abstract cancelDeferredPower(params: {}): Promise abstract repairDisk(params: {}): Promise diff --git a/projects/start-os/web/ui/src/app/services/api/embassy-live-api.service.ts b/projects/start-os/web/ui/src/app/services/api/embassy-live-api.service.ts index 202f0f88e4..17cd52a32a 100644 --- a/projects/start-os/web/ui/src/app/services/api/embassy-live-api.service.ts +++ b/projects/start-os/web/ui/src/app/services/api/embassy-live-api.service.ts @@ -13,9 +13,12 @@ import { } from '@start9labs/shared' import { T } from '@start9labs/start-core' import { Dump, pathFromArray } from 'patch-db-client' -import { filter, firstValueFrom, Observable } from 'rxjs' +import { Observable } from 'rxjs' import { webSocket, WebSocketSubject } from 'rxjs/webSocket' -import { PATCH_CACHE } from 'src/app/services/patch-db/patch-db-source' +import { + PATCH_CACHE, + waitForPatchSequence, +} from 'src/app/services/patch-db/patch-db-source' import { AuthService } from '../auth.service' import { DataModel } from '../patch-db/data-model' import { @@ -235,14 +238,18 @@ export class LiveApiService extends ApiService { return this.rpcRequest({ method: 'server.update', params }) } - async restartServer(params: {}): Promise { + async restartServer(params: Partial): Promise { return this.rpcRequest({ method: 'server.restart', params }) } - async shutdownServer(params: {}): Promise { + async shutdownServer(params: Partial): Promise { return this.rpcRequest({ method: 'server.shutdown', params }) } + async cancelDeferredPower(params: {}): Promise { + return this.rpcRequest({ method: 'server.cancel-deferred-power', params }) + } + async repairDisk(params: {}): Promise { return this.rpcRequest({ method: 'disk.repair', params }) } @@ -780,9 +787,7 @@ export class LiveApiService extends ApiService { const patchSequence = res.headers.get('x-patch-sequence') if (patchSequence) - await firstValueFrom( - this.cache$.pipe(filter(({ id }) => id >= Number(patchSequence))), - ) + await waitForPatchSequence(this.cache$, Number(patchSequence)) return body.result } diff --git a/projects/start-os/web/ui/src/app/services/api/embassy-mock-api.service.ts b/projects/start-os/web/ui/src/app/services/api/embassy-mock-api.service.ts index ace3facf09..52592c8373 100644 --- a/projects/start-os/web/ui/src/app/services/api/embassy-mock-api.service.ts +++ b/projects/start-os/web/ui/src/app/services/api/embassy-mock-api.service.ts @@ -31,6 +31,7 @@ import { } from 'src/app/services/patch-db/data-model' import { toAuthorityUrl } from 'src/app/utils/acme' import { AuthService } from '../auth.service' +import { PATCH_CACHE, waitForPatchSequence } from '../patch-db/patch-db-source' import { Mock } from './api.fixures' import { ActionRes, @@ -106,7 +107,10 @@ const INIT_PROGRESS: T.FullProgress = { export class MockApiService extends ApiService { readonly mockWsSource$ = new Subject() private readonly storage = inject(WA_SESSION_STORAGE) + private readonly cache$ = inject(PATCH_CACHE) private readonly revertTime = 1800 + private backingUp = false + private deferredPowerAction: T.PowerAction | null = null sequence = 0 constructor() { @@ -399,9 +403,14 @@ export class MockApiService extends ApiService { } } - async restartServer(params: {}): Promise { + async restartServer(params: Partial): Promise { await pauseFor(2000) + if (!params.force && this.backingUp) { + return this.deferPower('restart') + } + await this.deferPower(null) + const patch = [ { op: PatchOp.REPLACE, @@ -425,9 +434,14 @@ export class MockApiService extends ApiService { return null } - async shutdownServer(params: {}): Promise { + async shutdownServer(params: Partial): Promise { await pauseFor(2000) + if (!params.force && this.backingUp) { + return this.deferPower('shutdown') + } + await this.deferPower(null) + const patch = [ { op: PatchOp.REPLACE, @@ -451,6 +465,11 @@ export class MockApiService extends ApiService { return null } + async cancelDeferredPower(params: {}): Promise { + await pauseFor(1000) + return this.deferPower(null) + } + async repairDisk(params: {}): Promise { await pauseFor(2000) return null @@ -945,6 +964,7 @@ export class MockApiService extends ApiService { async createBackup(params: T.BackupParams): Promise { await pauseFor(2000) + this.backingUp = true const serverPath = '/serverInfo/statusInfo/backupProgress' const ids = params.packageIds || [] // One phase per package plus a trailing "OS Data" phase (the host @@ -1022,6 +1042,13 @@ export class MockApiService extends ApiService { }, ] this.mockRevision(lastPatch) + this.backingUp = false + if (this.deferredPowerAction) { + const action = this.deferredPowerAction + await this[action === 'restart' ? 'restartServer' : 'shutdownServer']( + {}, + ) + } // Feature 1: a completed backup whose target still holds a legacy (V1) // folder raises a warning notification — bumps the unread badge and @@ -2333,6 +2360,19 @@ export class MockApiService extends ApiService { this.mockRevision(patch) } + private async deferPower(action: T.PowerAction | null): Promise { + this.deferredPowerAction = action + this.mockRevision([ + { + op: PatchOp.REPLACE, + path: '/serverInfo/statusInfo/deferredPowerAction', + value: action, + }, + ]) + await waitForPatchSequence(this.cache$, this.sequence) + return null + } + private mockData(path: string): any { const parts = path.split('/').filter(Boolean) let obj: any = mockPatchData diff --git a/projects/start-os/web/ui/src/app/services/api/mock-patch.ts b/projects/start-os/web/ui/src/app/services/api/mock-patch.ts index 6039a0666c..0fa2c7e9cf 100644 --- a/projects/start-os/web/ui/src/app/services/api/mock-patch.ts +++ b/projects/start-os/web/ui/src/app/services/api/mock-patch.ts @@ -302,6 +302,7 @@ export const mockPatchData: DataModel = { shuttingDown: false, backupProgress: null, restart: null, + deferredPowerAction: null, }, hostname: 'random-words', pubkey: 'npub1sg6plzptd64u62a878hep2kev88swjh3tw00gjsfl8f237lmu63q0uf63m', diff --git a/projects/start-os/web/ui/src/app/services/patch-db/patch-db-source.ts b/projects/start-os/web/ui/src/app/services/patch-db/patch-db-source.ts index 6ddaf6b72c..785f52aa0b 100644 --- a/projects/start-os/web/ui/src/app/services/patch-db/patch-db-source.ts +++ b/projects/start-os/web/ui/src/app/services/patch-db/patch-db-source.ts @@ -1,6 +1,6 @@ import { inject, Injectable, InjectionToken } from '@angular/core' import { Dump, Revision, Update } from 'patch-db-client' -import { BehaviorSubject, EMPTY, Observable, throwError, timer } from 'rxjs' +import { BehaviorSubject, EMPTY, firstValueFrom, Observable } from 'rxjs' import { bufferTime, catchError, @@ -24,6 +24,13 @@ export const PATCH_CACHE = new InjectionToken('', { }), }) +export async function waitForPatchSequence( + cache$: Observable>, + sequence: number, +): Promise { + await firstValueFrom(cache$.pipe(filter(({ id }) => id >= sequence))) +} + @Injectable({ providedIn: 'root', }) diff --git a/projects/start-os/web/ui/src/app/services/power.service.ts b/projects/start-os/web/ui/src/app/services/power.service.ts new file mode 100644 index 0000000000..d4067c4796 --- /dev/null +++ b/projects/start-os/web/ui/src/app/services/power.service.ts @@ -0,0 +1,63 @@ +import { inject, Injectable } from '@angular/core' +import { toSignal } from '@angular/core/rxjs-interop' +import { DialogService, TaskService } from '@start9labs/shared' +import { T } from '@start9labs/start-core' +import { PatchDB } from 'patch-db-client' +import { defer, filter, map, Observable, switchMap, take } from 'rxjs' +import { POWER } from 'src/app/routes/portal/components/header/power.component' +import { ApiService } from 'src/app/services/api/embassy-api.service' +import { OSService } from 'src/app/services/os.service' +import { DataModel } from 'src/app/services/patch-db/data-model' + +@Injectable({ providedIn: 'root' }) +export class PowerService { + private readonly api = inject(ApiService) + private readonly dialog = inject(DialogService) + private readonly tasks = inject(TaskService) + private readonly patch = inject>(PatchDB) + readonly backingUp = toSignal(inject(OSService).backingUp$, { + initialValue: false, + }) + + /** + * Requests backup-safe power unless the user chooses immediate interruption. + * Emits whether server-owned power is deferred after a successful request. + * Dismissal and failed requests emit nothing. + */ + power(action: T.PowerAction): Observable { + if (!this.backingUp()) return this.run(action, false) + + return this.dialog + .openComponent(POWER, { + label: action === 'restart' ? 'Restart' : 'Warning', + size: 's', + data: action, + }) + .pipe(switchMap(force => this.run(action, force))) + } + + cancel() { + this.tasks.run(async () => await this.api.cancelDeferredPower({})) + } + + private run(action: T.PowerAction, force: boolean): Observable { + return defer(() => + this.tasks.run( + async () => + action === 'restart' + ? await this.api.restartServer({ force }) + : await this.api.shutdownServer({ force }), + !force && this.backingUp() + ? 'Wait for backup to complete' + : `Beginning ${action}`, + ), + ).pipe( + filter(Boolean), + switchMap(() => + this.patch + .watch$('serverInfo', 'statusInfo', 'deferredPowerAction') + .pipe(take(1), map(Boolean)), + ), + ) + } +} diff --git a/shared-libs/crates/start-core/locales/i18n.yaml b/shared-libs/crates/start-core/locales/i18n.yaml index 4885a16f47..b09b82f305 100644 --- a/shared-libs/crates/start-core/locales/i18n.yaml +++ b/shared-libs/crates/start-core/locales/i18n.yaml @@ -1669,6 +1669,13 @@ backup.bulk.service-not-ready: fr_FR: "Impossible de créer une sauvegarde d'un service encore en cours d'initialisation ou en état d'erreur" pl_PL: "Nie można utworzyć kopii zapasowej usługi, która jest jeszcze inicjalizowana lub znajduje się w stanie błędu" +backup.bulk.powering-down: + en_US: "Server is shutting down or restarting" + de_DE: "Server wird heruntergefahren oder neu gestartet" + es_ES: "El servidor se está apagando o reiniciando" + fr_FR: "Le serveur est en cours d'arrêt ou de redémarrage" + pl_PL: "Serwer jest wyłączany lub uruchamiany ponownie" + backup.bulk.reclaim-failed: en_US: "Failed to reclaim old-backup space: %{error}" de_DE: "Speicherplatz der alten Sicherung konnte nicht freigegeben werden: %{error}" @@ -3714,6 +3721,13 @@ help.arg.force-clear-task: fr_FR: "Forcer la suppression de la tâche même si elle est en cours" pl_PL: "Wymuś wyczyszczenie zadania nawet jeśli jest uruchomione" +help.arg.force-power: + en_US: "Interrupt a running backup instead of waiting for it to finish" + de_DE: "Eine laufende Sicherung unterbrechen, statt auf ihren Abschluss zu warten" + es_ES: "Interrumpir una copia de seguridad en curso en lugar de esperar a que termine" + fr_FR: "Interrompre une sauvegarde en cours au lieu d'attendre sa fin" + pl_PL: "Przerwij trwającą kopię zapasową zamiast czekać na jej zakończenie" + help.arg.force-remove-package: en_US: "Force removal even if the package has versions" de_DE: "Entfernung erzwingen, auch wenn das Paket Versionen hat" @@ -5125,6 +5139,13 @@ about.calculate-blake3-hash-for-file: fr_FR: "Calculer le hachage blake3 d'un fichier" pl_PL: "Oblicz hash blake3 dla pliku" +about.cancel-deferred-power: + en_US: "Cancel a restart or shutdown that is waiting for a backup to finish" + de_DE: "Einen Neustart oder ein Herunterfahren abbrechen, der bzw. das auf den Abschluss einer Sicherung wartet" + es_ES: "Cancelar un reinicio o apagado que está esperando a que termine una copia de seguridad" + fr_FR: "Annuler un redémarrage ou un arrêt en attente de la fin d'une sauvegarde" + pl_PL: "Anuluj ponowne uruchomienie lub wyłączenie oczekujące na zakończenie kopii zapasowej" + about.cancel-install-package: en_US: "Cancel an install of a package" de_DE: "Eine Paketinstallation abbrechen" diff --git a/shared-libs/crates/start-core/src/backup/backup_bulk.rs b/shared-libs/crates/start-core/src/backup/backup_bulk.rs index fb517f079a..103d0e1bf0 100644 --- a/shared-libs/crates/start-core/src/backup/backup_bulk.rs +++ b/shared-libs/crates/start-core/src/backup/backup_bulk.rs @@ -315,11 +315,14 @@ pub async fn backup_all( #[instrument(skip(db, initial))] fn assure_backing_up(db: &mut DatabaseModel, initial: &FullProgress) -> Result<(), Error> { - let backing_up = db - .as_public_mut() - .as_server_info_mut() - .as_status_info_mut() - .as_backup_progress_mut(); + let status = db.as_public_mut().as_server_info_mut().as_status_info_mut(); + if status.as_shutting_down().de()? || status.as_restarting().de()? { + return Err(Error::new( + eyre!("{}", t!("backup.bulk.powering-down")), + ErrorKind::InvalidRequest, + )); + } + let backing_up = status.as_backup_progress_mut(); if backing_up.transpose_ref().is_some() { return Err(Error::new( eyre!("{}", t!("backup.bulk.already-backing-up")), @@ -491,3 +494,123 @@ async fn perform_backup( Ok(backup_report) } + +#[cfg(test)] +mod test { + use imbl_value::json; + use patch_db::ModelExt; + + use super::*; + use crate::db::model::public::PowerAction; + + fn db_with(shutting_down: bool, restarting: bool) -> DatabaseModel { + DatabaseModel::from_value(json!({ + "public": { "serverInfo": { "statusInfo": { + "backupProgress": null, + "updateProgress": null, + "shuttingDown": shutting_down, + "restarting": restarting, + "restart": null, + "deferredPowerAction": null, + } } } + })) + } + + fn initial_progress() -> FullProgress { + let progress = FullProgressTracker::new(); + progress.add_phase("Initializing".into(), None).start(); + progress.snapshot() + } + + #[test] + fn starts_a_backup_on_an_idle_server() { + let mut db = db_with(false, false); + let initial = initial_progress(); + let mut expected = db.clone(); + expected + .as_public_mut() + .as_server_info_mut() + .as_status_info_mut() + .as_backup_progress_mut() + .ser(&Some(initial.clone())) + .unwrap(); + + assure_backing_up(&mut db, &initial).unwrap(); + + assert_eq!(db.as_value(), expected.as_value()); + } + + #[test] + fn refuses_an_existing_backup_without_mutating_the_database() { + let mut db = db_with(false, false); + assure_backing_up(&mut db, &initial_progress()).unwrap(); + let before = db.as_value().clone(); + + let error = assure_backing_up(&mut db, &FullProgress::new()).unwrap_err(); + + assert_eq!(error.kind, ErrorKind::InvalidRequest); + assert_eq!( + error.display_src().to_string(), + t!("backup.bulk.already-backing-up") + ); + assert_eq!(db.as_value(), &before); + } + + #[test] + fn refuses_a_backup_once_a_power_action_has_begun() { + for (shutting_down, restarting) in [(true, false), (false, true)] { + for existing_backup in [None, Some(initial_progress())] { + let mut db = db_with(shutting_down, restarting); + db.as_public_mut() + .as_server_info_mut() + .as_status_info_mut() + .as_backup_progress_mut() + .ser(&existing_backup) + .unwrap(); + let before = db.as_value().clone(); + + let error = assure_backing_up(&mut db, &initial_progress()).unwrap_err(); + + assert_eq!(error.kind, ErrorKind::InvalidRequest); + assert_eq!( + error.display_src().to_string(), + t!("backup.bulk.powering-down") + ); + assert_eq!(db.as_value(), &before); + } + } + } + + #[test] + fn admits_a_backup_while_a_power_action_is_only_deferred() { + for action in [PowerAction::Shutdown, PowerAction::Restart] { + let mut db = db_with(false, false); + db.as_public_mut() + .as_server_info_mut() + .as_status_info_mut() + .as_deferred_power_action_mut() + .ser(&Some(action)) + .unwrap(); + let initial = initial_progress(); + + assure_backing_up(&mut db, &initial).unwrap(); + + let status = db.as_public().as_server_info().as_status_info(); + assert_eq!( + status.as_deferred_power_action().de().unwrap(), + Some(action) + ); + assert!(status.as_backup_progress().de().unwrap().is_some()); + assert!(!status.as_shutting_down().de().unwrap()); + assert!(!status.as_restarting().de().unwrap()); + let before = db.as_value().clone(); + let error = assure_backing_up(&mut db, &initial).unwrap_err(); + assert_eq!(error.kind, ErrorKind::InvalidRequest); + assert_eq!( + error.display_src().to_string(), + t!("backup.bulk.already-backing-up") + ); + assert_eq!(db.as_value(), &before); + } + } +} diff --git a/shared-libs/crates/start-core/src/bins/startd.rs b/shared-libs/crates/start-core/src/bins/startd.rs index c80f5c0598..a51addc94a 100644 --- a/shared-libs/crates/start-core/src/bins/startd.rs +++ b/shared-libs/crates/start-core/src/bins/startd.rs @@ -18,6 +18,7 @@ use crate::net::web_server::{Acceptor, WebServer}; use crate::prelude::*; use crate::shutdown::Shutdown; use crate::system::launch_metrics_task; +use crate::util::future::NonDetachingJoinHandle; use crate::util::io::append_file; use crate::util::logger::LOGGER; @@ -63,6 +64,23 @@ async fn inner_main( ) .await?; + // Every status here is written by a task that died with the previous + // process, and only `init` — which this branch skips — would otherwise + // clear them. `restart` is deliberately left: it is a reboot-needed + // marker meant to outlive one. Before the RPC surface goes live, so + // nothing races the reset. + ctx.db + .mutate(|db| { + let status = db.as_public_mut().as_server_info_mut().as_status_info_mut(); + status.as_backup_progress_mut().ser(&None)?; + status.as_update_progress_mut().ser(&None)?; + status.as_deferred_power_action_mut().ser(&None)?; + status.as_shutting_down_mut().ser(&false)?; + status.as_restarting_mut().ser(&false) + }) + .await + .result?; + server.serve_ui_for(ctx.clone()); handle.complete(); @@ -102,6 +120,15 @@ async fn inner_main( .expect("send shutdown signal"); }); + let deferred_power_ctx = rpc_ctx.clone(); + let _deferred_power = NonDetachingJoinHandle::from(tokio::spawn( + crate::shutdown::run_deferred_power_actions(deferred_power_ctx), + )); + #[cfg(target_os = "linux")] + let _power_key = NonDetachingJoinHandle::from(tokio::spawn( + crate::power_key::watch_power_key(rpc_ctx.clone()), + )); + let metrics_ctx = rpc_ctx.clone(); let metrics_task = tokio::spawn(async move { launch_metrics_task(&metrics_ctx.metrics_cache, || { diff --git a/shared-libs/crates/start-core/src/db/model/public.rs b/shared-libs/crates/start-core/src/db/model/public.rs index bd77b6c905..bb9548eb84 100644 --- a/shared-libs/crates/start-core/src/db/model/public.rs +++ b/shared-libs/crates/start-core/src/db/model/public.rs @@ -130,6 +130,7 @@ impl Public { shutting_down: false, restarting: false, restart: None, + deferred_power_action: None, }, unread_notification_count: 0, pubkey: ssh_key::PublicKey::from(&account.ssh_key) @@ -215,6 +216,14 @@ pub enum RestartReason { Update, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, TS)] +#[serde(rename_all = "lowercase")] +#[ts(export)] +pub enum PowerAction { + Restart, + Shutdown, +} + #[derive(Debug, Default, Deserialize, Serialize, HasModel, TS)] #[serde(rename_all = "camelCase")] #[model = "Model"] @@ -448,6 +457,10 @@ pub struct ServerStatus { pub restarting: bool, #[serde(default)] pub restart: Option, + /// A restart or shutdown that was asked for while a backup was running, and + /// which StartOS carries out once the backup finishes. + #[serde(default)] + pub deferred_power_action: Option, } #[derive(Debug, Default, Deserialize, Serialize, HasModel, TS)] diff --git a/shared-libs/crates/start-core/src/init.rs b/shared-libs/crates/start-core/src/init.rs index 0b4c0ffad2..bc19040e7c 100644 --- a/shared-libs/crates/start-core/src/init.rs +++ b/shared-libs/crates/start-core/src/init.rs @@ -413,6 +413,7 @@ pub async fn init( shutting_down: false, restarting: false, restart: None, + deferred_power_action: None, }; db.mutate(|v| { let server_info = v.as_public_mut().as_server_info_mut(); diff --git a/shared-libs/crates/start-core/src/lib.rs b/shared-libs/crates/start-core/src/lib.rs index 357133fa60..ae430df571 100644 --- a/shared-libs/crates/start-core/src/lib.rs +++ b/shared-libs/crates/start-core/src/lib.rs @@ -67,6 +67,8 @@ pub mod middleware; pub mod net; pub mod notifications; pub mod os_install; +#[cfg(target_os = "linux")] +pub mod power_key; pub mod prelude; pub mod progress; pub mod registry; @@ -364,6 +366,13 @@ pub fn server() -> ParentHandler { .with_about("about.restart-server") .with_call_remote::(), ) + .subcommand( + "cancel-deferred-power", + from_fn_async(shutdown::cancel_deferred_power) + .no_display() + .with_about("about.cancel-deferred-power") + .with_call_remote::(), + ) .subcommand( "rebuild", from_fn_async(shutdown::rebuild) diff --git a/shared-libs/crates/start-core/src/power_key.rs b/shared-libs/crates/start-core/src/power_key.rs new file mode 100644 index 0000000000..7161095b8b --- /dev/null +++ b/shared-libs/crates/start-core/src/power_key.rs @@ -0,0 +1,273 @@ +//! The case power button, while a backup is running. +//! +//! systemd-logind powers the server off the moment the power key is pressed, +//! which cuts a running backup off mid-write and can corrupt the service being +//! written. For as long as a backup is underway startd therefore holds a logind +//! `block` inhibitor on `handle-power-key`, and reads the key itself so the +//! press still means something: it records a deferred shutdown, which the web +//! UI surfaces and which StartOS carries out once the backup finishes. +//! +//! It names `handle-power-key` and not `shutdown` on purpose — blocking +//! `shutdown` would also block the power-off StartOS itself asks systemd for at +//! the end of a graceful teardown. Both the inhibitor and the readers live for +//! one backup and are set up again for the next, so a device that came or went +//! in between is picked up and one that failed does not stand the feature down +//! for good. Failure gives the button back to logind rather than taking it +//! away. + +use std::io::Read; +use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; +use std::path::{Path, PathBuf}; + +use futures::FutureExt; +use futures::future::select_all; +use nix::sys::stat; +use patch_db::TypedDbWatch; +use patch_db::json_ptr::JsonPointer; +use tokio::io::unix::AsyncFd; +use zbus::proxy; +use zbus::zvariant::OwnedFd; + +use crate::context::RpcContext; +use crate::db::model::public::{PowerAction, ServerStatus}; +use crate::prelude::*; +use crate::shutdown::{STATUS_INFO_PTR, defer_until_backup_complete}; +use crate::sound::BEP; + +const EV_KEY: u16 = 0x01; +/// Both codes logind acts on, handled by one arm of its own `case` — matching +/// that is what keeps a press meaning here what it would have meant to logind. +const KEY_POWER: [u16; 2] = [116, 356]; +const KEY_PRESSED: i32 = 1; + +const EVENT_SIZE: usize = std::mem::size_of::(); +/// Offset of `struct input_event`'s trailing `type`, `code` and `value`. The +/// leading timestamp's width varies by architecture; those three fields are +/// always the last 8 bytes. +const EVENT_TAIL: usize = EVENT_SIZE - 8; + +const POWER_SWITCH_TAG_DIR: &str = "/run/udev/tags/power-switch"; + +#[proxy( + interface = "org.freedesktop.login1.Manager", + default_service = "org.freedesktop.login1", + default_path = "/org/freedesktop/login1" +)] +trait Login1Manager { + /// The returned file descriptor *is* the lock — it is released when dropped. + fn inhibit(&self, what: &str, who: &str, why: &str, mode: &str) -> Result; +} + +pub async fn watch_power_key(ctx: RpcContext) { + let manager = match logind().await { + Ok(manager) => manager, + Err(e) => { + tracing::warn!("cannot reach systemd-logind: {e}"); + tracing::debug!("{e:?}"); + return; + } + }; + let mut watch = ctx + .db + .watch(STATUS_INFO_PTR.parse::().unwrap()) + .await + .typed::(); + if let Err(e) = guard_backups(&ctx, &manager, &mut watch).await { + tracing::error!("stopped guarding backups from the power button: {e}"); + tracing::debug!("{e:?}"); + } +} + +async fn guard_backups( + ctx: &RpcContext, + manager: &Login1ManagerProxy<'_>, + watch: &mut TypedDbWatch, +) -> Result<(), Error> { + loop { + watch + .wait_for(|status: &ServerStatus| status.backup_progress.is_some()) + .await?; + // Every way of failing to guard one backup leaves the key to logind for + // that backup only; the next one sets up from scratch. + if let Err(e) = guard_backup(ctx, manager, watch).await { + tracing::error!("not guarding this backup from the power button: {e}"); + tracing::debug!("{e:?}"); + } + watch + .wait_for(|status: &ServerStatus| status.backup_progress.is_none()) + .await?; + } +} + +async fn guard_backup( + ctx: &RpcContext, + manager: &Login1ManagerProxy<'_>, + watch: &mut TypedDbWatch, +) -> Result<(), Error> { + // Enumerated per backup rather than once: udev tags every key-capable + // device, so the set changes whenever a keyboard is plugged in. + let devices = power_key_devices().await?; + if devices.is_empty() { + tracing::info!("no power-switch input device to read the power key from"); + return Ok(()); + } + let lock = manager + .inhibit( + "handle-power-key", + "StartOS", + "A backup is running", + "block", + ) + .await?; + let backup_over = watch.wait_for(|status: &ServerStatus| status.backup_progress.is_none()); + tokio::pin!(backup_over); + tokio::select! { + over = &mut backup_over => { over?; } + // Never inhibit a key nobody is reading. + _ = read_power_key(devices, ctx) => tracing::warn!( + "stopped reading the power key for this backup; systemd-logind has it back" + ), + } + drop(lock); + Ok(()) +} + +/// Returns as soon as any one device stops being readable: there is no telling +/// which of them the firmware reports presses on, so a partial failure has to +/// count as a failure. +async fn read_power_key(devices: Vec, ctx: &RpcContext) { + select_all( + devices + .into_iter() + .map(|path| read_device(path, ctx.clone()).boxed()), + ) + .await; +} + +async fn read_device(path: PathBuf, ctx: RpcContext) { + let device = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NONBLOCK) + .open(&path) + .and_then(AsyncFd::new); + let device = match device { + Ok(device) => device, + Err(e) => { + tracing::warn!("could not read the power key from {}: {e}", path.display()); + return; + } + }; + // evdev only ever hands back whole events, so a batch never splits one. + let mut buf = [0u8; EVENT_SIZE * 16]; + loop { + let read = match device.readable().await { + Ok(mut guard) => guard.try_io(|fd| { + let mut file = fd.get_ref(); + file.read(&mut buf) + }), + Err(e) => { + tracing::warn!("stopped reading {}: {e}", path.display()); + return; + } + }; + match read { + Err(_would_block) => continue, + Ok(Err(e)) => { + tracing::warn!("stopped reading {}: {e}", path.display()); + return; + } + Ok(Ok(len)) => { + for event in buf[..len].chunks_exact(EVENT_SIZE) { + if is_power_key_press(event) { + on_power_key(&ctx).await; + } + } + } + } + } +} + +async fn on_power_key(ctx: &RpcContext) { + match defer_until_backup_complete(ctx, PowerAction::Shutdown).await { + Ok(true) => { + tracing::info!("power key pressed during a backup; shutting down once it finishes"); + // The only feedback whoever pressed it has. Spawned so a contended + // sound device cannot stall the reader. + tokio::spawn(async { BEP.play().await.log_err() }); + } + Ok(false) => (), + Err(e) => { + tracing::error!("could not defer the shutdown for the running backup: {e}"); + tracing::debug!("{e:?}"); + } + } +} + +async fn logind() -> Result, Error> { + Ok(Login1ManagerProxy::new(&zbus::Connection::system().await?).await?) +} + +/// The devices systemd-logind itself treats as power switches: udev tags them +/// `power-switch`, and names each one by device number in that tag's directory. +/// Reading logind's own device set rather than picking devices by capability is +/// what keeps a press meaning here exactly what it would have meant to logind. +async fn power_key_devices() -> Result, Error> { + let mut devices = Vec::new(); + let mut dir = tokio::fs::read_dir("/dev/input").await?; + while let Some(entry) = dir.next_entry().await? { + if !entry.file_name().as_encoded_bytes().starts_with(b"event") { + continue; + } + let id = device_id(entry.metadata().await?.rdev()); + if Path::new(POWER_SWITCH_TAG_DIR).join(id).exists() { + devices.push(entry.path()); + } + } + Ok(devices) +} + +fn device_id(rdev: u64) -> String { + format!("c{}:{}", stat::major(rdev), stat::minor(rdev)) +} + +fn is_power_key_press(event: &[u8]) -> bool { + let tail = &event[EVENT_TAIL..]; + u16::from_ne_bytes([tail[0], tail[1]]) == EV_KEY + && KEY_POWER.contains(&u16::from_ne_bytes([tail[2], tail[3]])) + && i32::from_ne_bytes([tail[4], tail[5], tail[6], tail[7]]) == KEY_PRESSED +} + +#[cfg(test)] +mod test { + use super::*; + + /// `/dev/input/event0` is char device 13:64, and udev names its tag entry + /// after exactly that. + #[test] + fn names_a_device_the_way_udev_tags_it() { + assert_eq!(device_id(stat::makedev(13, 64)), "c13:64"); + assert_eq!(device_id(stat::makedev(13, 71)), "c13:71"); + } + + #[test] + fn recognizes_a_power_key_press() { + let mut event = [0u8; EVENT_SIZE]; + event[EVENT_TAIL..EVENT_TAIL + 2].copy_from_slice(&EV_KEY.to_ne_bytes()); + event[EVENT_TAIL + 2..EVENT_TAIL + 4].copy_from_slice(&KEY_POWER[0].to_ne_bytes()); + event[EVENT_TAIL + 4..].copy_from_slice(&KEY_PRESSED.to_ne_bytes()); + assert!(is_power_key_press(&event)); + + // KEY_POWER2, which logind acts on identically. + event[EVENT_TAIL + 2..EVENT_TAIL + 4].copy_from_slice(&KEY_POWER[1].to_ne_bytes()); + assert!(is_power_key_press(&event)); + + // A release, which must not power anything off. + event[EVENT_TAIL + 4..].copy_from_slice(&0i32.to_ne_bytes()); + assert!(!is_power_key_press(&event)); + + // Some other key. + event[EVENT_TAIL + 2..EVENT_TAIL + 4].copy_from_slice(&30u16.to_ne_bytes()); + event[EVENT_TAIL + 4..].copy_from_slice(&KEY_PRESSED.to_ne_bytes()); + assert!(!is_power_key_press(&event)); + } +} diff --git a/shared-libs/crates/start-core/src/shutdown.rs b/shared-libs/crates/start-core/src/shutdown.rs index 50ae71f855..6b1f060d21 100644 --- a/shared-libs/crates/start-core/src/shutdown.rs +++ b/shared-libs/crates/start-core/src/shutdown.rs @@ -1,9 +1,14 @@ +use std::time::Duration; + use clap::Parser; +use patch_db::json_ptr::JsonPointer; use serde::{Deserialize, Serialize}; use ts_rs::TS; use crate::PLATFORM; use crate::context::RpcContext; +use crate::db::model::DatabaseModel; +use crate::db::model::public::{PowerAction, ServerStatus}; use crate::disk::main::export; use crate::init::{STANDBY_MODE_PATH, SYSTEM_REBUILD_PATH}; use crate::prelude::*; @@ -119,7 +124,7 @@ fn systemd_is_stopping() -> bool { .unwrap_or(false) } -#[derive(Debug, Clone, Deserialize, Serialize, Parser, TS)] +#[derive(Debug, Clone, Default, Deserialize, Serialize, Parser, TS)] #[group(skip)] #[ts(export)] #[serde(rename_all = "camelCase")] @@ -129,11 +134,21 @@ pub struct ShutdownParams { /// frontend omits this and gets an immediate reply). Cleared with /// `--nowait`. The wait can't outlive the webserver teardown that follows /// container shutdown, so the connection drops once services are stopped. + /// A deferred action replies immediately, before teardown begins. #[arg(long = "nowait", action = clap::ArgAction::SetFalse, help = "help.arg.nowait")] #[serde(default)] wait: bool, + /// Interrupt a running backup instead of waiting for it to finish. + #[arg(long, help = "help.arg.force-power")] + #[serde(default)] + force: bool, } +pub(crate) const STATUS_INFO_PTR: &str = "/public/serverInfo/statusInfo"; +/// How long to leave a failing patch-db alone before trying to take the +/// deferred action again. +const TAKE_RETRY: Duration = Duration::from_secs(30); + async fn begin_shutdown(ctx: &RpcContext, restart: bool, wait: bool) { ctx.shutdown .send(Some(Shutdown { @@ -147,43 +162,367 @@ async fn begin_shutdown(ctx: &RpcContext, restart: bool, wait: bool) { } } +/// Records `action` as the deferred power action if a backup is underway, and +/// reports whether it did. Unlike [`defer_or_begin`] it never performs the +/// action, which is what the power key needs: with no backup to wait for, the +/// press is logind's to act on. +pub async fn defer_until_backup_complete( + ctx: &RpcContext, + action: PowerAction, +) -> Result { + ctx.db + .mutate(|db| defer_if_backing_up(db, action)) + .await + .result +} + +fn defer_if_backing_up(db: &mut DatabaseModel, action: PowerAction) -> Result { + let status = db.as_public_mut().as_server_info_mut().as_status_info_mut(); + if status.as_backup_progress().transpose_ref().is_none() { + return Ok(false); + } + status.as_deferred_power_action_mut().ser(&Some(action))?; + Ok(true) +} + +/// Either records `action` for after the backup, or commits to performing it +/// now — in one mutation, so a backup cannot start in the window between +/// deciding and acting. Returns whether it was deferred. +async fn defer_or_begin(ctx: &RpcContext, action: PowerAction, force: bool) -> Result { + ctx.db + .mutate(|db| defer_or_begin_in(db, action, force)) + .await + .result +} + +fn defer_or_begin_in( + db: &mut DatabaseModel, + action: PowerAction, + force: bool, +) -> Result { + let status = db.as_public_mut().as_server_info_mut().as_status_info_mut(); + if !force && status.as_backup_progress().transpose_ref().is_some() { + status.as_deferred_power_action_mut().ser(&Some(action))?; + return Ok(true); + } + status.as_deferred_power_action_mut().ser(&None)?; + match action { + PowerAction::Restart => status.as_restarting_mut().ser(&true)?, + PowerAction::Shutdown => status.as_shutting_down_mut().ser(&true)?, + } + Ok(false) +} + +/// Reads the deferred action and clears it in one breath, so a cancellation that +/// lands first wins and the caller performs nothing. +fn take_deferred(db: &mut DatabaseModel) -> Result, Error> { + let status = db.as_public_mut().as_server_info_mut().as_status_info_mut(); + let action = status.as_deferred_power_action().de()?; + status.as_deferred_power_action_mut().ser(&None)?; + Ok(action) +} + +/// Carries out each deferred power action once the backup it was waiting on +/// finishes. Runs for the lifetime of startd: an action can be recorded at any +/// point during any backup — from the web UI, the CLI, or the power button — so +/// this must survive one having failed. +pub async fn run_deferred_power_actions(ctx: RpcContext) { + let mut watch = ctx + .db + .watch(STATUS_INFO_PTR.parse::().unwrap()) + .await + .typed::(); + loop { + if let Err(e) = watch + .wait_for(|status| { + status.deferred_power_action.is_some() && status.backup_progress.is_none() + }) + .await + { + // The db is gone, so there is nothing left to retry against. + tracing::error!("stopped watching for deferred power actions: {e}"); + tracing::debug!("{e:?}"); + return; + } + let taken = ctx.db.mutate(take_deferred).await.result; + let action = match taken { + Ok(action) => action, + Err(e) => { + // A failed mutation leaves the db untouched, so retrying + // immediately would spin against whatever is failing. + tracing::error!("could not take the deferred power action: {e}"); + tracing::debug!("{e:?}"); + tokio::time::sleep(TAKE_RETRY).await; + continue; + } + }; + // A backup that started since the take is waited for in turn. + let params = ShutdownParams::default(); + let performed = match action { + Some(PowerAction::Restart) => { + tracing::info!("backup finished; carrying out the deferred restart"); + restart(ctx.clone(), params).await + } + Some(PowerAction::Shutdown) => { + tracing::info!("backup finished; carrying out the deferred shutdown"); + shutdown(ctx.clone(), params).await + } + None => continue, + }; + if let Err(e) = performed { + tracing::error!("deferred power action failed: {e}"); + tracing::debug!("{e:?}"); + // Put it back rather than losing it, and give whatever failed room + // to recover before trying again. Not via `defer_or_begin`: with the + // backup already over it would commit to performing the action + // instead of recording it. + if let Some(action) = action { + ctx.db + .mutate(|db| { + db.as_public_mut() + .as_server_info_mut() + .as_status_info_mut() + .as_deferred_power_action_mut() + .ser(&Some(action)) + }) + .await + .result + .log_err(); + } + tokio::time::sleep(TAKE_RETRY).await; + } + } +} + pub async fn shutdown( ctx: RpcContext, - ShutdownParams { wait }: ShutdownParams, + ShutdownParams { wait, force }: ShutdownParams, ) -> Result<(), Error> { - ctx.db - .mutate(|db| { - db.as_public_mut() - .as_server_info_mut() - .as_status_info_mut() - .as_shutting_down_mut() - .ser(&true) - }) - .await - .result?; + if defer_or_begin(&ctx, PowerAction::Shutdown, force).await? { + return Ok(()); + } begin_shutdown(&ctx, false, wait).await; Ok(()) } pub async fn restart( ctx: RpcContext, - ShutdownParams { wait }: ShutdownParams, + ShutdownParams { wait, force }: ShutdownParams, ) -> Result<(), Error> { + if defer_or_begin(&ctx, PowerAction::Restart, force).await? { + return Ok(()); + } + begin_shutdown(&ctx, true, wait).await; + Ok(()) +} + +pub async fn cancel_deferred_power(ctx: RpcContext) -> Result<(), Error> { ctx.db .mutate(|db| { db.as_public_mut() .as_server_info_mut() .as_status_info_mut() - .as_restarting_mut() - .ser(&true) + .as_deferred_power_action_mut() + .ser(&None) }) .await - .result?; - begin_shutdown(&ctx, true, wait).await; - Ok(()) + .result } pub async fn rebuild(ctx: RpcContext) -> Result<(), Error> { tokio::fs::write(SYSTEM_REBUILD_PATH, b"").await?; - restart(ctx, ShutdownParams { wait: false }).await + restart(ctx, ShutdownParams::default()).await +} + +#[cfg(test)] +mod test { + use imbl_value::json; + use patch_db::ModelExt; + + use super::*; + + fn db_with(backup_progress: Value, deferred: Value) -> DatabaseModel { + DatabaseModel::from_value(json!({ + "public": { "serverInfo": { "statusInfo": { + "backupProgress": backup_progress, + "updateProgress": null, + "shuttingDown": false, + "restarting": false, + "restart": null, + "deferredPowerAction": deferred, + } } } + })) + } + + fn backing_up() -> Value { + json!({ "overall": { "done": 0, "total": 2, "units": null }, "phases": [] }) + } + + /// `(deferred action, shutting down, restarting)`. + fn status(db: &DatabaseModel) -> (Option, bool, bool) { + let status = db.as_public().as_server_info().as_status_info(); + ( + status.as_deferred_power_action().de().unwrap(), + status.as_shutting_down().de().unwrap(), + status.as_restarting().de().unwrap(), + ) + } + + #[test] + fn json_and_rust_defaults_protect_running_backups() { + let params: ShutdownParams = serde_json::from_str("{}").unwrap(); + assert!(!params.force); + assert!(!params.wait); + assert!(!ShutdownParams::default().force); + assert!(!ShutdownParams::default().wait); + let forced: ShutdownParams = serde_json::from_str(r#"{"force":true}"#).unwrap(); + assert!(forced.force); + assert!(!forced.wait); + let omitted: ShutdownParams = imbl_value::from_value(json!({})).unwrap(); + assert!(!omitted.force); + assert!(!omitted.wait); + for force in [false, true] { + let params: ShutdownParams = imbl_value::from_value(json!({ "force": force })).unwrap(); + assert_eq!(params.force, force); + } + } + + #[test] + fn cli_force_and_nowait_are_independent() { + for action in ["restart", "shutdown"] { + for (flags, force, wait) in [ + (vec![], false, true), + (vec!["--force"], true, true), + (vec!["--nowait"], false, false), + (vec!["--force", "--nowait"], true, false), + ] { + let command = rpc_toolkit::CliApp::new( + |_: crate::context::config::ClientConfig| -> Result { + unreachable!() + }, + crate::main_api(), + ) + .into_command(); + let matches = command + .try_get_matches_from(["start-cli", "server", action].into_iter().chain(flags)) + .unwrap(); + let params = ::from_arg_matches( + matches + .subcommand_matches("server") + .unwrap() + .subcommand_matches(action) + .unwrap(), + ) + .unwrap(); + assert_eq!((params.force, params.wait), (force, wait)); + let serialized = imbl_value::to_value(¶ms).unwrap(); + assert_eq!(serialized, json!({ "wait": wait, "force": force })); + let rpc: ShutdownParams = imbl_value::from_value(serialized).unwrap(); + assert_eq!((rpc.force, rpc.wait), (force, wait)); + } + assert!(ShutdownParams::try_parse_from([action, "--after-backup"]).is_err()); + } + } + + #[test] + fn power_actions_defer_by_default_and_force_interrupts_backups() { + for action in [PowerAction::Restart, PowerAction::Shutdown] { + for backup in [false, true] { + for params in [ + serde_json::from_str::("{}").unwrap(), + ShutdownParams::default(), + serde_json::from_str(r#"{"force":true}"#).unwrap(), + ] { + let mut db = db_with( + if backup { backing_up() } else { json!(null) }, + json!("restart"), + ); + let deferred = backup && !params.force; + assert_eq!( + defer_or_begin_in(&mut db, action, params.force).unwrap(), + deferred + ); + assert_eq!( + status(&db), + if deferred { + (Some(action), false, false) + } else { + ( + None, + action == PowerAction::Shutdown, + action == PowerAction::Restart, + ) + } + ); + } + } + } + } + + #[test] + fn systemd_teardown_forces_both_power_actions() { + for (unit, action) in [ + ( + include_str!("../../../../projects/start-os/startos-restart.service"), + "restart", + ), + ( + include_str!("../../../../projects/start-os/startos-shutdown.service"), + "shutdown", + ), + ] { + let command = unit + .lines() + .find_map(|line| line.strip_prefix("ExecStop=")) + .unwrap(); + let args: Vec<_> = command.split_whitespace().collect(); + assert_eq!(&args[..3], &["/usr/bin/start-cli", "server", action]); + let params = ShutdownParams::try_parse_from( + std::iter::once("power").chain(args[3..].iter().copied()), + ) + .unwrap(); + assert!(params.wait); + assert!(params.force); + } + } + + /// Why [`run_deferred_power_actions`] cannot re-arm through this function: + /// with the backup over it takes the other branch and commits to the action, + /// which as a re-arm would leave the server flagged as powering down with + /// nothing left to do it. + #[test] + fn beginning_an_action_clears_any_pending_one() { + let mut db = db_with(json!(null), json!("restart")); + assert!(!defer_or_begin_in(&mut db, PowerAction::Shutdown, false).unwrap()); + assert_eq!(status(&db), (None, true, false)); + } + + #[test] + fn the_power_key_records_but_never_begins() { + let mut db = db_with(backing_up(), json!(null)); + assert!(defer_if_backing_up(&mut db, PowerAction::Shutdown).unwrap()); + assert_eq!(status(&db), (Some(PowerAction::Shutdown), false, false)); + + let mut db = db_with(json!(null), json!(null)); + assert!(!defer_if_backing_up(&mut db, PowerAction::Shutdown).unwrap()); + assert_eq!( + status(&db), + (None, false, false), + "no backup to protect, so the press is logind's to act on" + ); + } + + #[test] + fn taking_the_action_clears_it_so_only_one_pass_performs_it() { + let mut db = db_with(json!(null), json!("restart")); + assert_eq!(take_deferred(&mut db).unwrap(), Some(PowerAction::Restart)); + assert_eq!(take_deferred(&mut db).unwrap(), None); + } + + /// A cancellation that lands before the take wins outright. + #[test] + fn taking_a_cancelled_action_yields_nothing() { + let mut db = db_with(json!(null), json!(null)); + assert_eq!(take_deferred(&mut db).unwrap(), None); + } } diff --git a/shared-libs/crates/start-core/src/version/mod.rs b/shared-libs/crates/start-core/src/version/mod.rs index 8ad43fa5b1..76e29cc9c9 100644 --- a/shared-libs/crates/start-core/src/version/mod.rs +++ b/shared-libs/crates/start-core/src/version/mod.rs @@ -42,6 +42,7 @@ mod v0_3_6_alpha_18; mod v0_4_0; mod v0_4_0_1; mod v0_4_0_2; +mod v0_4_0_3; mod v0_4_0_alpha_0; mod v0_4_0_alpha_1; mod v0_4_0_alpha_2; @@ -79,7 +80,7 @@ mod v0_4_0_beta_7; mod v0_4_0_beta_8; mod v0_4_0_beta_9; -pub type Current = v0_4_0_2::Version; // VERSION_BUMP +pub type Current = v0_4_0_3::Version; // VERSION_BUMP impl Current { #[instrument(skip(self, db))] @@ -237,7 +238,8 @@ enum Version { V0_4_0_beta_10(Wrapper), V0_4_0(Wrapper), V0_4_0_1(Wrapper), - V0_4_0_2(Wrapper), // VERSION_BUMP + V0_4_0_2(Wrapper), + V0_4_0_3(Wrapper), // VERSION_BUMP Other(exver::Version), } @@ -317,7 +319,8 @@ impl Version { Self::V0_4_0_beta_10(v) => DynVersion(Box::new(v.0)), Self::V0_4_0(v) => DynVersion(Box::new(v.0)), Self::V0_4_0_1(v) => DynVersion(Box::new(v.0)), - Self::V0_4_0_2(v) => DynVersion(Box::new(v.0)), // VERSION_BUMP + Self::V0_4_0_2(v) => DynVersion(Box::new(v.0)), + Self::V0_4_0_3(v) => DynVersion(Box::new(v.0)), // VERSION_BUMP Self::Other(v) => { return Err(Error::new( eyre!("unknown version {v}"), @@ -389,7 +392,8 @@ impl Version { Version::V0_4_0_beta_10(Wrapper(x)) => x.semver(), Version::V0_4_0(Wrapper(x)) => x.semver(), Version::V0_4_0_1(Wrapper(x)) => x.semver(), - Version::V0_4_0_2(Wrapper(x)) => x.semver(), // VERSION_BUMP + Version::V0_4_0_2(Wrapper(x)) => x.semver(), + Version::V0_4_0_3(Wrapper(x)) => x.semver(), // VERSION_BUMP Version::Other(x) => x.clone(), } } @@ -777,6 +781,39 @@ mod tests { ); } + #[tokio::test] + async fn revision_three_upgrades_and_rolls_back_to_revision_two() { + let previous = v0_4_0_2::Version; + let current = Current::default(); + let resolved = Version::from_exver_version(current.semver()); + assert!(matches!(resolved, Version::V0_4_0_3(_))); + let target = resolved.as_version_t().unwrap(); + let mut db = json!({ "public": { "serverInfo": { + "version": "0.4.0.2", + "packageVersionCompat": ">=0.3.0 <0.5.0", + "postInitMigrationTodos": {}, + "latestMigrationRevision": 4, + } }, "payload": { "unchanged": true } }); + let payload = db["payload"].clone(); + let pre_ups = PreUps::load(&previous, &target).await.unwrap(); + migrate_from_unchecked(&previous, &target, pre_ups, &mut db).unwrap(); + assert_eq!(db["public"]["serverInfo"]["version"], json!("0.4.0.3")); + assert_eq!( + db["public"]["serverInfo"]["packageVersionCompat"], + to_value(previous.compat()).unwrap() + ); + assert_eq!( + db["public"]["serverInfo"]["postInitMigrationTodos"]["0.4.0.3"], + Value::Null + ); + assert_eq!(applied_migration_revision(&mut db).unwrap(), 0); + assert_eq!(db["payload"], payload); + rollback_to_unchecked(&target, &previous, &mut db).unwrap(); + assert_eq!(db["public"]["serverInfo"]["version"], json!("0.4.0.2")); + assert_eq!(applied_migration_revision(&mut db).unwrap(), 4); + assert_eq!(db["payload"], payload); + } + #[test] fn a_db_that_predates_revisions_reads_as_revision_zero() { let mut db = json!({ "public": { "serverInfo": {} } }); diff --git a/shared-libs/crates/start-core/src/version/v0_4_0_3.rs b/shared-libs/crates/start-core/src/version/v0_4_0_3.rs new file mode 100644 index 0000000000..aaffcf48fd --- /dev/null +++ b/shared-libs/crates/start-core/src/version/v0_4_0_3.rs @@ -0,0 +1,33 @@ +use exver::VersionRange; + +use super::v0_3_5::V0_3_0_COMPAT; +use super::{VersionT, v0_4_0_2}; +use crate::prelude::*; + +lazy_static::lazy_static! { + static ref V0_4_0_3: exver::Version = exver::Version::new([0, 4, 0, 3], []); +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct Version; + +impl VersionT for Version { + type Previous = v0_4_0_2::Version; + type PreUpRes = (); + + async fn pre_up(self) -> Result { + Ok(()) + } + fn semver(self) -> exver::Version { + V0_4_0_3.clone() + } + fn compat(self) -> &'static VersionRange { + &V0_3_0_COMPAT + } + fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result { + Ok(Value::Null) + } + fn down(self, _db: &mut Value) -> Result<(), Error> { + Ok(()) + } +} diff --git a/shared-libs/ts-modules/shared/src/i18n/dictionaries/de.ts b/shared-libs/ts-modules/shared/src/i18n/dictionaries/de.ts index 217ee9a61c..b830510007 100644 --- a/shared-libs/ts-modules/shared/src/i18n/dictionaries/de.ts +++ b/shared-libs/ts-modules/shared/src/i18n/dictionaries/de.ts @@ -825,4 +825,9 @@ export default { 933: 'Ihr Router verwendet seine Root-Zertifizierungsstelle, um SSL/TLS-Zertifikate für sich selbst zu erstellen. Diese Zertifikate werden verwendet, um die Netzwerkverbindung mit Ihren Geräten zu verschlüsseln.', 934: 'Befolgen Sie die Anweisungen für Ihr Betriebssystem. Wenn Sie Ihrer Root-Zertifizierungsstelle vertrauen, kann Ihr Gerät die Echtheit der verschlüsselten Kommunikation mit Ihrem Router überprüfen.', 935: 'Dies müssen Sie auf jedem Gerät wiederholen, mit dem Sie die StartWRT-Oberfläche aufrufen.', + 936: 'Derzeit läuft eine Sicherung. Eine Unterbrechung kann jetzt die Sicherung des gerade geschriebenen Dienstes beschädigen.', + 937: 'Auf Abschluss der Sicherung warten', + 938: 'Jetzt herunterfahren', + 939: 'Eine Sicherung läuft. Ihr Server wird nach deren Abschluss neu gestartet.', + 940: 'Eine Sicherung läuft. Ihr Server wird nach deren Abschluss heruntergefahren.', } satisfies i18n diff --git a/shared-libs/ts-modules/shared/src/i18n/dictionaries/en.ts b/shared-libs/ts-modules/shared/src/i18n/dictionaries/en.ts index e247195998..cb42213ce2 100644 --- a/shared-libs/ts-modules/shared/src/i18n/dictionaries/en.ts +++ b/shared-libs/ts-modules/shared/src/i18n/dictionaries/en.ts @@ -826,4 +826,9 @@ export const ENGLISH: Record = { 'Your router uses its Root CA to generate SSL/TLS certificates for itself. These certificates are then used to encrypt network traffic with your client devices.': 933, 'Follow instructions for your OS. By trusting your Root CA, your device can verify the authenticity of encrypted communications with your router.': 934, 'You will need to repeat this on every device you use to connect to the StartWRT UI.': 935, + 'A backup is currently running. Interrupting it now can corrupt the backup of the service being written.': 936, + 'Wait for backup to complete': 937, + 'Shut down now': 938, + 'A backup is running. Your server will restart when it finishes.': 939, + 'A backup is running. Your server will shut down when it finishes.': 940, } diff --git a/shared-libs/ts-modules/shared/src/i18n/dictionaries/es.ts b/shared-libs/ts-modules/shared/src/i18n/dictionaries/es.ts index 4e2efddb64..de08a757e5 100644 --- a/shared-libs/ts-modules/shared/src/i18n/dictionaries/es.ts +++ b/shared-libs/ts-modules/shared/src/i18n/dictionaries/es.ts @@ -825,4 +825,9 @@ export default { 933: 'Tu router usa su CA raíz para generar certificados SSL/TLS para sí mismo. Estos certificados se utilizan para cifrar el tráfico de red con tus dispositivos cliente.', 934: 'Sigue las instrucciones para tu sistema operativo. Al confiar en tu CA raíz, tu dispositivo puede verificar la autenticidad de las comunicaciones cifradas con tu router.', 935: 'Tendrás que repetir esto en cada dispositivo que uses para acceder a la interfaz de StartWRT.', + 936: 'Hay una copia de seguridad en curso. Interrumpirla ahora puede dañar la copia de seguridad del servicio que se está escribiendo.', + 937: 'Esperar a que termine la copia de seguridad', + 938: 'Apagar ahora', + 939: 'Hay una copia de seguridad en curso. Su servidor se reiniciará cuando termine.', + 940: 'Hay una copia de seguridad en curso. Su servidor se apagará cuando termine.', } satisfies i18n diff --git a/shared-libs/ts-modules/shared/src/i18n/dictionaries/fr.ts b/shared-libs/ts-modules/shared/src/i18n/dictionaries/fr.ts index ba5b41d132..39b0e03d3c 100644 --- a/shared-libs/ts-modules/shared/src/i18n/dictionaries/fr.ts +++ b/shared-libs/ts-modules/shared/src/i18n/dictionaries/fr.ts @@ -825,4 +825,9 @@ export default { 933: 'Votre routeur utilise son certificat racine pour générer des certificats SSL/TLS pour lui-même. Ces certificats servent ensuite à chiffrer le trafic réseau avec vos appareils clients.', 934: 'Suivez les instructions pour votre système d’exploitation. En donnant votre confiance au certificat racine, votre appareil pourra vérifier l’authenticité des communications chiffrées avec votre routeur.', 935: 'Vous devrez répéter cette opération sur chaque appareil utilisé pour accéder à l’interface de StartWRT.', + 936: 'Une sauvegarde est en cours. L’interrompre maintenant peut corrompre la sauvegarde du service en cours d’écriture.', + 937: 'Attendre la fin de la sauvegarde', + 938: 'Éteindre maintenant', + 939: 'Une sauvegarde est en cours. Votre serveur redémarrera une fois celle-ci terminée.', + 940: 'Une sauvegarde est en cours. Votre serveur s’éteindra une fois celle-ci terminée.', } satisfies i18n diff --git a/shared-libs/ts-modules/shared/src/i18n/dictionaries/pl.ts b/shared-libs/ts-modules/shared/src/i18n/dictionaries/pl.ts index ca54462d8d..c83ad82aeb 100644 --- a/shared-libs/ts-modules/shared/src/i18n/dictionaries/pl.ts +++ b/shared-libs/ts-modules/shared/src/i18n/dictionaries/pl.ts @@ -825,4 +825,9 @@ export default { 933: 'Twój router używa swojego głównego CA do generowania certyfikatów SSL/TLS dla siebie. Te certyfikaty są następnie używane do szyfrowania ruchu sieciowego z Twoimi urządzeniami klienckimi.', 934: 'Postępuj zgodnie z instrukcjami dla swojego systemu operacyjnego. Zaufanie głównemu CA pozwala Twojemu urządzeniu weryfikować autentyczność szyfrowanej komunikacji z routerem.', 935: 'Będziesz musiał powtórzyć tę czynność na każdym urządzeniu, którego używasz do łączenia się z interfejsem StartWRT.', + 936: 'Trwa tworzenie kopii zapasowej. Przerwanie jej teraz może uszkodzić kopię zapasową aktualnie zapisywanej usługi.', + 937: 'Poczekaj na zakończenie kopii zapasowej', + 938: 'Wyłącz teraz', + 939: 'Trwa tworzenie kopii zapasowej. Serwer zostanie ponownie uruchomiony po jej zakończeniu.', + 940: 'Trwa tworzenie kopii zapasowej. Serwer zostanie wyłączony po jej zakończeniu.', } satisfies i18n diff --git a/shared-libs/ts-modules/start-core/lib/osBindings/PowerAction.ts b/shared-libs/ts-modules/start-core/lib/osBindings/PowerAction.ts new file mode 100644 index 0000000000..008c8992db --- /dev/null +++ b/shared-libs/ts-modules/start-core/lib/osBindings/PowerAction.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type PowerAction = 'restart' | 'shutdown' diff --git a/shared-libs/ts-modules/start-core/lib/osBindings/ServerStatus.ts b/shared-libs/ts-modules/start-core/lib/osBindings/ServerStatus.ts index 94cf76c898..783ce3296a 100644 --- a/shared-libs/ts-modules/start-core/lib/osBindings/ServerStatus.ts +++ b/shared-libs/ts-modules/start-core/lib/osBindings/ServerStatus.ts @@ -1,5 +1,6 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { FullProgress } from './FullProgress' +import type { PowerAction } from './PowerAction' import type { RestartReason } from './RestartReason' export type ServerStatus = { @@ -8,4 +9,9 @@ export type ServerStatus = { shuttingDown: boolean restarting: boolean restart: RestartReason | null + /** + * A restart or shutdown that was asked for while a backup was running, and + * which StartOS carries out once the backup finishes. + */ + deferredPowerAction: PowerAction | null } diff --git a/shared-libs/ts-modules/start-core/lib/osBindings/ShutdownParams.ts b/shared-libs/ts-modules/start-core/lib/osBindings/ShutdownParams.ts index 21a7d7c0d9..c51d7df2ce 100644 --- a/shared-libs/ts-modules/start-core/lib/osBindings/ShutdownParams.ts +++ b/shared-libs/ts-modules/start-core/lib/osBindings/ShutdownParams.ts @@ -6,6 +6,11 @@ export type ShutdownParams = { * frontend omits this and gets an immediate reply). Cleared with * `--nowait`. The wait can't outlive the webserver teardown that follows * container shutdown, so the connection drops once services are stopped. + * A deferred action replies immediately, before teardown begins. */ wait: boolean + /** + * Interrupt a running backup instead of waiting for it to finish. + */ + force: boolean } diff --git a/shared-libs/ts-modules/start-core/lib/osBindings/index.ts b/shared-libs/ts-modules/start-core/lib/osBindings/index.ts index 182f262214..522ee3778d 100644 --- a/shared-libs/ts-modules/start-core/lib/osBindings/index.ts +++ b/shared-libs/ts-modules/start-core/lib/osBindings/index.ts @@ -230,6 +230,7 @@ export { Percentage } from './Percentage' export { PluginHostnameInfo } from './PluginHostnameInfo' export { PluginId } from './PluginId' export { PortForward } from './PortForward' +export { PowerAction } from './PowerAction' export { PreDownloadAlert } from './PreDownloadAlert' export { PreDownloadAlertWhen } from './PreDownloadAlertWhen' export { Progress } from './Progress'