diff --git a/.githooks/lib/fog-version.sh b/.githooks/lib/fog-version.sh index 6c0b8f6d8e..dbd2bec985 100644 --- a/.githooks/lib/fog-version.sh +++ b/.githooks/lib/fog-version.sh @@ -11,8 +11,17 @@ # (locally or in CI) without leaving a dirty working tree behind. Pair with # apply-fog-version.sh to actually write the result somewhere. # -# Usage: fog-version.sh [branch-name] +# Usage: fog-version.sh [branch-name] [mode] # branch-name defaults to the currently checked out branch. +# mode 0 (default) report honestly; if drifted, print the version the +# next commit should carry. +# 1 a commit is being written right now - always print the +# version that commit should carry. Used by pre-commit. +# head print what the commit that ALREADY EXISTS should carry, with +# no +1 - i.e. verify rather than write. +# +# No local hook calls this any more: FOG_VERSION is written only on a base +# branch, after a merge, by CI. See the header of .githooks/pre-commit. set -e @@ -20,9 +29,13 @@ project_dir=$(git rev-parse --show-toplevel) system_file="$project_dir/packages/web/lib/fog/system.class.php" gitbranch="${1:-$(git branch --show-current)}" -local="${2:-0}" +mode="${2:-0}" -gitcom=$(git rev-list --tags --no-walk --max-count=1) +# Release tags only. A release tag is the version string itself (1.5.10.2253), +# so it starts with a digit. Any other tag -- archive/feature-fog2-gui, pushed +# on 2026-09-06 -- would otherwise become the base version whenever it is the +# newest tag, and its slash then breaks the sed in apply-fog-version.sh. +gitcom=$(git rev-list --tags='[0-9]*' --no-walk --max-count=1) git fetch origin master:master 2>/dev/null || true gitcount=$(git rev-list master..HEAD --count) @@ -46,13 +59,13 @@ compute_version() { case "$branchon" in dev) - tagversion=$(git describe --tags "$gitcom") + tagversion=$(git describe --tags --match '[0-9]*' "$gitcom") baseversion=${tagversion%.*} trunkversion="${baseversion}.${count}" channel="Patches" ;; stable) - tagversion=$(git describe --tags "$gitcom") + tagversion=$(git describe --tags --match '[0-9]*' "$gitcom") baseversion=${tagversion%.*} count=$(git rev-list master..dev-branch --count) # Get the gitcount from dev-branch instead trunkversion="${baseversion}.${count}" @@ -85,6 +98,16 @@ compute_version() { # new commit right now. compute_version "$gitcount" +# rc is the one branch type with no count-verifiable answer: it increments +# off whatever suffix is already committed rather than off a commit count, +# so the pass above always returns "one more than what is there". For the +# modes that are about to write a commit that is exactly right. For head +# mode, which asks whether the committed value is already correct, it would +# be a permanent false positive - so there, the committed value stands. +if [ "$mode" = "head" ] && [ "$branchon" = "rc" ]; then + trunkversion="$current_version" +fi + drifted=false [ "$trunkversion" != "$current_version" ] && drifted=true # dev-branch and stable deliberately carry no FOG_CHANNEL line at all (a @@ -94,17 +117,26 @@ drifted=false if { [ -n "$current_channel" ] && [ "$channel" != "$current_channel" ]; }; then drifted=true fi -if [ "$local" -eq 1 ]; then +if [ "$mode" = "1" ]; then drifted=true fi -if [ "$drifted" = true ] || [ "$local" -eq 1 ]; then +# head mode stops here. Every other mode answers "what should the NEXT +# commit say"; head answers "what should the commit that already exists +# say", so adding 1 for a commit nobody is writing would defeat the whole +# point of it. +if [ "$mode" != "head" ] && [ "$drifted" = true ]; then # What's committed disagrees, so whatever calls this script is about # to add one more real commit to this branch to fix it. Recompute with # gitcount+1 - the count that will actually be true once that commit # exists - so the fix is correct the instant it lands instead of being # wrong by exactly the commit that made it. Without this, the very next # check finds "drift" again and fixes it again, forever. + # + # This is also why mode=1 lands one too high on an --amend: an amend + # REPLACES HEAD rather than extending it, so the +1 counts a commit that + # will never exist. That was the standing hazard of stamping a version + # from a local hook, and is one of the reasons no local hook does it now. compute_version "$((gitcount + 1))" fi diff --git a/.githooks/pre-commit b/.githooks/pre-commit index ae2d0f95f2..628c7e5871 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -57,22 +57,48 @@ updateLanguage psrfix +# FOG_VERSION IS NOT STAMPED HERE ANY MORE. That is deliberate. +# +# The version is `git rev-list master..HEAD --count` put through +# .githooks/lib/fog-version.sh -- a function of the commit graph. Stamping it +# into a tracked line of packages/web/lib/fog/system.class.php on every commit +# meant that any two branches open at once wrote DIFFERENT values to the SAME +# line, so merging the base back into either one conflicted, every time, by +# construction. Not a race: there is no ordering of commits that avoids it. +# +# It is also what left a dangling staged system.class.php bump behind after a +# commit, which this branch's CLAUDE.md used to have to warn about. +# +# The version is now written in exactly one place -- on the base branch, after +# a merge, by .github/workflows/sync-generated-files.yml -- where there is only +# ever one writer and so nothing to conflict with. fog-workflows' daily sweep +# still covers direct pushes and rc-*/feature-* branches. +# +# The consequence, stated so it is not rediscovered as a bug: a commit made +# locally leaves FOG_VERSION reading whatever the last merge into the base +# branch set. It is BEHIND on a feature branch, on purpose. Do not "fix" that +# by putting the stamp back; a version that is a few commits stale costs a +# wrong number in a build string, and stamping it per-commit costs a +# hand-resolved conflict on every open pull request every time anything +# merges. +# +# .githooks/pre-push, which refused a push whose committed version disagreed +# with the branch, was removed in the same change: with nothing stamping a +# version there is nothing for it to guard, and it would have blocked every +# push instead. +# +# Ported from working-1.6 (GH-1510), where the bug was diagnosed. -# Define the path to the system file -system_file="$project_dir/packages/web/lib/fog/system.class.php" - -# Recompute FOG_VERSION/FOG_CHANNEL for the current branch. The formula is -# shared with fog-workflows' CI check (see .githooks/lib/fog-version.sh) so -# a local commit and CI's periodic sweep never compute two different -# answers for the same branch state. -gitbranch=$(git branch --show-current) -fog_version=$(sh "$project_dir/.githooks/lib/fog-version.sh" $gitbranch 1) -if [ "$(printf '%s\n' "$fog_version" | sed -n '3p')" = "true" ]; then - sh "$project_dir/.githooks/lib/apply-fog-version.sh" \ - "$(printf '%s\n' "$fog_version" | sed -n '1p')" \ - "$(printf '%s\n' "$fog_version" | sed -n '2p')" - - # Add the modified system file to the staging area - git add "$system_file" -fi - +# GH-1581: the hook's exit status is the status of the last command, which was +# psrfix -- and psrfix returns 1 when php-cs-fixer is not installed, via +# require_tools. So a machine without the optional tools printed "the commit +# will proceed WITHOUT those changes" and then blocked the commit, with git +# exiting 1 and saying nothing. +# +# Every step above is deliberately advisory: require_tools exists so a missing +# formatter is a skip, not a failure, and CI regenerates all of it. Nothing in +# this hook is a gate. Say so explicitly rather than leaving the status to +# whichever helper happens to run last -- post-commit already ends this way. +# +# Ported from working-1.6, where it was diagnosed and fixed first. +exit 0 diff --git a/.github/workflows/sync-generated-files.yml b/.github/workflows/sync-generated-files.yml new file mode 100644 index 0000000000..f23d3819dd --- /dev/null +++ b/.github/workflows/sync-generated-files.yml @@ -0,0 +1,94 @@ +name: Sync generated files and version + +# Merge-triggered trigger stub. All the logic lives in FOGProject/fog-workflows' +# update-lang-fix-psr-and-sync-version.yml; this file exists only because GitHub +# Actions has no cross-repo merge trigger, so something has to live here to react +# to a merge. +# +# THIS IS THE ONLY THING THAT WRITES FOG_VERSION ON A MERGE, and since GH-1510 +# it is the only thing that writes it on this branch at all outside the daily +# sweep. .githooks/pre-commit used to stamp a version on every local commit; +# that made every branch open at the same time hold a different value on the +# same tracked line, so each merge left the others with a hand-resolved +# conflict on system.class.php. Its header has the full reasoning. +# +# A client-side hook could not have covered this case anyway: a PR merged +# through GitHub's web UI (squash, merge commit or rebase) runs no local hook, +# so FOG_VERSION would go stale until the daily sweep fires at 10:10 UTC. This +# closes that window. +# +# WHY `pull_request`, AND NOT `pull_request_target` +# +# `pull_request_target` looks like the right answer -- it is the variant that +# gets secrets on fork PRs -- and the first version of this file used it. It +# never fired once. GitHub reads a `pull_request_target` workflow from the +# repository's DEFAULT branch (`stable` here), not from the PR's base branch, so +# a copy living on working-1.6 and dev-branch is simply never consulted: the +# workflow did not even appear in the Actions list, and four PRs merged into +# working-1.6 without it running. +# +# `pull_request` is read from the base branch instead, so this file works where +# it actually lives. Do not "fix" it back to `pull_request_target` without also +# putting the file on `stable` -- and note that then only stable's copy would +# execute, which makes editing the version on this branch a no-op. +# +# WHY NOT `push` +# +# That distinction is the whole safety argument, not caution. The sweep pushes +# its fixup commit straight to the branch, and a direct push is not a PR merge, +# so it cannot re-fire this stub. The 2026-07-28 runaway that put ~30 commits on +# dev-branch in about 20 minutes was a push-triggered stub doing exactly that. +# The schedule in fog-workflows stays exactly as it is, and remains the backstop +# for direct pushes and for rc-*/feature-* branches. +# +# WHY THE SAME-REPO GUARD +# +# `pull_request` withholds secrets from fork PRs, and the reusable workflow needs +# FOG_WORKFLOWS_PRIVATE_KEY to mint its App token -- so on a fork PR it would +# fail rather than work. Skipping is right: a merged fork PR is picked up by the +# daily sweep, exactly as a direct push already is. Better a gap the schedule +# already covers than a red X on every external contribution. +# +# One file, identical on every branch that carries it, for the same reason +# tests.yml is: fixing it should not mean editing it on three branches. Each +# branch's copy only ever acts on merges into that branch, and the allowlist +# below is what scopes it. + +on: + pull_request: + types: [closed] + +concurrency: + group: fog-sync-on-merge-${{ github.event.pull_request.base.ref }} + cancel-in-progress: false + +jobs: + sync: + # An allowlist, not "not stable". Branches cut from working-1.6 inherit this + # file, and an rc-*/feature-* branch must NOT be merge-synced: fog-version.sh + # reports drift on every run for rc (it increments off the committed suffix + # rather than a commit count), so a per-merge sync would bump the RC suffix + # on every merge. Those stay on the daily sweep. stable is excluded because + # its version is owned entirely by fog-workflows' stable-releases.yml. + # + # `closed` fires on abandoned PRs too, hence the merged check -- and the + # reusable workflow uses its `branch` input verbatim, with no validation + # against its own watched list, so constraining it is this file's job. + if: >- + github.event.pull_request.merged == true + && github.event.pull_request.head.repo.full_name == github.repository + && contains(fromJson('["working-1.6", "dev-branch"]'), github.event.pull_request.base.ref) + + # Least privilege, stated rather than inherited. Everything the reusable + # workflow writes -- the commit, the push, and the version badge -- uses a + # GitHub App token, so nothing on this path needs a writable GITHUB_TOKEN. A + # called workflow can never hold more permission than its caller, so leaving + # this to the repo-wide default would make the effective permission whatever + # that happens to be. See fos' create_release.yml for the same reasoning. + permissions: + contents: read + + uses: FOGProject/fog-workflows/.github/workflows/update-lang-fix-psr-and-sync-version.yml@main + with: + branch: ${{ github.event.pull_request.base.ref }} + secrets: inherit diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000000..da9ab5ad12 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,23 @@ +name: Tests + +# A pull_request trigger can only fire from the repository the PR is opened +# against, so this stub has to live here even though every other fogproject +# workflow lives in FOGProject/fog-workflows. It is deliberately six lines: +# all of the logic is in the reusable workflow, so fixing the runner does not +# mean editing this file on three branches. +# +# For pull_request events GitHub reads workflows from the merge of head into +# base, so one copy on each BASE branch (working-1.6, dev-branch, stable) +# covers every PR opened against it -- the contributor's branch needs +# nothing. +# +# Not a push trigger, and not because of caution: the runaway that put ~30 +# commits on dev-branch in 20 minutes happened because the triggered +# workflow pushed a commit back here. This one only reads. + +on: + pull_request: + +jobs: + suite: + uses: FOGProject/fog-workflows/.github/workflows/fogproject-tests.yml@main diff --git a/CLAUDE.md b/CLAUDE.md index 563f3da7b6..5059204990 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -204,11 +204,29 @@ self::$HookManager->processEvent('EVENT_NAME', array('data' => &$data)); ## Pre-commit hook (IMPORTANT — explains "files I didn't touch" in commits) -`core.hooksPath` is `.githooks/`, so `.githooks/pre-commit` runs on **every** `git commit` (and `pre-merge-commit` delegates to it). It auto-modifies and `git add`s files beyond what you staged — this is expected, not a bug. Do **not** revert these. It does three things: +`core.hooksPath` is `.githooks/`, so `.githooks/pre-commit` runs on **every** `git commit` (and `pre-merge-commit` delegates to it). It auto-modifies and `git add`s files beyond what you staged — this is expected, not a bug. Do **not** revert these. It does two things: 1. **`updateLanguage()`** — regenerates `management/languages/messages.pot` via `xgettext`, sorts with `msgcat`, then `msgmerge`-updates every `.po`. Adds the whole `languages/` dir. Skipped if those tools aren't installed. 2. **`psrfix()`** — runs `php-cs-fixer fix packages/web --rules=@PSR2` and **`git add packages/web`** unconditionally. Two consequences: your code may be auto-reformatted to PSR-2, and **any other dirty file under `packages/web/` gets swept into your commit** regardless of what you staged. Commit files outside `packages/web/` (like this `CLAUDE.md`) separately if you need them isolated. -3. **Version bump** — derives a version from the branch name + commit count and rewrites `FOG_VERSION`/`FOG_CHANNEL` in `packages/web/lib/fog/system.class.php`. On `dev`/`stable` branches the channel is `Patches`. This step also tends to leave a **dangling staged `system.class.php`** bump after the commit; discard it with `git checkout -- packages/web/lib/fog/system.class.php` if you don't want it in the next commit. +It used to do a third — a **version bump**, rewriting `FOG_VERSION`/`FOG_CHANNEL` in `packages/web/lib/fog/system.class.php` from the branch name and the commit count, which is also what left a **dangling staged `system.class.php`** behind after every commit. That was removed, along with `.githooks/pre-push` (which existed only to refuse a push whose committed version had drifted). + +### Why `FOG_VERSION` is not written on a branch + +`FOG_VERSION` is `git rev-list master..HEAD --count` — a property of the commit +graph — but it is *stored* on a single tracked line of +`packages/web/lib/fog/system.class.php`. Writing it per-commit made every branch +open at the same time hold a **different value on the same line**, so each merge +left the others with a hand-resolved conflict on `system.class.php` before they +could be updated and re-tested. That is structural, not a race. + +The version now has exactly one writer: `.github/workflows/sync-generated-files.yml`, +on the base branch, after a merge — plus fog-workflows' daily sweep for direct +pushes and `rc-*`/`feature-*` branches. A feature branch's `FOG_VERSION` is +*behind* while it is open, on purpose. The channel is `Patches` on +`dev`/`stable`. + +Diagnosed on `working-1.6` in GH-1510; this branch has the same defect and takes +the same fix. --- diff --git a/bin/fog-node-key.php b/bin/fog-node-key.php new file mode 100644 index 0000000000..25b17025b7 --- /dev/null +++ b/bin/fog-node-key.php @@ -0,0 +1,239 @@ + + * + * Running daemons cache settings for FOG_SETTING_CACHE_TTL (300s default), + * so a change can take up to five minutes to reach one that is already up. + * + * PHP version 7.4+ + * + * @category Utility + * @package FOGProject + * @author Tom Elliott + * @license http://opensource.org/licenses/gpl-3.0 GPLv3 + * @link https://fogproject.org + */ + +/** + * Loads the DB constants out of a FOG install's config.class.php. + * + * Same reader as bin/schema-manifest.php, and for the same reason: the + * constants are the only thing needed and parsing them is cheaper and far + * more robust than booting the application to reach them. + * + * @param string $root The web root of the FOG install. + * + * @return PDO + */ +function fogNodeKeyConnect($root) +{ + $config = rtrim($root, '/') . '/lib/fog/config.class.php'; + if (!file_exists($config)) { + fwrite(STDERR, "No config.class.php under $root\n"); + fwrite(STDERR, "Pass the web root with --web /path/to/fog\n"); + exit(1); + } + $src = file_get_contents($config); + $vals = []; + foreach (['HOST', 'NAME', 'USERNAME', 'PASSWORD'] as $key) { + if (preg_match( + "/define\(\s*'DATABASE_$key'\s*,\s*'(.*?)'\s*\)/s", + $src, + $m + )) { + $vals[$key] = $m[1]; + } + } + if (!isset($vals['NAME'])) { + fwrite(STDERR, "Could not read DATABASE_* from $config\n"); + exit(1); + } + return new \PDO( + sprintf( + 'mysql:host=%s;dbname=%s', + $vals['HOST'] ?: 'localhost', + $vals['NAME'] + ), + $vals['USERNAME'], + $vals['PASSWORD'], + [\PDO::ATTR_ERRMODE => \PDO::ERRMODE_EXCEPTION] + ); +} + +$roots = [ + '/var/www/html/fog', + '/var/www/fog', +]; +$root = ''; +$set = null; +$show = false; + +$argvv = array_slice($argv, 1); +for ($i = 0; $i < count($argvv); $i++) { + switch ($argvv[$i]) { + case '--web': + $root = isset($argvv[$i + 1]) ? $argvv[++$i] : ''; + break; + case '--set': + $set = isset($argvv[$i + 1]) ? $argvv[++$i] : ''; + break; + case '--show': + $show = true; + break; + case '-h': + case '--help': + fwrite( + STDOUT, + "Usage:\n" + . " php bin/fog-node-key.php [--web ] [--show]\n" + . " php bin/fog-node-key.php [--web ] --set \n\n" + . "Reads or sets FOG_NODE_API_KEY. Needed only on a peer that\n" + . "runs its own FOG database; set it to the value held in the\n" + . "master's storage node record for this peer.\n" + ); + exit(0); + default: + fwrite(STDERR, "Unknown argument: {$argvv[$i]}\n"); + exit(1); + } +} + +// Validated here rather than next to the write, so a bad invocation is +// rejected on any machine -- before locating a web root, before opening a +// connection, and without needing either to be working. +if (null !== $set) { + $set = trim((string)$set); + if ($set === '') { + fwrite(STDERR, "--set needs a value\n"); + exit(1); + } + // The master generates these as 32 random bytes hex encoded. Not + // enforced, because an administrator is entitled to choose their own + // and a length rule here would be a second opinion about a value the + // other end already accepted -- but short enough to be a typo is worth + // saying out loud. + if (strlen($set) < 32) { + fwrite( + STDERR, + 'Warning: that key is ' . strlen($set) . " characters. FOG\n" + . "generates 64. Setting it anyway.\n" + ); + } +} + +if ($root === '') { + foreach ($roots as $candidate) { + if (file_exists($candidate . '/lib/fog/config.class.php')) { + $root = $candidate; + break; + } + } +} +if ($root === '') { + fwrite( + STDERR, + "Could not find a FOG web root. Pass one with --web /path/to/fog\n" + ); + exit(1); +} + +$db = fogNodeKeyConnect($root); + +if (null === $set) { + $stmt = $db->prepare( + 'SELECT `settingValue` FROM `globalSettings` WHERE `settingKey` = ?' + ); + $stmt->execute(['FOG_NODE_API_KEY']); + $row = $stmt->fetch(\PDO::FETCH_ASSOC); + $value = $row ? trim((string)$row['settingValue']) : ''; + if ($value === '') { + fwrite(STDOUT, "FOG_NODE_API_KEY: (not set)\n"); + fwrite( + STDOUT, + "This install has never signed a request, or is a peer that\n" + . "only receives them. If it is a peer, set this to the value\n" + . "in the master's storage node record for it.\n" + ); + exit(0); + } + // Printed in full on purpose: the only reason to run --show is to + // compare or copy the value, and a truncated secret cannot be either. + // This needs shell access to the server already. + fwrite(STDOUT, "FOG_NODE_API_KEY: $value\n"); + exit(0); +} + +// INSERT ... ON DUPLICATE KEY UPDATE rather than an UPDATE, because the row +// is normally absent on the machine that needs this run -- see the header. +// The UNIQUE KEY on settingKey is what makes the upsert well defined. +$stmt = $db->prepare( + 'INSERT INTO `globalSettings` ' + . '(`settingKey`, `settingDesc`, `settingValue`, `settingCategory`) ' + . 'VALUES (?, ?, ?, ?) ' + . 'ON DUPLICATE KEY UPDATE `settingValue` = VALUES(`settingValue`)' +); +$stmt->execute( + [ + 'FOG_NODE_API_KEY', + 'Shared secret FOG signs its own server-to-server requests with. ' + . 'On a peer running its own database this must match the Node API ' + . 'Signing Key held in the master storage node record for this peer.', + $set, + 'FOG Storage Nodes', + ] +); + +$stmt = $db->prepare( + 'SELECT `settingValue` FROM `globalSettings` WHERE `settingKey` = ?' +); +$stmt->execute(['FOG_NODE_API_KEY']); +$row = $stmt->fetch(\PDO::FETCH_ASSOC); +// Read back rather than trusting the write: this is the whole point of the +// utility, and a silent no-op here would look identical to success and then +// fail later as an unexplained 401 on the node. +if (!$row || trim((string)$row['settingValue']) !== $set) { + fwrite(STDERR, "The key did not land. Nothing has been changed.\n"); + exit(1); +} +fwrite(STDOUT, "FOG_NODE_API_KEY set on $root\n"); +fwrite( + STDOUT, + "Running daemons cache settings for up to FOG_SETTING_CACHE_TTL\n" + . "(300s by default), so give them that long to pick it up.\n" +); +exit(0); diff --git a/bin/migrate-menu-translations.php b/bin/migrate-menu-translations.php new file mode 100644 index 0000000000..d376f3dbdd --- /dev/null +++ b/bin/migrate-menu-translations.php @@ -0,0 +1,534 @@ + + * @license http://opensource.org/licenses/gpl-3.0 GPLv3 + * @link https://fogproject.org + */ + +$write = in_array('--write', array_slice($argv, 1), true); +$root = dirname(__DIR__); +$langDir = $root . '/packages/web/management/languages'; + +// node => [singular noun msgid, list msgid, add msgid]. The nouns are the +// msgids _buildSubMenuItems() fed to sprintf -- ucfirst() of the node name -- +// so composing with them reproduces the old label exactly. +$nodes = [ + 'Group' => ['Group', 'List All Groups', 'Create New Group'], + 'Host' => ['Host', 'List All Hosts', 'Create New Host'], + 'Image' => ['Image', 'List All Images', 'Create New Image'], + 'Printer' => ['Printer', 'List All Printers', 'Create New Printer'], + 'Snapin' => ['Snapin', 'List All Snapins', 'Create New Snapin'], + 'StorageGroup' => ['StorageGroup', 'List All Storage Groups', 'Create New Storage Group'], + 'StorageNode' => ['StorageNode', 'List All Storage Nodes', 'Create New Storage Node'], + 'User' => ['User', 'List All Users', 'Create New User'], +]; + +/** + * Every msgid => msgstr in a .po, single- and multi-line forms both. + * + * Not a general gettext parser and does not need to be: it reads the two + * shapes msgcat emits and ignores obsolete (#~) entries, which is the whole + * of what these files contain. + * + * @param string $path .po file + * + * @return array + */ +function poRead($path) +{ + $out = []; + $lines = file($path, FILE_IGNORE_NEW_LINES); + if (false === $lines) { + return $out; + } + $id = ''; + $buf = ''; + $in = ''; + // Written as a flat loop rather than with a flush closure. A closure + // capturing $id/$in by reference is correct at runtime but opaque to + // static analysis -- phpstan types the captured variables from their + // INITIALIZERS, so every comparison inside reads as always-false and the + // build fails on five findings that are not bugs. Inlining the two-line + // flush costs a repetition and keeps the file analyzable. + foreach ($lines as $line) { + $t = trim($line); + if ('' === $t || '#' === substr($t, 0, 1)) { + // Comments and obsolete (#~) entries alike: an obsolete entry is + // not compiled and is not shown to anyone, so it is not a value + // this script may read a translation out of. + continue; + } + $start = ''; + if (0 === strpos($t, 'msgid ')) { + $start = 'msgid'; + $frag = substr($t, 6); + } elseif (0 === strpos($t, 'msgstr ')) { + $start = 'msgstr'; + $frag = substr($t, 7); + } + if ('' !== $start) { + if ('msgstr' === $in && '' !== $id) { + $out[$id] = $buf; + } elseif ('msgid' === $in) { + $id = $buf; + } + $in = $start; + $buf = poUnquote((string)$frag); + continue; + } + if ('"' === substr($t, 0, 1) && '' !== $in) { + $buf .= poUnquote($t); + } + } + if ('msgstr' === $in && '' !== $id) { + $out[$id] = $buf; + } + return $out; +} + + +/** + * The msgids in a .po whose entry is flagged `#, fuzzy`. + * + * A fuzzy entry is msgmerge's GUESS, carried over from a similar msgid and + * never confirmed by a person. msgfmt excludes fuzzy entries from the compiled + * .mo, so nothing in one has ever been shown to a user -- which is exactly why + * they are so wrong here. At HEAD every single `Create New X` entry in every + * catalog is fuzzy, and de_DE's guesses include "Neuen Drucker erstellen" + * (Create New PRINTER) under msgid `Create New Storage Node` and "Neuen + * Schlüssel erstellen" (Create New KEY) under `Create New Group`. + * + * That matters because this change makes those msgids live for the first time: + * the labels used to be composed at runtime and never looked these up. Treating + * a fuzzy msgstr as a real translation would therefore not preserve anything -- + * it would PROMOTE nine catalogs' worth of unreviewed guesses into text users + * finally see. So the seeder treats fuzzy as untranslated and composes over it. + * + * @param string $path .po file + * + * @return array msgid => true + */ +function poFuzzy($path) +{ + $out = []; + $lines = file($path, FILE_IGNORE_NEW_LINES); + if (false === $lines) { + return $out; + } + $fuzzy = false; + foreach ($lines as $line) { + $t = trim($line); + if (0 === strpos($t, '#,')) { + $fuzzy = (false !== strpos($t, 'fuzzy')); + continue; + } + if (preg_match('/^msgid "(.*)"$/', $t, $m)) { + if ($fuzzy) { + $out[poUnquote('"' . $m[1] . '"')] = true; + } + $fuzzy = false; + continue; + } + if ('' === $t) { + $fuzzy = false; + } + } + return $out; +} + +/** + * The value of one quoted .po string fragment. + * + * @param string $s fragment including its surrounding quotes + * + * @return string + */ +function poUnquote($s) +{ + $s = trim($s); + if ('"' !== substr($s, 0, 1)) { + return ''; + } + $s = substr($s, 1, -1); + return str_replace( + ['\\\\n', '\\\\t', '\\\\"', '\\\\\\\\'], + ["\n", "\t", '"', '\\\\'], + $s + ); +} + +/** + * A .po-safe quoted literal. + * + * @param string $s raw value + * + * @return string + */ +function poQuote($s) +{ + return '"' . str_replace( + ['\\\\', '"', "\n", "\t"], + ['\\\\\\\\', '\\\\"', '\\\\n', '\\\\t'], + $s + ) . '"'; +} + +/** + * sprintf that reports failure instead of throwing or warning. + * + * PHP 8 throws on a bad specifier, 7.4 warns and returns false. Both mean + * "this catalog's format string is unusable", and both must skip. + * + * @param string $fmt format string + * @param string $val substitution + * + * @return string|null null when the format is unusable + */ +function safeFormat($fmt, $val) +{ + if (false === strpos($fmt, '%s')) { + return null; + } + try { + $r = @sprintf($fmt, $val); + } catch (\Throwable $e) { + return null; + } + return '' === (string)$r ? null : (string)$r; +} + + +/** + * Sets each msgid's msgstr in a .po, in place. + * + * Appending instead of rewriting is what the first cut of this script did, + * and msgfmt rejected all nine catalogs with `duplicate message definition`. + * Two reasons a msgid that "needs seeding" is already in the file: + * + * - an entry with an EMPTY msgstr is still an entry; + * - a msgid that fell out of the sources is kept as an OBSOLETE entry, + * commented with #~. That is exactly what gettext keeps them for -- so a + * returning string can be revived rather than retyped -- and these + * strings are returning. fr_FR carries `#~ msgid "List All Roles"`. + * + * So: rewrite a live entry where one exists, revive an obsolete one where it + * does not, and only append when the msgid is genuinely absent. A `#, fuzzy` + * marker immediately above is dropped in both cases -- the value being + * written is derived deliberately, and leaving it flagged fuzzy invites + * msgmerge to discard it again. + * + * @param string $path .po file + * @param array $set msgid => msgstr + * + * @return void + */ +function poWrite($path, array $set) +{ + $lines = explode("\n", file_get_contents($path)); + $out = []; + $seen = []; + $n = count($lines); + for ($i = 0; $i < $n; $i++) { + $line = $lines[$i]; + // Order matters: preg_match EMPTIES $m when it fails, so testing for + // an obsolete entry after a live one has already matched would throw + // the live capture away and send every live msgid down the append + // path instead -- which is how the first cut of this produced nine + // catalogs full of duplicate definitions. + $live = preg_match('/^msgid "(.*)"$/', $line, $m); + $dead = $live ? 0 : preg_match('/^#~[ \t]*msgid "(.*)"$/', $line, $m); + if ((!$live && !$dead) || !array_key_exists($m[1], $set)) { + $out[] = $line; + continue; + } + $prefix = $dead ? '/^#~[ \t]*/' : null; + $j = $i + 1; + // Skip to the end of this entry's msgstr, continuation lines included. + while ($j < $n + && !preg_match($dead ? '/^#~[ \t]*msgstr/' : '/^msgstr/', $lines[$j]) + ) { + $j++; + } + if ($j < $n) { + $j++; + while ($j < $n + && preg_match( + $dead ? '/^#~[ \t]*"/' : '/^"/', + $lines[$j] + ) + ) { + $j++; + } + } + while (count($out) + && 0 === strpos(end($out), '#,') + && false !== strpos(end($out), 'fuzzy') + ) { + array_pop($out); + } + $out[] = 'msgid ' . poQuote($m[1]); + $out[] = 'msgstr ' . poQuote($set[$m[1]]); + $seen[$m[1]] = true; + $i = $j - 1; + unset($prefix); + } + $text = rtrim(implode("\n", $out), "\n") . "\n"; + $missing = array_diff_key($set, $seen); + if (count($missing)) { + $text .= "\n#. GH-435: seeded from this catalog's own \"List All %s\" /\n" + . "#. \"Create New %s\" and noun, so the label renders exactly as it\n" + . "#. did before those format strings became whole phrases.\n"; + ksort($missing); + foreach ($missing as $msgid => $msgstr) { + $text .= "\nmsgid " . poQuote($msgid) . "\nmsgstr " . poQuote($msgstr) . "\n"; + } + } + file_put_contents($path, $text); +} + + +// Correct French for every per-node label. Written out rather than derived: +// agreement is the entire point of GH-435, and no rule generates it from the +// English. `machine` and `image` are feminine (toutes les / une nouvelle), +// `utilisateur` is masculine but vowel-initial (nouvel, not nouveau) -- the +// three cases the reporter cited, in that order. +$hand = []; +$hand['fr_FR'] = [ + 'List All Groups' => 'Lister tous les groupes', + 'Create New Group' => 'Créer un nouveau groupe', + 'List All Hosts' => 'Lister toutes les machines', + 'Create New Host' => 'Créer une nouvelle machine', + 'List All Images' => 'Lister toutes les images', + 'Create New Image' => 'Créer une nouvelle image', + 'List All Printers' => 'Lister toutes les imprimantes', + 'Create New Printer' => 'Créer une nouvelle imprimante', + 'List All Snapins' => 'Lister tous les snapins', + 'Create New Snapin' => 'Créer un nouveau snapin', + 'List All Storage Groups' => 'Lister tous les groupes de stockage', + 'Create New Storage Group' => 'Créer un nouveau groupe de stockage', + 'List All Storage Nodes' => 'Lister tous les nœuds de stockage', + 'Create New Storage Node' => 'Créer un nouveau nœud de stockage', + 'List All Users' => 'Lister tous les utilisateurs', + 'Create New User' => 'Créer un nouvel utilisateur', + // Not menu labels, but the same catalog damage and the same one-line fix. + // `Site` read "minutes" and `Role` read "Nom du module" -- which is how + // "Lister tous les minutess" was being generated at runtime while this + // was still composed. Corrected here because the composed form is what + // made them invisible; they are wrong wherever else they are used too. +]; + +// Correct Spanish. Unlike working-1.6 this branch's bare nouns are all sound, +// so only the composed labels need writing out. `imagen` and `impresora` are +// feminine (todas las / nueva); the rest are masculine. `equipo` rather than +// `anfitrion` because that is the word this catalog already uses for Host. +$hand['es_ES'] = [ + 'List All Groups' => 'Listar todos los grupos', + 'Create New Group' => 'Crear nuevo grupo', + 'List All Hosts' => 'Listar todos los equipos', + 'Create New Host' => 'Crear nuevo equipo', + 'List All Images' => 'Listar todas las imágenes', + 'Create New Image' => 'Crear nueva imagen', + 'List All Printers' => 'Listar todas las impresoras', + 'Create New Printer' => 'Crear nueva impresora', + 'List All Snapins' => 'Listar todos los snapins', + 'Create New Snapin' => 'Crear nuevo snapin', + 'List All Storage Groups' => 'Listar todos los grupos de almacenamiento', + 'Create New Storage Group' => 'Crear nuevo grupo de almacenamiento', + 'List All Storage Nodes' => 'Listar todos los nodos de almacenamiento', + 'Create New Storage Node' => 'Crear nuevo nodo de almacenamiento', + 'List All Users' => 'Listar todos los usuarios', + 'Create New User' => 'Crear nuevo usuario', +]; + + +$locales = glob($langDir . '/*.UTF-8/LC_MESSAGES/messages.po'); +sort($locales); +$grandAdded = 0; +$grandSkipped = 0; + +foreach ($locales as $po) { + preg_match('#/([^/]+)\.UTF-8/#', $po, $lm); + $locale = $lm[1] ?? $po; + // The msgid IS the English, so seeding en_US would add nothing and would + // make every entry look translated. It still gets the stray-%s sweep + // below: an en_US msgstr reading "Create New %s" under msgid "Create New + // Group" shows that %s to an English user, and blanking it falls back to + // the msgid, which is already the right words. + $seed = ('en_US' !== $locale); + $tr = poRead($po); + $fz = poFuzzy($po); + $listFmt = $tr['List All %s'] ?? ''; + $addFmt = $tr['Create New %s'] ?? ''; + + $added = []; + $skipped = []; + foreach ($seed ? $nodes : [] as $node => $spec) { + list($noun, $listId, $addId) = $spec; + // Where the catalog never translated the noun, compose with the + // English one rather than skipping. Skipping drops the whole label to + // English; composing keeps the locale's own verb, which is what these + // catalogs actually rendered before. It is also strictly better than + // before: the old code passed _(ucfirst($node)), so an untranslated + // German storage group read "Alle Storagegroups auflisten" -- a word + // that exists in no language. The properly spaced English noun is + // carried by the msgid itself, so it needs no second table. + $nounTr = $tr[$noun] ?? ''; + $listArg = '' !== $nounTr + ? $nounTr . 's' + : substr($listId, strlen('List All ')); + $addArg = '' !== $nounTr + ? $nounTr + : substr($addId, strlen('Create New ')); + foreach ([[$listFmt, $listId, $listArg], [$addFmt, $addId, $addArg]] as $job) { + list($fmt, $msgid, $arg) = $job; + // Never overwrite a real translation -- but a msgstr carrying a + // literal %s is not one. Those came from a fuzzy match against + // `Create New %s` that somebody accepted, so the catalog claims + // `Create New Host` is translated and renders "Neue %s erstellen" + // on screen. Composing over it is strictly better, and it is what + // keeps this migration lossless: blanking such an entry instead + // would drop that locale to English for a label it can express. + $cur = (string)($tr[$msgid] ?? ''); + if (isset($fz[$msgid]) || false !== strpos($cur, '%s')) { + $cur = ''; + } + if ('' !== $cur) { + continue; + } + if ('' === $fmt) { + $skipped[] = $msgid . ' (no format string in this catalog)'; + continue; + } + $val = safeFormat($fmt, $arg); + if (null === $val) { + $skipped[] = $msgid . ' (format has no usable %s)'; + continue; + } + $added[$msgid] = $val; + } + } + + printf( + "%-8s %2d seeded, %2d skipped\n", + $locale, + count($added), + count($skipped) + ); + foreach ($skipped as $s) { + printf(" skip %s\n", $s); + } + $grandAdded += count($added); + $grandSkipped += count($skipped); + + if (!$write) { + continue; + } + + // A msgstr that still carries a literal %s is SHOWING that %s to users: + // it came from a fuzzy match against `Create New %s` that somebody + // accepted, and these msgids already existed because the codebase writes + // _('Create New User') and friends as page titles. Blanking makes gettext + // fall back to the English msgid, which is the wrong language but a real + // phrase; writing German or Chinese here would be guessing. + $blank = []; + foreach ($nodes as $spec) { + foreach ([$spec[1], $spec[2]] as $msgid) { + if (isset($added[$msgid])) { + continue; + } + if (false !== strpos((string)($tr[$msgid] ?? ''), '%s')) { + $blank[$msgid] = ''; + } + } + } + if (count($blank)) { + printf("%-8s %2d msgstr(s) carrying a literal %%s blanked\n", $locale, count($blank)); + $added += $blank; + } + + if (isset($hand[$locale])) { + // Overwrites the seeds and the blanks above on purpose: seeding + // reproduces what the catalog rendered before, and for these two + // locales what it rendered before is precisely the bug. + $added = $hand[$locale] + $added; + printf( + "%-8s %2d hand-written entries applied\n", + $locale, + count($hand[$locale]) + ); + } + + // Last word: a CONFIRMED translation is never overwritten, by any pass. + // Confirmed means a live, non-empty, non-fuzzy msgstr that does not carry + // a stray %s -- a person wrote it and reviewed it. Everything above is + // reconstruction, and reconstruction does not get to overrule that, not + // even the hand-written tables: fr_FR already holds "Creer un nouveau + // Snapin" for `Create New Snapin`, and the only thing the French table + // would change about it is the capital S. + foreach (array_keys($added) as $msgid) { + $cur = (string)($tr[$msgid] ?? ''); + if ('' === $cur || isset($fz[$msgid]) || false !== strpos($cur, '%s')) { + continue; + } + unset($added[$msgid]); + } + + if (count($added)) { + poWrite($po, $added); + } +} + +printf("\n%d entries seeded, %d skipped\n", $grandAdded, $grandSkipped); diff --git a/bin/updatefog.sh b/bin/updatefog.sh new file mode 100644 index 0000000000..8936f4ea71 --- /dev/null +++ b/bin/updatefog.sh @@ -0,0 +1,370 @@ +#!/bin/bash +# +# FOG is a computer imaging solution. +# Copyright (C) 2007 Chuck Syperski & Jian Zhang +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Moves this 1.5 server's checkout onto another FOG line and runs the installer +# it finds there. Its reason to exist is the 1.5 -> 1.6 crossing, and the +# channel most people will pass is rc. +# +# WHY THIS IS STANDALONE, AND WILL STAY THAT WAY +# +# The 1.6 line has a bin/updatefog.sh that shares a channel map, a set of +# managed .fogsettings keys and a pile of helper functions with the rest of the +# installer. None of that exists on 1.5, and back-porting it would mean surgery +# on lib/common/functions.sh on a line that is heading for end of life -- every +# edit of which risks the 1.5 installs it is supposed to help. +# +# So this file sources nothing, and duplicates the small amount it needs. It +# WILL diverge from the 1.6 implementation, and that is fine: this branch is +# terminal and does not track 1.6's future. What matters is that it can carry +# a server across, once. +# +# THE HAZARD THIS SCRIPT IS BUILT AROUND +# +# It replaces itself. Checking out 1.6 rewrites bin/updatefog.sh underneath a +# bash process that is still reading it -- bash reads a script incrementally +# and seeks by byte offset, so a file that changes length mid-run makes it +# resume in the middle of a different line. The failure is silent, arbitrary, +# and happens after the checkout has already succeeded. +# +# So the first thing this does is copy itself somewhere git will not touch and +# re-exec from there. Everything after the re-exec is running from a file +# nothing is going to rewrite. See relaunchFromCopy(). +# +# Exit codes: +# 1 not root, or no install found 3 bad argument +# 6 git failed 7 no terminal for an interactive install + +# Resolved BEFORE the cd, and kept: relaunchFromCopy() reads this path, and $0 +# is whatever the caller typed. Invoked as `bash bin/updatefog.sh` from the +# checkout root -- or from a cron entry with a relative path -- $0 is +# `bin/updatefog.sh`, which stops resolving the moment this cd happens, and the +# copy this script cannot run without would fail. +selfpath=$(readlink -f "$BASH_SOURCE") +bindir=$(dirname "$selfpath") +cd "$bindir" +workingdir=$(pwd) + +usage() { + echo -e "Usage: $0 [-h?y] [--channel rc|beta|stable|patches] [--branch ]" + echo -e "\t-h -? --help\t\tDisplay this info" + echo -e "\t --channel\tWhich line to move this server to:" + echo -e "\t \t\t rc the current 1.6 release candidate" + echo -e "\t \t\t beta the 1.6 development line" + echo -e "\t \t\t stable the 1.5 stable line (where you are)" + echo -e "\t \t\t patches the 1.5 patches line" + echo -e "\t \t\tDefaults to rc" + echo -e "\t --branch\tCheck out a literal branch instead of a channel" + echo -e "\t-y --yes\t\tSkip the confirmation AND run the installer" + echo -e "\t \t\tunattended. NOT recommended for the crossing to" + echo -e "\t \t\t1.6 -- that upgrade asks questions 1.5 never did" + echo -e "\n\tGoing from 1.5 to 1.6 is a MAJOR upgrade. Take a backup first." + echo -e "\tThe 1.6 tree carries bin/revertfog.sh, which puts a server back on" + echo -e "\tits pre-upgrade 1.5 database and web tree from the dump the" + echo -e "\tinstaller takes. That dump is the only supported way back." + exit 0 +} + +fail() { + echo + echo " * $1" + shift + while [[ $# -gt 1 ]]; do + echo " | $1" + shift + done + echo + exit "$1" +} + +# --------------------------------------------------------------------------- +# Re-exec from a copy, before anything can rewrite this file. See the header. +# +# FOG_UPDATE_RELAUNCHED marks the copy so it does not do this again. The copy +# is removed on exit by the trap below, not here -- it is the file currently +# being read. +# --------------------------------------------------------------------------- +relaunchFromCopy() { + local copy + copy=$(mktemp -t fog-updatefog.XXXXXX) || return 1 + cat "$selfpath" > "$copy" || { rm -f "$copy"; return 1; } + chmod +x "$copy" + FOG_UPDATE_RELAUNCHED=1 FOG_UPDATE_ORIGIN="$workingdir" \ + FOG_UPDATE_COPY="$copy" exec bash "$copy" "$@" +} + + +# Kept before the option loop consumes them: the relaunch below re-execs with +# the ORIGINAL arguments, and by then "$@" has been shifted empty. +origArgs=("$@") + +repo="https://github.com/FOGProject/fogproject.git" +channel="rc" +branch="" +autoYes="" +sgitpath="" + +while [[ $# -gt 0 ]]; do + case $1 in + -h | -\? | --help) usage ;; + --channel) + [[ -n $2 ]] || fail "--channel requires a value" 3 + channel="$2"; shift 2 ;; + --branch) + [[ -n $2 ]] || fail "--branch requires a value" 3 + branch="$2"; shift 2 ;; + --git-path) + [[ -n $2 && $2 == /* ]] || fail "--git-path requires an absolute path" 3 + sgitpath="${2%/}"; shift 2 ;; + -y | --yes) autoYes=1; shift ;; + *) fail "Unknown option: $1" "Run with --help for the list." 3 ;; + esac +done + +# Parsing comes first so --help works for anyone, and a typo is answered +# before the user is told to go and find root. +if [[ ! $EUID -eq 0 ]]; then + echo "FOG updates must be run as root user" + exit 1 +fi + +if [[ -z $FOG_UPDATE_RELAUNCHED ]]; then + relaunchFromCopy "${origArgs[@]}" || fail "Could not copy this script to a temporary location." \ + "The update has to run from a copy, because checking out another branch" \ + "rewrites this file while bash is still reading it." 1 +fi +[[ -n $FOG_UPDATE_COPY ]] && trap 'rm -f "$FOG_UPDATE_COPY"' EXIT +# After the re-exec, $bindir is the temp directory. The checkout is where the +# ORIGINAL copy lived. +workingdir="${FOG_UPDATE_ORIGIN:-$workingdir}" +gitpath=$(cd "$workingdir/.." && pwd) +[[ -n $sgitpath ]] && gitpath="$sgitpath" + +# --------------------------------------------------------------------------- +# The 1.6 channel vocabulary, copied. Retired spellings honoured exactly as +# lib/common/functions.sh on working-1.6 honours them -- an admin pasting a +# command from an older forum post should get a working upgrade. +# +# Note `dev` means BETA, not dev-branch, despite a branch by that name. That +# is the 1.6 vocabulary and this must not invent a different one. +# --------------------------------------------------------------------------- +normalizeChannel() { + case "$1" in + stable) echo "stable" ;; + patches|staging) echo "patches" ;; + beta|dev) echo "beta" ;; + rc) echo "rc" ;; + *) return 1 ;; + esac +} + +# Asked of the remote, by version order. -v:refname so rc-1.6.10 beats +# rc-1.6.2; the remote advertises no commit dates, so a "newest" answer could +# not be date-based even if that were preferable. +# +# refs/heads/rc-*, NOT a bare rc-*. ls-remote matches a pattern against the +# TAIL of each ref at slash boundaries, so `rc-*` also matches +# refs/heads/feat/rc-anything -- and against origin it does. That would put a +# 1.5 server onto a feature branch while telling its admin it was the current +# release candidate, on the one channel this whole script recommends. The sed +# re-checks the extracted name for a further slash: two layers, because the +# answer decides what a major upgrade checks out. +rcBranch() { + local ref + ref=$(git ls-remote --heads --sort=-v:refname "$repo" 'refs/heads/rc-*' 2>/dev/null \ + | sed -n 's#^[0-9a-f]\{7,\}[[:space:]]\{1,\}refs/heads/\(rc-[^/]\{1,\}\)$#\1#p' \ + | head -n1) || return 1 + [[ -n $ref ]] || return 1 + echo "$ref" +} + +channelToBranch() { + case "$(normalizeChannel "$1")" in + stable) echo "stable" ;; + patches) echo "dev-branch" ;; + beta) echo "working-1.6" ;; + rc) rcBranch || return 2 ;; + *) return 1 ;; + esac +} + +# --------------------------------------------------------------------------- +# Find the install. 1.5 has no fog_git_path, so the checkout is simply where +# this script lives -- which is what the pre-relaunch $workingdir recorded. +# --------------------------------------------------------------------------- +[[ -z $fogprogramdir && -r /etc/fog/fog.conf ]] && . /etc/fog/fog.conf +[[ -z $fogprogramdir ]] && fogprogramdir="/opt/fog" +fogprogramdir="${fogprogramdir%/}" + +if [[ ! -r "$fogprogramdir/.fogsettings" ]]; then + fail "No existing FOG install found at ${fogprogramdir} (.fogsettings missing)." \ + "This updates an EXISTING install. For a new one, run installfog.sh." 1 +fi + +if [[ ! -d "${gitpath}/.git" ]]; then + fail "${gitpath} is not a git checkout." \ + "This server was installed from a tarball or a copied directory, so" \ + "there is no branch to move. Use the bootstrap installer instead --" \ + "it clones a checkout and runs the installer over this install, which" \ + "finds the existing server through /etc/fog/fog.conf:" \ + "" \ + " curl -fsSL https://raw.githubusercontent.com/FOGProject/fogproject/working-1.6/bin/bootstrap.sh | bash -s -- --channel ${channel}" 1 +fi + +if [[ -z $branch ]]; then + branch=$(channelToBranch "$channel") + case $? in + 0) ;; + 2) fail "No release candidate is currently published, so the rc channel" \ + "has nothing to check out. This is normal between releases." \ + "" \ + "Use --channel beta for the 1.6 development line, or wait." 3 ;; + *) fail "Unknown channel: ${channel}" \ + "Expected rc, beta, stable or patches. The retired names staging" \ + "and dev are also accepted, and mean patches and beta." 3 ;; + esac +fi + +# --------------------------------------------------------------------------- +# Say what this is before doing it. A 1.5 -> 1.6 move is not an update. +# --------------------------------------------------------------------------- +currentBranch=$(git -C "$gitpath" rev-parse --abbrev-ref HEAD 2>/dev/null) +currentCommit=$(git -C "$gitpath" rev-parse HEAD 2>/dev/null) + +echo " * FOG Update" +echo " Checkout: ${gitpath}" +echo " Now on: ${currentBranch:-unknown}" +echo " Moving to: ${branch}" +echo + +crossing=0 +case $branch in + working-1.6|rc-*) [[ $currentBranch != working-1.6 && $currentBranch != rc-* ]] && crossing=1 ;; +esac + +if [[ $crossing -eq 1 ]]; then + echo " * This is a MAJOR upgrade, 1.5 to 1.6, not a patch." + echo " |" + echo " | The database schema is migrated forward and the web tree is" + echo " | replaced. The 1.6 installer takes a database dump before it starts," + echo " | and the 1.6 tree carries bin/revertfog.sh, which uses that dump to" + echo " | put the server back. That dump is the ONLY supported way back --" + echo " | there is no down-migration and there will not be one." + echo " |" + echo " | Take your own backup as well, and read" + echo " | docs/SUPPORTED_CUSTOMIZATIONS.md in the 1.6 tree afterwards." + echo +fi + +# Checked BEFORE the checkout, not after. +# +# It used to sit beside the installer invocation, which meant a headless run +# -- cron, CI, a container with no tty -- moved the working copy to 1.6 and +# only then discovered it could not run the installer, leaving a 1.5 server +# with a 1.6 source tree. bin/bootstrap.sh gets this ordering right and so +# should this: the test costs nothing here and refuses before anything moves. +if [[ -z $autoYes ]] && ! (exec < /dev/tty) 2>/dev/null; then + fail "No terminal is available, so the installer cannot run interactively." \ + "Nothing has been changed." \ + "" \ + "Re-run from a terminal, or with --yes for an unattended install." \ + "For a 1.5 -> 1.6 crossing --yes is a poor idea: the 1.6 installer" \ + "asks about settings your .fogsettings has never held." 7 +fi + +if [[ -z $autoYes ]]; then + echo -n " * Continue? (Y/N) " + read confirmGo + case $confirmGo in + [Yy] | [Yy][Ee][Ss]) ;; + # A deliberate no is not a failure. EOF and a typo are not a no: + # with nobody at the keyboard `read` returns empty, and exiting 0 + # there told the caller an update had succeeded when none had been + # attempted. The terminal check above catches the usual headless + # case; this covers a terminal that exists but answers nothing. + [Nn] | [Nn][Oo]) echo " * Canceled."; exit 0 ;; + "") echo " * No answer given -- canceled. Pass --yes to run unattended."; exit 7 ;; + *) echo " * Answer not recognized -- canceled."; exit 3 ;; + esac +fi + +# --------------------------------------------------------------------------- +# Move the checkout. Safe to do now: this process is running from the copy. +# --------------------------------------------------------------------------- +echo " * Fetching" +git -C "$gitpath" fetch --all || fail "git fetch failed." 6 +echo " * Checking out ${branch}" +git -C "$gitpath" checkout "$branch" || fail "Could not check out ${branch}." 6 +git -C "$gitpath" reset --hard "origin/${branch}" || fail "Could not reset to origin/${branch}." 6 + +if [[ ! -x "${gitpath}/bin/installfog.sh" && ! -f "${gitpath}/bin/installfog.sh" ]]; then + fail "${branch} has no bin/installfog.sh." \ + "Nothing has been installed. To go back:" \ + " git -C ${gitpath} checkout --detach ${currentCommit}" 6 +fi + +# --------------------------------------------------------------------------- +# Hand over to the installer that is now on disk -- which for a crossing is +# 1.6's, not the one this script shipped beside. +# +# INTERACTIVE unless --yes. This is the case the whole file exists for, and it +# is exactly the case where -Y is wrong: the 1.6 installer asks about settings +# 1.5's .fogsettings has never held, and unattended it takes a default for each +# of them without saying so. +# --------------------------------------------------------------------------- +if [[ -n $autoYes ]]; then + echo " * Starting the installer unattended" + (cd "${gitpath}/bin" && bash installfog.sh -Y) + installStatus=$? +else + # No tty test here any more -- it happens before the checkout now, so by + # this point a terminal is known to exist. + echo " * Starting the installer" + echo + (cd "${gitpath}/bin" && bash installfog.sh < /dev/tty) + installStatus=$? +fi + +if [[ $installStatus -eq 0 ]]; then + echo + echo " * Update completed successfully." + [[ $crossing -eq 1 ]] && echo " * This server is now on FOG 1.6. From here, use its own bin/updatefog.sh." + exit 0 +fi + +echo +echo " * installfog.sh failed (exit ${installStatus})." +echo " | Nothing has been reverted. To put the checkout back where it was:" +echo " |" +# --detach, not reset --hard. This is the crossing case by definition: the +# checkout is on working-1.6 or an rc-* branch while $currentCommit belongs +# to stable, so resetting would point the NEW branch ref at an old commit and +# leave it diverged. Detaching moves only HEAD, which is all that is wanted. +# +# The opposite of what a reset --hard onto origin/ is for, which is +# discarding local mess so an update can proceed. Going back through history +# is a different job. +echo " | git -C ${gitpath} checkout --detach ${currentCommit}" +echo " | cd ${gitpath}/bin && ./installfog.sh" +echo " |" +if [[ $crossing -eq 1 ]]; then + echo " | If the database was already migrated, the checkout alone is not" + echo " | enough -- use bin/revertfog.sh from the 1.6 tree, which restores the" + echo " | pre-upgrade dump as well." + echo " |" +fi +exit "$installStatus" diff --git a/docs/MULTI_SERVER_CA.md b/docs/MULTI_SERVER_CA.md new file mode 100644 index 0000000000..1787f63238 --- /dev/null +++ b/docs/MULTI_SERVER_CA.md @@ -0,0 +1,299 @@ +# One trust anchor across several FOG servers + +You have more than one FOG server — separate installs, each with its own +database, not storage nodes of one another. Each generated its own +`FOG Server CA` at install time, so every browser, every `curl`, and every +system trust store needs one certificate **per server**. Five servers, five +imports, five things to redo when one is rebuilt. + +This document covers the ways to collapse that to one, what each costs, and how +to tell which one you are actually in. + +> **Storage nodes on this line still generate their own self-signed CA.** +> Automatic issuance from the master is a 1.6 feature and is not present here, +> so a fleet of five nodes has six unrelated CAs. Option B below applies to +> nodes just as it does to separate servers — treat each node as a server for +> the purposes of this document. + +--- + +## Table of contents + +- [First: which problem do you have?](#first-which-problem-do-you-have) +- [The options](#the-options) +- [Option A: import each server's root](#option-a-import-each-servers-root) +- [Option B: a hub FOG server issues to the others](#option-b-a-hub-fog-server-issues-to-the-others) +- [Option C: your own enterprise PKI or internal ACME CA](#option-c-your-own-enterprise-pki-or-internal-acme-ca) +- [What is and is not unified](#what-is-and-is-not-unified) +- [Verifying](#verifying) +- [Troubleshooting](#troubleshooting) + +--- + +## First: which problem do you have? + +Two different things both show up as "the certificate is invalid", and they have +different fixes. Sort this out before changing anything. + +**1. The client does not trust the CA.** The certificate is fine; nothing told +this machine to trust the issuer. Browsers are the usual case, and note that +importing into the *system* store does not fix them — Firefox carries its own +NSS store, Chrome reads a per-user one. + +**2. The certificate does not chain to the CA you trusted.** You trusted server +A's root and are browsing to server B, which has its own unrelated root. Adding +more trust does not fix this; the servers have to be re-issued. + +Tell them apart in one command, run against each server: + +```bash +echo | openssl s_client -connect :443 2>/dev/null | openssl x509 -out /tmp/l.pem +openssl verify -CAfile /path/to/the/root/you/trusted.pem /tmp/l.pem +``` + +`OK` means you are in case 1 — a trust distribution problem. A verify error +means case 2, and the rest of this document applies. + +> **Do not compare issuer names.** Every FOG install names its root +> `CN=FOG Server CA`, so two unrelated servers look identical by name. Compare +> the root's `subjectKeyIdentifier` to the leaf's `authorityKeyIdentifier`, or +> just use `openssl verify` as above. + +## The options + +| | Effort | Servers must trust each other | Best when | +|---|---|---|---| +| **A. Import each root** | One import per server, forever | no | 2–3 stable servers | +| **B. Hub FOG server issues** | One-time per server | yes — each satellite holds a CA the hub issued it (never the hub's own key) | several FOG servers, no existing PKI | +| **C. Your own PKI / ACME** | Depends on your PKI | no | you already run a CA | + +There is no option where the servers keep their independence *and* share an +anchor. Something has to sign for everyone. + +## Option A: import each server's root + +The baseline, and genuinely fine for a small number of servers. Nothing changes +on the FOG side; you distribute N certificates instead of one. + +Each server publishes its own anchor at: + +``` +https:///fog/management/other/ca.cert.der +``` + +Since the trust-store change, each server also anchors *itself* in its own +system store at install time, so `curl` and `wget` **on** a FOG server work +against that server without `-k`. That is per-server and does not federate; +`--no-ca-trust` opts out. + +Browsers still need a manual import — see [What is and is not +unified](#what-is-and-is-not-unified). + +## Option B: a hub FOG server issues to the others + +Pick one server as the hub. Its root stays where it is. Every other server gets +its **own** Web CA, signed by the hub's root and constrained to that server's +names, and uses it to sign its own web certificate. All leaves then chain to one +root, so one anchor covers the fleet. + +``` + hub: FOG Server CA (root, key never leaves the hub) + ├── FOG Web CA - fog1.lan → fog1's web certificate + ├── FOG Web CA - fog2.lan → fog2's web certificate + └── FOG Web CA - fog3.lan → fog3's web certificate +``` + +### 0. Check that your version can do this + +Option B needs `--web-ca-cert/--web-ca-key/--web-ca-root` on every satellite's +installer, and `packages/pki/fog-mint-web-ca` on the hub. `dev-branch` carries +both, and so does the 1.6 line. A **released 1.5.x `stable` carries neither** — +there is no `packages/pki` on it at all — so `git pull` on a `stable` checkout +will not produce these flags however many times you run it; you have to be on a +line that ships them. + +From each checkout: + +```bash +test -x packages/pki/fog-mint-web-ca && grep -q web-ca-cert bin/installfog.sh \ + && echo "Option B available" \ + || echo "not on this line — Option A or C only" +``` + +A server without them can be neither hub nor satellite. + +### 1. Issue a CA per server, on the hub + +```bash +sudo packages/pki/fog-mint-web-ca [extra-dns-name ...] +``` + +`` must be what the far server's own `hostname` reports, and any +`--extra-server-name`/`--internal-domain` values that server installs with must +be passed as extra arguments. Both go into the CA's name constraints, and a CA +constrained to the wrong names cannot sign the certificate it exists for. The +script signs a probe certificate carrying the names that server will actually +request and refuses to emit a CA that would reject it, so a mismatch fails here +rather than on the far server. + +Each run writes `/root/fog-web-cas/-webca.tar.gz` containing +`webca.pem`, `webca.key`, `fog-root.pem`. + +### 2. Copy the bundle to each server + +The bundle sits under `/root` because it carries a CA private key. Push it from +the hub rather than pulling it — `sshd` ships `PermitRootLogin +prohibit-password` on most distributions, and an unprivileged account cannot +write to `/root` on the far end either, so the obvious +`scp root@:/root/... /root/` fails at both ends with nothing more +informative than `Permission denied`: + +```bash +# on the hub +sudo cp /root/fog-web-cas/-webca.tar.gz ~/ +sudo chown $USER: ~/-webca.tar.gz +scp ~/-webca.tar.gz @:~/ +rm -f ~/-webca.tar.gz +``` + +### 3. Install on each server + +```bash +# on the far server +sudo mkdir -p /root/webca +sudo tar -xzf ~/-webca.tar.gz -C /root/webca + +cd ~/fogproject/bin +sudo ./installfog.sh --web-ca-cert /root/webca/webca.pem \ + --web-ca-key /root/webca/webca.key \ + --web-ca-root /root/webca/fog-root.pem +``` + +Unpacking as root keeps `webca.key` unreadable to other accounts; it stays on +this server permanently, so where it lands matters. Remove the tarball from your +home directory afterwards. + +Passing any one of the three is what marks the install as using an external CA; +there is no separate flag to set. You pass them **once** — the files are +imported into the web zone and later upgrades reuse the import without the +flags. + +### 4. Anchor the hub root wherever you need it + +One certificate now covers every server. On a Linux client: + +```bash +curl -k -o /tmp/fogca.der https:///fog/management/other/ca.cert.der +openssl x509 -inform DER -in /tmp/fogca.der -out /tmp/fogca.crt +sudo cp /tmp/fogca.crt /etc/pki/ca-trust/source/anchors/fog-server-ca.crt # RHEL family +sudo update-ca-trust extract +``` + +Debian/Ubuntu/Alpine use `/usr/local/share/ca-certificates` + +`update-ca-certificates`; Arch uses `/etc/ca-certificates/trust-source/anchors` ++ `trust extract-compat`. + +### The cost, stated plainly + +**Each satellite holds a CA private key.** That is the trade. The mitigation is +name constraints: `fog2`'s key can only mint certificates for `fog2`'s own +names, so a stolen key does not become a fleet-wide signing capability. This is +why each server gets its own CA rather than a copy of one shared key — never +distribute the same intermediate to several servers, and never distribute the +hub's root key at all. + +If that trade is unacceptable, use Option C, or accept Option A. + +## Option C: your own enterprise PKI or internal ACME CA + +If you already run a CA, issue each FOG server an intermediate from it and use +the same `--web-ca-*` flags. FOG does not care that the CA came from a hub FOG +server or from step-ca; it validates the same three things either way — the key +matches the certificate, the certificate is `CA:TRUE`, and it chains to the root +you supply. + +This is better than Option B when it is available: no FOG server holds signing +authority for another, and your existing rotation and revocation processes +apply. + +An internal ACME CA (step-ca and similar) is the best fit overall, because the +anchor is stable while the leaves rotate automatically. See the Let's Encrypt +and ACME section of [PKI_ZONES.md](PKI_ZONES.md#lets-encrypt-and-acme) for the +renewal model and the caveats that apply to public Let's Encrypt. + +## What is and is not unified + +Unifying the **Web** zone is what all of this does. The other zones are separate +questions. + +| Zone | Unified by this? | Notes | +|---|---|---| +| Web (HTTPS vhost) | **yes** | what `--web-ca-*` targets | +| Client communication | **no** | each server keeps its own root; fog-client pins per server | +| Secure Boot | **no** | its own zone. On this line you supply a key with `--secure-boot-key`/`--secure-boot-cert`; there is no `--secureboot-ca-cert` here, that is 1.6 only. See [PKI_ZONES.md](PKI_ZONES.md) | + +**fog-client is deliberately untouched.** It pins the root of the server it +registered against, and that root is not replaced by `--web-ca-*` — which is +precisely what makes this safe to do on a running fleet without re-registering a +single machine. Clients registered to fog2 keep trusting fog2. + +**Browsers are not covered by the system trust store.** Firefox uses its own +NSS store; Chrome reads a per-user one. Import the hub root by hand, once: + +- **Firefox** — Settings → Privacy & Security → Certificates → View + Certificates → Authorities → Import, tick *Trust this CA to identify + websites*. +- **Chrome/Chromium on Linux** — + `certutil -d sql:$HOME/.pki/nssdb -A -t "C,," -n "FOG Server CA" -i fogca.crt` + +## Verifying + +From any machine, per server: + +```bash +echo | openssl s_client -connect :443 2>/dev/null | openssl x509 -out /tmp/l.pem +openssl verify -CAfile /path/to/hub-root.pem /tmp/l.pem +``` + +You want `OK`, and the issuer should read `CN=FOG Web CA - `: + +```bash +echo | openssl s_client -connect :443 2>/dev/null | grep -E '^ [0-9] s:| *i:' +``` + +## Troubleshooting + +**The certificate did not change after installing with `--web-ca-*`.** +Fixed, but check your version. The installer used to decide whether to re-sign +the web leaf by hashing the SAN set alone, so switching CAs imported the new one +and then skipped reissue because the *names* had not changed — a clean install +that changed nothing. The signing CA is now part of that check. On a version +with the fix, the next run reissues once by itself. + +**The far server's web tier will not start, or its certificate does not +verify.** Its leaf carries a name the CA does not permit. Every FOG leaf +includes `fogserver` and `fog-server` regardless of hostname, plus the host's +long and short names and any `--extra-server-name`. Re-mint with the missing +names passed as extra arguments. `fog-mint-web-ca` probes for this before +emitting, so this mostly appears when a CA was built by hand. + +**The installer prompts for CA paths even though you passed the flags.** +Fixed. Passing `--web-ca-*` set `externalca=yes`, which triggered the +interactive prompt for the *flat* `extcacert`/`extcakey`/`extcaroot` paths — +and those were then ignored, because the command-line values take precedence. +Pressing Enter through it was harmless. On a version with the fix the run prints +the paths it is using instead. + +**`Refusing to continue: the root ... carries pathlen:0`.** That root cannot +anchor an intermediate, so nothing beneath it would verify. Use a root that can, +or Option A. + +**The root key is offline.** `fog-mint-web-ca` needs it to sign. Restore it, mint +every CA you need in one sitting, then take it away again — see +[Taking a key offline](PKI_ZONES.md#taking-a-key-offline). + +## See also + +- [PKI_ZONES.md](PKI_ZONES.md) — the three zones, layout, name constraints, key protection +- `packages/pki/fog-mint-web-ca` — issue a Web CA for another FOG server +- `packages/pki/fog-offline-ca-key` — move a CA private key off the server diff --git a/lib/common/config.sh b/lib/common/config.sh index eae1383823..cf37989f18 100755 --- a/lib/common/config.sh +++ b/lib/common/config.sh @@ -27,6 +27,16 @@ # rather than whatever happened to be tagged the day someone installed. [[ -z $ipxeVer ]] && ipxeVer="$(awk -F\' /"define\('FOG_IPXE_VERSION'[,](.*)"/'{print $4}' ../packages/web/lib/fog/system.class.php 2>/dev/null | tr -d '[[:space:]]')" [[ -z $ipxeVer ]] && ipxeVer="v2.0.0-fog.6" +# Bounds for every network fetch the installer makes, and the answer +# checkInternetConnection works out for the fetches that follow it. Overridable +# from .fogsettings for a link slow enough that fifteen seconds is genuinely too +# short, which is the only reason to raise them -- they exist so an unreachable +# host costs seconds instead of libcurl's 300 second default connect timeout. +# internet_ok starts optimistic so any path that reaches a download without +# having run the check behaves exactly as it did before. +[[ -z $inetConnectTimeout ]] && inetConnectTimeout=5 +[[ -z $inetMaxTime ]] && inetMaxTime=15 +[[ -z $internet_ok ]] && internet_ok=1 [[ -z $udpcastsrc ]] && udpcastsrc="../packages/udpcast-20250223.tar.gz" [[ -z $udpcastout ]] && udpcastout="udpcast-20250223" [[ -z $servicesrc ]] && servicesrc="../packages/service" diff --git a/lib/common/functions.sh b/lib/common/functions.sh index 2c8c9a3c14..366083eeb1 100755 --- a/lib/common/functions.sh +++ b/lib/common/functions.sh @@ -97,19 +97,109 @@ checkDatabaseConnection() { fi errorStat $connected } +# Reports one node<->master maintenance POST that did not land, and says how. +# +# GH-575: the two calls below post to this node's own web tier, and what +# actually reaches that web tier is not always what the installer aimed at. +# Three things intercept it, and none of them is a connection failure -- curl +# exits 0 every time: +# +# * an inline filtering proxy answering for the address (the reporter's was +# an iboss appliance returning ERR_CONNECT_FAIL as an HTML block page), +# * this node's own web tier bouncing every request to ?node=schema when it +# cannot read the master's database, +# * anything else in front of the server that answers 200 with markup. +# +# So both a status check and a body check are needed, and they catch different +# halves: a 3xx has no markup in it, and an interception answering 200 has no +# bad status. create_update_node.php outputs nothing at all on success -- it +# has no echo in it, and base.inc.php emits headers only -- so a '<' in the +# body is the response of something that is not it. +# +# Not fatal, in either caller. By this point the node's shares, services and +# FTP are configured, and both operations have a normal by-hand recovery in +# Storage Management: say plainly what failed, then carry on. +# +# $1 status, $2 response body, $3 what the caller was trying to do. +_reportNodePostFailure() { + local status="${1:-000}" body="$2" what="$3" + echo "Failed" + echo " * ${httpproto}://${ipaddress}${webroot}maintenance/create_update_node.php" + case $status in + 000) + # curl's own placeholder when no HTTP response arrived at all -- + # refused, timed out, TLS handshake failed. Not an interception. + echo " could not be reached, so ${what}." + ;; + *) + echo " answered HTTP ${status}, so ${what}." + ;; + esac + case $status in + 3*) + echo " * A redirect here usually means this node's own web tier cannot" + echo " reach the master's database and is bouncing every request to" + echo " the schema page -- check for SELinux denials with:" + echo " ausearch -m avc -ts recent" + ;; + esac + if [[ $body == *'<'* ]]; then + echo " * The reply was markup, not this server's answer, so something on" + echo " the network answered in its place. A filtering proxy in front of" + echo " ${ipaddress} is the usual cause; exempt this server from it." + fi + echo " * Fix the cause and re-run this installer, or set it by hand under" + echo " Storage Management in the web UI." +} registerStorageNode() { # GH-529: this defaulted to "/" while installfog.sh defaults to "/fog/", so # the two disagreed about where the app lives whenever webroot arrived # unset. Every fallback in this file now matches the installer's. [[ -z $webroot ]] && webroot="/fog/" dots "Checking if this node is registered" - storageNodeExists=$(wget --no-check-certificate -qO - ${httpproto}://${ipaddress}${webroot}/maintenance/check_node_exists.php --post-data="ip=${ipaddress}") + # --no-check-certificate stays here, and in the two calls below, ON PURPOSE. + # Every other unverified call in this installer has been removed; these + # three are the genuine chicken-and-egg. On a fresh storage node + # installfog.sh runs registerStorageNode -> updateStorageNodeCredentials + # -> _installCATrustAnchor in that order, so at this moment the node holds + # no anchor for anything and verification cannot succeed -- the thing that + # would make it possible is what registering is a precondition of. + # + # What that costs is bounded and worth stating: an attacker on the path + # between this node and its own web tier sees the node's storage + # credentials. It does NOT see the database password, which never travels + # this way. Closing it properly needs the master to hand a node its anchor + # out of band, which is a design change, not a flag change. + storageNodeExists=$(wget --no-check-certificate -qO - ${httpproto}://${ipaddress}${webroot}maintenance/check_node_exists.php --post-data="ip=${ipaddress}") echo "Done" if [[ $storageNodeExists != exists ]]; then [[ -z $maxClients ]] && maxClients=10 dots "Node being registered" - curl -s -k -X POST -d "newNode" -d "name=$(echo -n $ipaddress|base64)" -d "path=$(echo -n $storageLocation|base64)" -d "ftppath=$(echo -n $storageLocation|base64)" -d "snapinpath=$(echo -n $snapindir|base64)" -d "sslpath=$(echo -n $sslpath|base64)" -d "ip=$(echo -n $ipaddress|base64)" -d "maxClients=$(echo -n $maxClients|base64)" -d "user=$(echo -n $username|base64)" --data-urlencode "pass=$(echo -n $password|base64)" -d "interface=$(echo -n $interface|base64)" -d "bandwidth=1" -d "webroot=$(echo -n $webroot|base64)" -d "fogverified" ${httpproto}://${ipaddress}${webroot}/maintenance/create_update_node.php - echo "Done" + # A status check and a body check, neither of which this call had. Both + # matter and they catch different halves -- see _reportNodePostFailure. + # + # Deliberately NOT -L. curl reports %{http_code} for the LAST transfer + # it made, so following a 308 to the schema page would report that + # page's 200 and turn the failure back into a green "Done". There is no + # legitimate redirect to lose: the URL is built from ${httpproto} and + # ${webroot}, both of which this installer set itself. + regbody=$(curl -s --noproxy '*' -k -w '\n%{http_code}' -X POST -d "newNode" -d "name=$(echo -n $ipaddress|base64)" -d "path=$(echo -n $storageLocation|base64)" -d "ftppath=$(echo -n $storageLocation|base64)" -d "snapinpath=$(echo -n $snapindir|base64)" -d "sslpath=$(echo -n $sslpath|base64)" -d "ip=$(echo -n $ipaddress|base64)" -d "maxClients=$(echo -n $maxClients|base64)" -d "user=$(echo -n $username|base64)" --data-urlencode "pass=$(echo -n $password|base64)" -d "interface=$(echo -n $interface|base64)" -d "bandwidth=1" -d "webroot=$(echo -n $webroot|base64)" -d "fogverified" ${httpproto}://${ipaddress}${webroot}maintenance/create_update_node.php) + regstatus=${regbody##*$'\n'} + regbody=${regbody%$'\n'*} + case $regstatus in + 2*) + if [[ $regbody == *'<'* ]]; then + _reportNodePostFailure "$regstatus" "$regbody" \ + "this node did not register itself with the master and will not appear in Storage Management" + else + echo "Done" + fi + ;; + *) + _reportNodePostFailure "$regstatus" "$regbody" \ + "this node did not register itself with the master and will not appear in Storage Management" + ;; + esac else echo " * Node is registered" fi @@ -117,8 +207,34 @@ registerStorageNode() { updateStorageNodeCredentials() { [[ -z $webroot ]] && webroot="/fog/" # see registerStorageNode, GH-529 dots "Ensuring node username and passwords match" - curl -s -k -X POST -d "nodePass" -d "ip=$(echo -n $ipaddress|base64)" -d "user=$(echo -n $username|base64)" --data-urlencode "pass=$(echo -n $password|base64)" -d "fogverified" $httpproto://$ipaddress${webroot}maintenance/create_update_node.php - echo "Done" + # -k on purpose -- see registerStorageNode. This is called from the node + # path before any anchor exists, and from the master path after one does; + # the shared function has to work in the earlier of the two. + # GH-575: this call had no -o, so whatever answered was written STRAIGHT to + # the installer's stdout, in the middle of the dotted line -- which is why + # the reporter's console read + # + # Node being registered..................... + # + # followed by a proxy's block page. Then it echoed "Done" regardless, + # because nothing looked at the status or at what came back. + credbody=$(curl -s --noproxy '*' -k -w '\n%{http_code}' -X POST -d "nodePass" -d "ip=$(echo -n $ipaddress|base64)" -d "user=$(echo -n $username|base64)" --data-urlencode "pass=$(echo -n $password|base64)" -d "fogverified" ${httpproto}://${ipaddress}${webroot}maintenance/create_update_node.php) + credstatus=${credbody##*$'\n'} + credbody=${credbody%$'\n'*} + case $credstatus in + 2*) + if [[ $credbody == *'<'* ]]; then + _reportNodePostFailure "$credstatus" "$credbody" \ + "this node's storage credentials were not written to the master" + else + echo "Done" + fi + ;; + *) + _reportNodePostFailure "$credstatus" "$credbody" \ + "this node's storage credentials were not written to the master" + ;; + esac } backupDB() { # --------------------------------------------------------- @@ -143,10 +259,32 @@ backupDB() { # backup is the worst outcome available here. local dbbackupstat=0 local dbbackupfile="" - if [[ -d $backupPath/fog_web_${version}.BACKUP ]]; then + # Ask the database whether there is anything to dump, rather than asking + # the filesystem whether configureHttpd happened to leave a + # fog_web_.BACKUP behind. That directory was only ever a proxy for + # "this is an upgrade", and it is a broken one: configureHttpd removes + # ${docroot}fog when it is a SYMLINK and then tests `-d $webdirdest` -- + # the same path -- to decide whether to make the backup, so on any + # install whose web root is a symlink the directory never appears and the + # pre-upgrade dump was silently skipped on every run. + # + # SHOW TABLES is also the honest question. The dump has nothing to do with + # the web tree, and a leftover .fogsettings pointing at a database that + # does not exist yet would make an $doupdate-based gate report a failure + # it did not have. configureMySql has run by here, so $sqloptionsuser and + # $snmysqlpass are settled; a fresh install has no tables and still skips. + local dbhastables="" + dbhastables=$(mysql $sqloptionsuser --password="${snmysqlpass}" --skip-column-names --execute="SHOW TABLES" $mysqldbname 2>>$error_log | head -n 1) + if [[ -n $dbhastables ]]; then [[ ! -d $backupPath/fogDBbackups ]] && mkdir -p $backupPath/fogDBbackups >>$error_log 2>&1 - dbbackupfile="$backupPath/fogDBbackups/fog_sql_${version}_$(date +"%Y%m%d_%I%M%S").sql" - wget --no-check-certificate -O "$dbbackupfile" "${httpproto}://${ipaddress}${webroot}/maintenance/backup_db.php" --post-data="type=sql&fogajaxonly=1" >>$error_log 2>&1 || dbbackupstat=1 + # %H, not %I: %I is the 12-hour clock with no AM/PM marker, so an + # update run at 05:57 and one at 17:57 on the same day produced the + # same filename and the second silently overwrote the first. + dbbackupfile="$backupPath/fogDBbackups/fog_sql_${version}_$(date +"%Y%m%d_%H%M%S").sql" + # Verified, not --no-check-certificate: this is an HTTPS call to this + # server, and _resolveSelfCacert names the CA it is serving under. + _resolveSelfCacert + wget "${selfCacertOpts[@]}" -O "$dbbackupfile" "${httpproto}://${ipaddress}${webroot}/maintenance/backup_db.php" --post-data="type=sql&fogajaxonly=1" >>$error_log 2>&1 || dbbackupstat=1 [[ ! -s $dbbackupfile ]] && dbbackupstat=1 fi if [[ -z $dbbackupfile ]]; then @@ -181,7 +319,8 @@ checkWebTier() { local probeBody=$(mktemp) # No -q on the body: we care whether bytes came back at all, not just about # the status code, because that is exactly what a pre-output fatal loses. - wget --no-check-certificate -q -O "$probeBody" --no-proxy "$probeUrl" >>$error_log 2>&1 + _resolveSelfCacert + wget "${selfCacertOpts[@]}" -q -O "$probeBody" --no-proxy "$probeUrl" >>$error_log 2>&1 local probeStat=$? local probeSize=$(stat -c %s "$probeBody" 2>/dev/null) [[ -z $probeSize ]] && probeSize=0 @@ -279,8 +418,40 @@ updateDB() { case $dbupdate in [Yy]|[Yy][Ee][Ss]) dots "Updating Database" - wget --no-check-certificate -qO - --header="X-Fog-Install-Token: ${installToken}" --post-data="schemaupdate=1" --no-proxy ${httpproto}://${ipaddress}${webroot}management/index.php?node=schema >>$error_log 2>&1 - errorStat $? + # Verified. This request carries X-Fog-Install-Token, which grants + # a schema deploy on a server that has no users yet; + # --no-check-certificate handed that to whoever answered on + # $ipaddress. + _resolveSelfCacert + wget "${selfCacertOpts[@]}" -qO - --header="X-Fog-Install-Token: ${installToken}" --post-data="schemaupdate=1" --no-proxy ${httpproto}://${ipaddress}${webroot}management/index.php?node=schema >>$error_log 2>&1 + local schemarc=$? + # errorStat tails $error_log, so wget's own certificate error is + # already visible -- but it does not say what to do about it, and + # this is the one place where verifying instead of skipping can + # stop an upgrade that used to finish. wget reports every TLS + # failure as exit 5. + if [[ $schemarc -eq 5 ]]; then + echo "Failed!" + echo + echo " * TLS verification failed talking to this server's own web tier at" + echo " ${httpproto}://${ipaddress}${webroot} -- so the schema was NOT deployed." + echo " * This step used to skip verification, which handed the schema" + echo " install token to whatever answered on that address. It no longer" + echo " does, so a certificate this host cannot verify now stops here." + echo " * Two causes, both fixable:" + echo " - the web certificate was replaced by hand, so ${rootCAPem:-the FOG CA}" + echo " is no longer what signed it" + echo " - the certificate does not cover the address ${ipaddress}" + echo " * Full error in $error_log" + echo + tail -n 5 $error_log + # Exit rather than fall through to errorStat: the schema not + # deploying has always been fatal here, and errorStat would + # reprint a generic banner over a message that has already + # said more than it can. + exit $schemarc + fi + errorStat $schemarc ;; *) echo @@ -342,7 +513,23 @@ updateDB() { mysql $sqloptionsuser --password="${snmysqlpass}" --execute="INSERT INTO globalSettings (settingKey, settingDesc, settingValue, settingCategory) VALUES ('FOG_STORAGENODE_MYSQLPASS', 'This setting defines the password the storage nodes should use to connect to the fog server.', \"$snmysqlstoragepass\", 'FOG Storage Nodes') ON DUPLICATE KEY UPDATE settingValue=\"$snmysqlstoragepass\"" $mysqldbname >>$error_log 2>&1 errorStat $? dots "Granting access to fogstorage database user" - mysql ${host} -s --user=fogstorage --password="${snmysqlstoragepass}" --execute="INSERT INTO $mysqldbname.taskLog VALUES ( 0, '999test', 3, '127.0.0.1', NOW(), 'fog');" >/dev/null 2>&1 + # The probe writes a throwaway row to find out whether fogstorage still + # holds INSERT; a failure here is read as "the grants need redoing", which + # is what sends the installer off to ask for the database root password. + # + # NAME THE COLUMNS. This was a positional INSERT, and schema 280 adds + # logType and logText to taskLog -- six values into an eight column table + # is error 1136, "Column count doesn't match value count", and the symptom + # is an upgrade demanding a database root password on a server whose + # grants are perfectly correct. 1.6 hit exactly this twice (schema 336, + # then 338) before naming the columns; see fogproject#1209. A named list + # cannot break that way -- a column added later takes its default and this + # INSERT does not care. + # + # id is AUTO_INCREMENT so it is omitted. The '999test' marker stays in + # taskID, which on 1.5 is still mediumtext, and the DELETE still keys on + # it -- this change is about the column list, not the marker. + mysql ${host} -s --user=fogstorage --password="${snmysqlstoragepass}" --execute="INSERT INTO $mysqldbname.taskLog (taskID, taskStateID, ip, createTime, createdBy) VALUES ('999test', 3, '127.0.0.1', NOW(), 'fog');" >/dev/null 2>&1 connect_as_fogstorage=$? if [[ $connect_as_fogstorage -eq 0 ]]; then mysql $sqloptionsuser --password="${snmysqlpass}" --execute="DELETE FROM $mysqldbname.taskLog WHERE taskID='999test' AND ip='127.0.0.1';" >/dev/null 2>&1 @@ -411,9 +598,21 @@ validip() { echo $stat } getCidr() { - local cidr - cidr=$(ip -f inet -o addr | grep $1 | awk -F'[ /]+' '/global/ {print $5}' | head -n2 | tail -n1) - echo $cidr + # Prefix length of address $2 on interface $1. When $2 is not given, or is + # not on that interface, the prefix of the interface's first global address. + # + # GH-1747: this grepped the whole address table for the interface name and + # printed the SECOND global match (head -n2 | tail -n1). On an interface + # with more than one address that is another address's prefix: a stray + # 169.254.x.x/16 turned a /24 into 255.255.0.0. The unanchored grep also + # let eth1 read eth10. + [[ -n $1 ]] || return 0 + ip -4 -o addr show dev "$1" 2>/dev/null | awk -v want="$2" ' + $3 != "inet" { next } + { split($4, addr, "/") } + want != "" && addr[1] == want { print addr[2]; found = 1; exit } + first == "" && / scope global / { first = addr[2] } + END { if (!found && first != "") print first }' } mask2cidr() { local submask=$1 @@ -442,8 +641,7 @@ mask2cidr() { break ;; 224) - let - nbits+=3 + let nbits+=3 break ;; 192) @@ -457,7 +655,8 @@ mask2cidr() { 0) ;; *) - echo "Error: $dec is not recognized" + # stderr: every caller takes stdout as the prefix length. + echo "Error: $dec is not recognized" >&2 exit 1 ;; esac @@ -466,6 +665,10 @@ mask2cidr() { echo "$nbits" } cidr2mask() { + # No prefix means no mask. "$((/8))" put an arithmetic syntax error on the + # screen instead (GH-1747), and every caller already treats an empty mask + # as unknown. + [[ $1 =~ ^[0-9]+$ && $1 -le 32 ]] || return 1 local i="" local mask="" local full_octets=$(($1/8)) @@ -514,11 +717,19 @@ interface2broadcast() { echo "No interface passed" >&2 return 1 fi - # One address per line means one brd per line, so an interface carrying a - # second address returned two. Take the first, matching the $ipaddress / - # $ipaddresses contract from GH-954. Empty is a legitimate answer -- a /32 - # or a point-to-point link has no broadcast -- and the caller falls back. - ip -4 addr show $interface | grep -oP 'brd \K\S+' | head -1 + # The brd of address $2 on that interface. Without $2, or when $2 is not + # there, the first brd on the interface. Empty is a legitimate answer -- a + # /32 or a point-to-point link has no broadcast -- and the caller falls back. + # + # GH-1747: this always took the first brd, which belongs to whichever + # address is listed first. A link-local 169.254.x.x listed ahead of the + # real address ended the DHCP pool at 169.254.255.254. + ip -4 -o addr show dev "$interface" 2>/dev/null | awk -v want="$2" ' + $3 != "inet" { next } + { split($4, addr, "/"); brd = ""; for (i = 5; i < NF; i++) if ($i == "brd") brd = $(i + 1) } + want != "" && addr[1] == want { print brd; found = 1; exit } + first == "" && brd != "" { first = brd } + END { if (!found) print first }' } subtract1fromAddress() { local ip=$1 @@ -649,67 +860,136 @@ getAllNetworkInterfaces() { fi echo -n $interfaces } +# One bounded reachability probe against a single host. Returns curl's exit +# status so the caller can name the cause without running three separate tests +# to find it out. +# +# Both bounds matter. Without --connect-timeout, curl inherits libcurl's 300 +# second default, which is exactly what a firewall that DROPs rather than +# REJECTs outbound traffic costs -- per host, per address family. Without +# --max-time, a connection that opens and then stalls never returns at all. +# +# Deliberately no -k. A proxy presenting its own CA passes an unverified probe +# and then fails the git clone that follows, and predicting that clone is the +# entire point of the check. Equally deliberately no -f: a host that answers 404 +# at "/" is still a reachable host, and reachability is what is being measured. +inetProbe() { + local host="$1" + if command -v curl >/dev/null 2>&1; then + curl -sS --connect-timeout $inetConnectTimeout --max-time $inetMaxTime \ + -o /dev/null "https://${host}/" >>$error_log 2>&1 + return $? + fi + # curl is in every distro's package list, but installPackages has not run + # yet at this point, so a minimal image can legitimately reach here without + # it. bash's own /dev/tcp keeps the fallback dependency free. It sees only + # the TCP handshake -- not TLS, and not a proxy -- so it reports the generic + # connect failure (7) rather than claiming to know more than it does. + # + # Bounded by the connect timeout rather than the total: a handshake is all + # this does, so there is no transfer phase for $inetMaxTime to govern. + timeout $inetConnectTimeout bash -c "exec 3<>/dev/tcp/${host}/443" >>$error_log 2>&1 + [[ $? -eq 0 ]] && return 0 + return 7 +} +# Probe the hosts this install is actually going to pull from, and record the +# answer somewhere the code that downloads can read it. +# +# This used to test DNS, then plain HTTP, then HTTPS, against httpbin.org, +# neverssl.com, github.com and fogproject.org -- none of them with a timeout, +# and none of them a host FOG needs. Worse, it opened by running +# `$packageinstaller curl`, so a connectivity check's first act was a package +# transaction that needed the very connectivity it was about to test: metadata +# refresh against every configured mirror, unbounded on Debian/Ubuntu whenever +# unattended-upgrades holds the dpkg lock (apt has no lock timeout), and on Arch +# a full system upgrade, because $packageinstaller there is `pacman -Syu`. All +# of it redirected to the error log, so the screen showed "Testing internet +# connection" and nothing else for minutes at a time. +# +# Nothing read the result either. dns_ok/http_ok/https_ok were set and never +# looked at, both failure paths returned rather than exited, and the caller +# ignored the status -- so the install proceeded identically either way and the +# stall bought a message and nothing more. +# +# What the install genuinely needs from the internet is the distro's package +# repositories -- which installPackages reports on for itself -- and the host +# behind $ipxegit/$ipxeurl, for the iPXE sources and the iPXE and Secure Boot +# release assets. Those are what is probed, so pointing them at an internal +# mirror tests the mirror instead of github.com rather than as well as it. One +# HTTPS request per host settles DNS, TCP and TLS together, and curl's exit +# status says which of the three failed, so the old three-stage ladder is not +# needed to produce a specific message. +# +# Failure stays non-fatal, as before: offline installs are supported and +# documented (pre-placed iPXE sources, a pre-placed release tarball), so the +# output is advice plus $internet_ok, not an exit. $internet_ok is what +# fetchipxeasset, downloadfiles and prepareiPXEsource read to avoid +# re-attempting a fetch that has already been shown to be unreachable. checkInternetConnection() { dots "Testing internet connection" - DEBIAN_FRONTEND=noninteractive $packageinstaller curl >>$error_log 2>&1 - - http_sites=("httpbin.org" "neverssl.com") - https_sites=("github.com" "fogproject.org") - dns_ok=0 - http_ok=0 - https_ok=0 - - for dnsname in "${http_sites[@]}" "${https_sites[@]}"; do - echo -n "Testing DNS name resolution (${dnsname})... " >> $error_log - getent hosts ${dnsname} >/dev/null 2>&1 - if [[ $? -ne 0 ]]; then - echo "Failed" >> $error_log - continue - fi - dns_ok=1 - echo "OK" >> $error_log - break - done - if [[ $dns_ok -eq 0 ]]; then - echo "Failed" - echo - echo "There seems to be a DNS problem. Check the contents of /etc/resolv.conf" | tee -a $error_log - echo "If this is CentOS, RHEL, or Fedora or an other RH variant, also check" | tee -a $error_log - echo "the DNS entries in /etc/sysconfig/network-scripts/ifcfg-*" | tee -a $error_log - echo - return - fi - for url in "${http_sites[@]}"; do - echo -n "Testing HTTP connection (http://${url})... " >> $error_log - curl --silent http://${url} >/dev/null 2>>$error_log - if [[ $? -ne 0 ]]; then - echo "Failed" >> $error_log - continue - fi - http_ok=1 - echo "OK" >> $error_log - break - done - for url in "${https_sites[@]}"; do - echo -n "Testing HTTPS connection (https://${url})... " >> $error_log - curl --silent -k https://${url} >/dev/null 2>>$error_log - if [[ $? -ne 0 ]]; then - echo "Failed" >> $error_log + internet_ok=0 + local url host rc failhost="" failrc=0 + # Deduplicated because $ipxeurl is derived from $ipxegit, so the stock + # configuration is one host probed once rather than github.com probed twice. + local hosts=$( + for url in "$ipxegit" "$ipxeurl"; do + host="${url#*://}" + echo "${host%%/*}" + done | grep . | sort -u + ) + for host in $hosts; do + echo -n "Testing connection to ${host}... " >> $error_log + inetProbe "$host" + rc=$? + if [[ $rc -eq 0 ]]; then + echo "OK" >> $error_log continue fi - https_ok=1 - echo "OK" >> $error_log - break + echo "Failed (curl exit ${rc})" >> $error_log + failhost="$host" + failrc=$rc done - if [[ $http_ok -eq 0 && $https_ok -eq 0 ]]; then - echo "Failed" - echo - echo "There was no interface with an active internet connection found." | tee -a $error_log - echo "If you are using a proxy server, please export http_proxy and https_proxy or use .curlrc" | tee -a $error_log - echo - return + if [[ -z $failhost ]]; then + internet_ok=1 + echo "Done" + return 0 fi - echo "Done" + echo "Failed" + echo + case $failrc in + 6) + echo "Could not resolve ${failhost}. Check the contents of /etc/resolv.conf," | tee -a $error_log + echo "and on RHEL, CentOS, Fedora or another RH variant also the DNS settings" | tee -a $error_log + echo "on the connection itself (nmcli con show | grep ipv4.dns)." | tee -a $error_log + ;; + # 7 and 28 share a message on purpose. A firewall that DROPs outbound + # traffic -- the usual cause on an isolated or corporate network -- + # produces 28 (the connect timeout expiring), not 7; 7 is what an + # explicit REJECT or "network unreachable" gives. Naming only $inetMaxTime + # against a 28 would report the wrong bound, since it is almost always + # $inetConnectTimeout that fired. + 7|28) + echo "Could not reach ${failhost} on port 443 within ${inetConnectTimeout}s to connect" | tee -a $error_log + echo "or ${inetMaxTime}s in total. A firewall that drops outbound traffic rather than" | tee -a $error_log + echo "rejecting it looks exactly like this." | tee -a $error_log + ;; + 35|60|77) + echo "TLS to ${failhost} failed. If a proxy or filter is intercepting HTTPS," | tee -a $error_log + echo "its CA has to be trusted by this machine -- git and curl will both fail" | tee -a $error_log + echo "the same way until it is." | tee -a $error_log + ;; + *) + echo "Could not reach ${failhost} (curl exit ${failrc})." | tee -a $error_log + ;; + esac + echo + echo "The install will continue. FOG needs ${failhost} for the iPXE sources and" | tee -a $error_log + echo "the iPXE release binaries, so those steps are the ones expected to fail." | tee -a $error_log + echo "If you are using a proxy server, please export http_proxy and https_proxy or use .curlrc" | tee -a $error_log + echo "For a deliberate offline install, pre-place those sources -- each download" | tee -a $error_log + echo "step below prints the exact path it looks in." | tee -a $error_log + echo + return 0 } join() { local IFS="$1" @@ -732,6 +1012,67 @@ installFOGServices() { chmod +x -R $servicedst/ mkdir -p $servicelogs errorStat $? + # Where the web tier records what FOS told it (service/taskerror.php). + # Its own subdirectory rather than group-write on $servicelogs: that + # directory is root's and holds the daemons' logs, and rotation renames + # and unlinks, so shared write would let the web user delete them. + dots "Creating FOS report log directory" + mkdir -p $servicelogs/fos >>$error_log 2>&1 + chown ${apacheuser}:${apacheuser} $servicelogs/fos >>$error_log 2>&1 + errorStat $? + # Outside the dots/errorStat pair, like every other caller: + # setSELinuxContext prints its own line. The _rw_ type is not optional -- + # /opt/fog inherits usr_t and httpd_t may READ usr_t but not write it, so + # without this the directory exists, looks right, and every report is + # dropped with nothing but an AVC to say so. + setSELinuxContext "$servicelogs/fos" httpd_sys_rw_content_t + # Where FOGBase::logFault() records database operations that did not + # happen. Its own subdirectory for the same reason the one above has its. + # + # Unlike that one, BOTH tiers write here -- the web user, and root for the + # daemons -- so logFault() writes faults-web.log and faults-service.log + # rather than one shared file, whose owner would be whichever tier hit a + # failure first. The directory is the web user's; root writes into it + # regardless of mode. + dots "Creating FOG fault log directory" + mkdir -p $servicelogs/faults >>$error_log 2>&1 + chown ${apacheuser}:${apacheuser} $servicelogs/faults >>$error_log 2>&1 + # 0750, not the 0755 the other log directories carry. A fault line names + # the class, the table and the shape of the statement that failed, which + # is more than any local account needs; #1261 already cut the bound + # values out of it, and this stops the rest being world-readable. The web + # user owns the directory and root ignores the mode, so both writers are + # unaffected. + chmod 0750 $servicelogs/faults >>$error_log 2>&1 + errorStat $? + # Outside the dots/errorStat pair, like every other caller, and the _rw_ + # label is as load-bearing here as it is for fos above (GH-964). + setSELinuxContext "$servicelogs/faults" httpd_sys_rw_content_t + # FOG's own PHP session store (FOG_SESSION_DIR in commons/init.php, which + # points session.save_path here at runtime). FOG used to share the distro's + # session directory, where session.gc_maxlifetime is 1440 -- 24 minutes on + # every distro we support -- so PHP reaped the session file long before + # FOG_INACTIVITY_TIMEOUT said to, and the user was silently bounced to the + # login page. gc_maxlifetime applies to the whole save_path, so FOG cannot + # raise it without imposing its retention on every other PHP application on + # the box. Hence a private directory. + dots "Creating FOG session directory" + mkdir -p $fogprogramdir/sessions >>$error_log 2>&1 + # 0700 and owned by the pool user -- stricter than the 0750 above, because + # a session file IS an authentication token: anything that can read this + # directory can resume an admin session, and unlike the fault log there is + # no second writer to accommodate. Safe as a single-owner directory because + # the php-fpm pool is pinned to $apacheuser further down this same install + # (the `user = ${apacheuser}` rewrite in the pool file), which is the same + # variable used here. + chown ${apacheuser}:${apacheuser} $fogprogramdir/sessions >>$error_log 2>&1 + chmod 0700 $fogprogramdir/sessions >>$error_log 2>&1 + errorStat $? + # Same GH-964 reasoning as the fault log above: /opt/fog inherits usr_t and + # httpd_t may read but not write it. Unlabelled, PHP cannot write a session + # file on an enforcing host -- which does not degrade, it means nobody can + # log in at all, with only an AVC denial to say so. + setSELinuxContext "$fogprogramdir/sessions" httpd_sys_rw_content_t } configureUDPCast() { dots "Setting up UDPCast" @@ -743,8 +1084,15 @@ configureUDPCast() { cd $udpcastout grep -q 'BCM[0-9][0-9][0-9][0-9]' /proc/cpuinfo >>$error_log 2>&1 if [[ $? -eq 0 ]]; then - wget -qO config.guess "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess" >>$error_log 2>&1 - wget -qO config.sub "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub" >>$error_log 2>&1 + # Bounded, and the retry count cut right down. wget defaults to + # --tries=20 with no connect timeout at all, so on a Pi that cannot + # reach savannah this sat here for twenty full SYN retry cycles, twice, + # silently. Both files are a few KB, so a 30 second read timeout cannot + # cut a legitimate transfer short. + wget -qO config.guess --connect-timeout=$inetConnectTimeout --read-timeout=30 --tries=2 \ + "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess" >>$error_log 2>&1 + wget -qO config.sub --connect-timeout=$inetConnectTimeout --read-timeout=30 --tries=2 \ + "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub" >>$error_log 2>&1 chmod +x config.guess config.sub >>$error_log 2>&1 fi errorStat $? @@ -821,7 +1169,23 @@ configureFTP() { configureDefaultiPXEfile() { dots 'Configuring default iPXE file' [[ -z $webroot ]] && webroot='/fog/' # see registerStorageNode, GH-529 - echo -e "#!ipxe\nset arch \${buildarch}\niseq \${arch} i386 && cpuid --ext 29 && set arch x86_64 ||\nparams\nparam mac0 \${net0/mac}\nparam arch \${arch}\nparam platform \${platform}\nparam product \${product}\nparam manufacturer \${product}\nparam ipxever \${version}\nparam filename \${filename}\nparam sysuuid \${uuid}\nisset \${net1/mac} && param mac1 \${net1/mac} || goto bootme\nisset \${net2/mac} && param mac2 \${net2/mac} || goto bootme\n:bootme\nchain ${httpproto}://$ipaddress${webroot}service/ipxe/boot.php##params" > "$tftpdirdst/default.ipxe" + # param manufacturer took ${product} for years, so every ipxeTable row + # recorded the model twice and the vendor never once. iPXE exposes the two + # as separate SMBIOS settings. + # + # macboot is ${netX/mac}, iPXE's alias for the device it booted from. It is + # NOT a replacement for mac0: netX is a pointer at one of net0..netN, so + # swapping it in would drop net0 from the set on a machine that booted off + # net1. boot.php unions every mac* field and array_unique()s the result, so + # sending both costs nothing when they are the same NIC and guarantees the + # booting NIC is present however many NICs the box has. It sits above the + # net1..net7 chain because that chain short-circuits to :bootme on the first + # absent interface, which on a single-NIC machine is net1. + # + # The enumeration used to stop at net2. Anything past three NICs was + # invisible to the host lookup, so a machine registered under only its + # fourth NIC could not be found at all. + echo -e "#!ipxe\nset arch \${buildarch}\niseq \${arch} i386 && cpuid --ext 29 && set arch x86_64 ||\nparams\nparam mac0 \${net0/mac}\nparam arch \${arch}\nparam platform \${platform}\nparam product \${product}\nparam manufacturer \${manufacturer}\nparam ipxever \${version}\nparam filename \${filename}\nparam sysuuid \${uuid}\nisset \${netX/mac} && param macboot \${netX/mac} ||\nisset \${net1/mac} && param mac1 \${net1/mac} || goto bootme\nisset \${net2/mac} && param mac2 \${net2/mac} || goto bootme\nisset \${net3/mac} && param mac3 \${net3/mac} || goto bootme\nisset \${net4/mac} && param mac4 \${net4/mac} || goto bootme\nisset \${net5/mac} && param mac5 \${net5/mac} || goto bootme\nisset \${net6/mac} && param mac6 \${net6/mac} || goto bootme\nisset \${net7/mac} && param mac7 \${net7/mac} || goto bootme\n:bootme\nchain ${httpproto}://$ipaddress${webroot}service/ipxe/boot.php##params" > "$tftpdirdst/default.ipxe" errorStat $? } prepareiPXEsource() { @@ -837,6 +1201,15 @@ prepareiPXEsource() { # subdirectory of upstream clones) and nothing here needs the network. dots "Preparing iPXE build sources" if [[ -d $buildipxesrc/.git ]]; then + # git has no connect timeout of its own, so an unreachable host stalls + # here for as long as the kernel retries the SYN. The fetch is only ever + # an update to a checkout that already works, so when the host is known + # to be unreachable, skip straight to using what is on disk -- the same + # outcome the failed-checkout branch below produces, minus the wait. + if [[ $internet_ok -ne 1 ]]; then + echo "Skipped (using existing checkout)" + return 0 + fi git -C "$buildipxesrc" fetch --tags --force "$ipxegit" >>$error_log 2>&1 if ! git -C "$buildipxesrc" checkout -q "$ipxeVer" >>$error_log 2>&1; then # Offline, or the tag does not exist yet. A usable checkout is @@ -885,12 +1258,32 @@ fetchipxeasset() { cd ../tmp/ local checksum=1 local cnt=0 - while [[ $checksum -ne 0 && $cnt -lt 10 ]]; do + # Ten rounds of two timeout-less curls is the most expensive stall in the + # whole installer: on a network that drops outbound traffic each of those + # twenty connects sat at libcurl's 300 second default before returning, all + # of it silent under one "Downloading iPXE binaries" line. When + # checkInternetConnection has already established the host is unreachable + # there is nothing to retry FOR, so make the one attempt and report. + local tries=10 + [[ $internet_ok -ne 1 ]] && tries=1 + while [[ $checksum -ne 0 && $cnt -lt $tries ]]; do [[ -f ${tarball}.sha256 ]] && sha256sum -c ${tarball}.sha256 >>$error_log 2>&1 checksum=$? if [[ $checksum -ne 0 ]]; then - curl --silent -fkOL "$url" >>$error_log 2>&1 - curl --silent -fkOL "${url}.sha256" >>$error_log 2>&1 + # --connect-timeout bounds an unreachable host; --speed-time/-limit + # bounds a connection that opens and then stalls. --max-time is + # deliberately NOT used here -- these are multi-megabyte tarballs + # and a slow but working link must be allowed to finish. + # No -k. This is an ordinary internet download from a host with a + # perfectly good certificate, and the sha256 below does not save + # us -- it is fetched over the same unverified connection, so + # whoever could substitute the tarball could substitute the hash + # with it. checkInternetConnection() already explains a TLS + # failure here as an untrusted intercepting proxy. + curl --silent -fOL --connect-timeout $inetConnectTimeout \ + --speed-time 30 --speed-limit 1024 "$url" >>$error_log 2>&1 + curl --silent -fOL --connect-timeout $inetConnectTimeout \ + --speed-time 30 --speed-limit 1024 "${url}.sha256" >>$error_log 2>&1 fi let cnt+=1 done @@ -994,7 +1387,32 @@ configureTFTPandPXE() { # staging tree the copy loop below already reads, so a locally built # binary lands exactly where a downloaded one would. "${buildipxesrc}/buildipxe.sh" "${sslpath}CA/.fogCA.pem" "$(readlink -f $tftpdirsrc)" >>$workingdir/error_logs/fog_ipxe-build_${version}.log 2>&1 - errorStat $? + local buildstat=$? + local ipxebuildlog="$workingdir/error_logs/fog_ipxe-build_${version}.log" + # errorStat tails $error_log, and this build does not write there -- its + # output goes to the file above. Tailing the wrong log printed five lines + # of unrelated noise from earlier steps (a DB backup line, the HTML body + # of the schema POST) and threw away the exit status, which is the one + # value that identifies the failure: buildipxe.sh returns a distinct + # status per stage -- 39/41 upstream checkout and patching, 40/48 BIOS, + # 79/80/91/95 x86 EFI, 82/93/97 the arm64 cross-compile. Report both, so + # a failed build can be diagnosed from what the installer prints instead + # of from a file nobody is told to look at. + if [[ $buildstat -ne 0 ]]; then + echo "Failed! (buildipxe.sh exit $buildstat)" + if [[ -z $exitFail ]]; then + echo + echo " * The iPXE build writes its own log, separate from $error_log." + echo " * Full build output: $ipxebuildlog" + echo " * Please include that file, and the exit status above, when" + echo " reporting this." + echo + tail -n 20 "$ipxebuildlog" + exit $buildstat + fi + else + errorStat 0 + fi cd $workingdir fi cd $tftpdirsrc @@ -1177,14 +1595,18 @@ addOndrejRepo() { } resolveDHCPEngine() { # Decide between Kea and ISC-DHCP for the optional FOG-hosted DHCP service. - # Only relevant when FOG is actually building DHCP and the ISC package is + # Only relevant when FOG is actually building DHCP and a DHCP package is # still in the install set (the storage-node and bldhcp=0 paths strip it in # doOSSpecificIncludes before we ever get here). Must run after repo setup # so the Kea availability probe sees enabled repos (e.g. EPEL on RHEL). [[ -z $keaconfig ]] && keaconfig="/etc/kea/kea-dhcp4.conf" [[ $bldhcp -eq 1 ]] || return 0 local iscpkg="$dhcpname" - [[ -n $iscpkg && $packages == *"$iscpkg"* ]] || return 0 + # Accept the Kea package as well. The swap below is saved with the package + # list, but $dhcpname and $dhcpconfig are re-seeded from the distro config + # on every run. Matching only the ISC name returned here on every re-run of + # a Kea install, which left the ISC config path for the Kea JSON (GH-1747). + [[ -n $iscpkg && ( $packages == *"$iscpkg"* || ( -n $keapackage && $packages == *"$keapackage"* ) ) ]] || return 0 # Honor an explicit/persisted choice; an existing install is never switched. dhcpengine="${dhcpengine,,}" if [[ -z $dhcpengine ]]; then @@ -2067,15 +2489,29 @@ doOSSpecificIncludes() { ;; esac currentdir=$(pwd) - case $currentdir in - *$webdirdest*|*$tftpdirdst*) - echo "Please change installation directory." - echo "Running from here will fail." - echo "You are in $currentdir which is a folder that will" - echo "be moved during installation." - exit 1 - ;; - esac + # Both variables are tested for non-emptiness FIRST, and that is the whole + # point rather than defensive noise: in a glob, `*$webdirdest*` with an + # empty $webdirdest is `**`, which matches EVERY path. So whenever this + # function reaches here without having sourced a distro config -- the `*)` + # arm above blanks osid and RETURNS rather than exiting -- the old form + # refused to run from any directory at all, and said so in a message about + # the install layout that had nothing to do with the real problem. + # + # Ported from working-1.6, where a .fogsettings key rename made this easy + # to hit: an unset id took the `*)` arm and the admin got "Sorry, answer + # not recognized" followed by "Please change installation directory" about + # a path that was fine. That rename is not on this branch and is not being + # ported, but the amplifier here never depended on it -- ANY route to this + # guard without a sourced distro config produces the same false message, + # and on a 1.5 server a hand-edited or truncated .fogsettings is the way in. + if { [[ -n $webdirdest ]] && [[ $currentdir == *"$webdirdest"* ]]; } \ + || { [[ -n $tftpdirdst ]] && [[ $currentdir == *"$tftpdirdst"* ]]; }; then + echo "Please change installation directory." + echo "Running from here will fail." + echo "You are in $currentdir which is a folder that will" + echo "be moved during installation." + exit 1 + fi } errorStat() { local status=$1 @@ -3511,6 +3947,18 @@ EOF chmod 0644 "${outdir}/${certfile}" >>$error_log 2>&1 return $st } +# Did $rootCAPem actually issue $sslcapem? +# +# The one question that separates a FOG-generated Web CA from one imported with +# --web-ca-cert, which no comparison of PATHS can answer -- the import lands on +# the same canonical filenames the generator uses. See the call site in +# createWebIntermediateCA for what regenerating the chain from the wrong root +# costs. +_rootIssuedWebCA() { + [[ -n $rootCAPem && -s $rootCAPem ]] || return 1 + [[ -n $sslcapem && -s $sslcapem ]] || return 1 + openssl verify -trusted "$rootCAPem" "$sslcapem" >/dev/null 2>&1 +} # The Web zone: an intermediate whose leaf is what the vhost serves. Replacing # this zone has zero endpoint impact -- browsers just need the root trusted, # and fog-client already trusts it, because the root is what it pins. @@ -3552,8 +4000,29 @@ $(_nameConstraints)" "FOG Web UI" # sslprivkey/sslpubcert. if [[ -z $sslcachain || $sslcachain == "${cadir}/.fogWebCAchain.pem" || $sslcachain == "$rootCAPem" ]]; then sslcachain="${cadir}/.fogWebCAchain.pem" - cat "$sslcapem" "$rootCAPem" > "$sslcachain" 2>>$error_log - chmod 0644 "$sslcachain" >>$error_log 2>&1 + # The root appended has to be the one that actually ISSUED $sslcapem, + # and the path guard above cannot tell. Under + # --web-ca-cert/--web-ca-key/--web-ca-root the Web CA was issued by + # ANOTHER server's root, validateExternalCA imports to this exact + # canonical path, and it deliberately leaves $rootCAPem pointing at + # THIS server's own root (see the comment there -- fog-client pins + # $rootCAPem, so it must not move). Every path test therefore says + # "FOG-managed default, safe to regenerate", and the cat then replaces + # the imported root with one that does not sign the intermediate above + # it. + # + # Nothing complains at the time; the file is only read on later runs. + # Checked as a property, not a path, because the import and the + # generator write the same filename and no path test can separate + # them. + # + # The -s fallback keeps a fresh install working when there is no chain + # on disk yet: a chain built from the wrong root is still better than + # no chain at all, and that is the pre-existing behaviour. + if _rootIssuedWebCA || [[ ! -s $sslcachain ]]; then + cat "$sslcapem" "$rootCAPem" > "$sslcachain" 2>>$error_log + chmod 0644 "$sslcachain" >>$error_log 2>&1 + fi fi } # The client communication certificate: the public half of the keypair @@ -3711,7 +4180,16 @@ _createWebLeaf() { # The name set, hashed. ca.cnf is rewritten from $ipaddresses/$hostname/ # $extraServerNames on every run, so a changed hostname or a new # --extra-server-name changes this and nothing else has to notice. - want=$(openssl md5 < "$sslpath/ca.cnf" 2>/dev/null) + # The signing CA is part of the stamp, not just the name set. It used to be + # ca.cnf alone, which meant switching the Web CA -- --web-ca-cert/-key/-root + # pointing this server at a CA another FOG server issued -- imported the new + # CA and then returned right here without re-signing anything, because the + # NAMES had not changed. The install reported success and the vhost went on + # serving a certificate signed by the CA that had just been replaced, with + # nothing anywhere saying so. + want=$( { cat "$sslpath/ca.cnf" 2>/dev/null + openssl x509 -in "$sslcapem" -noout -fingerprint -sha256 2>/dev/null + } | openssl md5 2>/dev/null) if [[ -e $sslpubcert && -e $stamp && "$(cat "$stamp" 2>/dev/null)" == "$want" ]]; then return 0 fi @@ -3738,23 +4216,52 @@ _createWebLeaf() { # well-formed certificate that no client will accept. Left undetected it # surfaces as a browser error days later with nothing connecting it to the # rename. - if [[ -n $sslcachain && -e $sslcachain ]] && \ - ! openssl verify -CAfile "$rootCAPem" -untrusted "$sslcachain" "$sslpubcert" >>$error_log 2>&1; then + # + # Verified against the root the CHAIN terminates in, not against + # $rootCAPem. Under --web-ca-* the leaf chains to the OTHER server's root + # while $rootCAPem is still this server's own -- validateExternalCA never + # reassigns it -- so the old form failed on every external-CA install and + # printed the box below unconditionally, telling the admin to delete a Web + # zone that was working correctly. + local vtmp vroot="" + vtmp=$(mktemp -d 2>>$error_log) + if [[ -n $vtmp && -n $sslcachain && -e $sslcachain ]]; then + _rootFromChain "$sslcachain" > "${vtmp}/root.pem" 2>>$error_log + if [[ -s ${vtmp}/root.pem ]]; then + vroot="${vtmp}/root.pem" + elif [[ -n $rootCAPem && -f $rootCAPem ]]; then + # A chain carrying no root of its own. FOG's is the only anchor + # available, and for a FOG-issued leaf it is also the right one. + vroot="$rootCAPem" + fi + fi + if [[ -n $vroot ]] && \ + ! openssl verify -CAfile "$vroot" -untrusted "$sslcachain" "$sslpubcert" >>$error_log 2>&1; then echo echo " ###################################################################" echo " # WARNING: the web certificate does not verify against the CA #" - echo " # that issued it. The usual cause is a name outside that CA's #" - echo " # name constraints -- this server was renamed, or gained an #" - echo " # --extra-server-name, after the CA was created. #" - echo " # #" - echo " # Re-run with the name permitted: #" - echo " # --internal-domain #" - echo " # A CA is never re-issued once it exists, so also remove it so #" - echo " # the new constraints take effect: #" - echo " # rm -rf $(_pkiZoneDir web)" + echo " # that issued it. #" + if [[ $externalca == yes ]]; then + echo " # #" + echo " # This server uses an external CA, so check that the leaf, the #" + echo " # intermediate and the root you supplied really belong together: #" + echo " # --web-ca-cert / --web-ca-key / --web-ca-root #" + echo " # Nothing under the FOG PKI tree needs removing for this. #" + else + echo " # The usual cause is a name outside that CA's name constraints #" + echo " # -- this server was renamed, or gained an --extra-server-name, #" + echo " # after the CA was created. #" + echo " # #" + echo " # Re-run with the name permitted: #" + echo " # --internal-domain #" + echo " # A CA is never re-issued once it exists, so also remove it so #" + echo " # the new constraints take effect: #" + echo " # rm -rf $(_pkiZoneDir web)" + fi echo " ###################################################################" echo fi + [[ -n $vtmp ]] && rm -rf "$vtmp" >>$error_log 2>&1 return 0 } # Put the PKI private keys back under root's control, and keep them there. @@ -3836,6 +4343,128 @@ _caTrustLayout() { # # Reads only the certificate, never a key, so it is deliberately placed on the # far side of _hardenPkiPermissions. +# The certificate-authority argument for an HTTPS call this server makes to +# ITSELF. +# +# Sets $selfCacertOpts, which callers splice in as "${selfCacertOpts[@]}". Empty +# when there is nothing to anchor, or when the install is serving plain HTTP -- +# which is the default here, so on most installs this is a no-op and the calls +# Split a PEM bundle into one file per certificate, c1.pem upward, in $2. +_splitPemBundle() { + local src="$1" dir="$2" f found=1 + [[ -n $src && -f $src && -n $dir && -d $dir ]] || return 1 + awk -v d="$dir" '/-----BEGIN CERTIFICATE-----/{n++} n{print > (d "/c" n ".pem")}' \ + "$src" 2>>$error_log + for f in "$dir"/c*.pem; do + [[ -f $f ]] && { found=0; break; } + done + return $found +} +# The self-signed certificate in a chain file, on stdout. That is the root, and +# it is the only member of the bundle whose identity does not depend on the +# file's ORDER -- the writers disagree about order (validateExternalCA writes +# the root first, createWebIntermediateCA appends it last), so selecting on the +# property is the only way to read either. +_rootFromChain() { + local bundle="$1" tmpd f subj issuer st=1 + [[ -n $bundle && -f $bundle ]] || return 1 + tmpd=$(mktemp -d) || return 1 + if _splitPemBundle "$bundle" "$tmpd"; then + for f in "$tmpd"/c*.pem; do + [[ -f $f ]] || continue + subj=$(openssl x509 -in "$f" -noout -subject 2>/dev/null) + issuer=$(openssl x509 -in "$f" -noout -issuer 2>/dev/null) + [[ -z $subj ]] && continue + # -subject prints "subject=..." and -issuer "issuer=...", so compare + # the values rather than the whole line. + if [[ ${subj#subject=} == "${issuer#issuer=}" ]]; then + cat "$f" + st=0 + break + fi + done + fi + rm -rf "$tmpd" >>$error_log 2>&1 + return $st +} +# Every root this server should accept for its OWN web certificate, in one +# file, and $trustAnchorPem naming it. +# +# Both roots, not one: $rootCAPem is what _installCATrustAnchor puts in the +# system store and what fog-client pins, while the root the served leaf +# actually chains to may be a DIFFERENT one entirely -- that is exactly the +# case under --web-ca-cert/--web-ca-key/--web-ca-root, where another server's +# root issued this server's Web CA and $rootCAPem is deliberately left alone. +# Anchoring on $rootCAPem by itself was therefore wrong on every external-CA +# install, and wrong in the silent direction: wget's --ca-certificate REPLACES +# the default bundle, so naming the local root does not add trust, it removes +# the only trust that would have worked. +# +# Deduplicated on fingerprint, not on path: on an ordinary FOG install these +# are the same certificate reached two ways, and appending it twice is +# pointless noise in a file an admin may well end up reading. +_resolveTrustAnchor() { + trustAnchorPem="" + local out="$(_pkiZoneDir web)/ca/.trustAnchor.pem" + local chainroot fp seen="" + mkdir -p "$(dirname "$out")" >>$error_log 2>&1 + : > "$out" 2>>$error_log || return 1 + + if [[ -n $rootCAPem && -f $rootCAPem ]]; then + fp=$(openssl x509 -in "$rootCAPem" -noout -fingerprint -sha256 2>/dev/null) + if [[ -n $fp ]]; then + cat "$rootCAPem" >> "$out" 2>>$error_log + seen="$fp" + fi + fi + if [[ -n $sslcachain && -f $sslcachain ]]; then + chainroot=$(_rootFromChain "$sslcachain") + if [[ -n $chainroot ]]; then + fp=$(printf '%s\n' "$chainroot" \ + | openssl x509 -noout -fingerprint -sha256 2>/dev/null) + if [[ -n $fp && $fp != "$seen" ]]; then + printf '%s\n' "$chainroot" >> "$out" 2>>$error_log + fi + fi + fi + [[ -s $out ]] || return 1 + trustAnchorPem="$out" + return 0 +} +# are unchanged. When it is empty the tool verifies against the system store, +# which is the right answer for a certificate from a public CA. +# +# Why a helper and not just the system store: _installCATrustAnchor() writes +# $rootCAPem there, but it runs at installfog.sh:805 -- AFTER configureHttpd, +# checkWebTier, backupDB and updateDB, which are the calls that need it. On a +# fresh install the store therefore does not know FOG's CA yet at the moment +# those fire. The file exists by then, so naming it explicitly is correct +# whatever the store happens to contain. +# +# $rootCAPem deliberately, not the chain: it is exactly what +# _installCATrustAnchor would have anchored, so the two agree by construction, +# and the leaf's issuing intermediate is served by the web server itself. +# +# Both callers are wget, whose flag is --ca-certificate. It REPLACES the +# default bundle rather than adding to it, and these calls address the server +# by $ipaddress, so both halves have to hold: the served chain must terminate +# in $rootCAPem, and the leaf must cover that address. FOG-issued certificates +# satisfy both by construction. An admin who hand-replaced the certificate has +# satisfied neither, and updateDB says so rather than failing blankly. +# +# These calls used to pass --no-check-certificate. That mattered more than it +# looks: the schema update carries X-Fog-Install-Token, a secret that grants a +# schema deploy on a server with no users yet, and it was being handed to +# whoever answered. +_resolveSelfCacert() { + selfCacertOpts=() + [[ $httpproto == https ]] || return 0 + # The chain's own root as well as $rootCAPem -- see _resolveTrustAnchor for + # why naming $rootCAPem alone broke every --web-ca-* install. + _resolveTrustAnchor >>$error_log 2>&1 || return 0 + [[ -s $trustAnchorPem ]] || return 0 + selfCacertOpts=(--ca-certificate="$trustAnchorPem") +} _installCATrustAnchor() { local anchor="$rootCAPem" st=0 # Default-on, --no-ca-trust to decline, persisted in .fogsettings -- the @@ -4198,6 +4827,53 @@ EOF echo " RewriteCond %{REQUEST_METHOD} ^(TRACE|TRACK)" >> "$etcconf" echo " RewriteRule .* - [F]" >> "$etcconf" echo " RewriteRule /management/other/ca.cert.der$ - [L]" >> "$etcconf" + # GH-978: every path a BOOTLOADER itself fetches must not be + # redirected to an HTTPS it cannot validate. On this line that + # is one directory -- service/ipxe/ -- holding boot.php and + # advanced.php, the menu artwork, refind, grub and memtest. + # + # This RESTORES the redirect's original scope rather than + # punching a new hole in it. It was written in 2017 as + # + # RewriteRule /management/ https://%{HTTP_HOST}%{REQUEST_URI}... + # + # and 2b8bacfed ("Make sure query string is passed properly", + # 2017-04-29) replaced the whole rule with `(.*)` to fix how the + # query string was carried. Widening it from the management UI + # to the entire site was collateral of that fix, not a decision; + # nothing in the commit or its message mentions scope. + # + # Why it cannot simply be "rebuild iPXE with the CA": the + # binaries that need this most are the ones FOG must NOT build. + # downloadipxesecureboot() stages upstream's Microsoft-signed + # shim and iPXE's signed loader, and those are built with no + # TRUST=/CERT= at all -- they can never trust a private CA, by + # construction. With $httpproto=https, default.ipxe chains over + # https and they fail validation; point one at http instead and + # the redirect below returned them to the same untrusted TLS. + # Either way the client prints "Permission denied" and stops, + # which is what GH-978 reported. Secure Boot and --force-https + # were mutually exclusive on this line until this condition. + # + # The cost, stated plainly because it is real: boot.php accepts + # a FOG admin username/password (bootmenu.class.php -- advanced + # menu access, host deletion, quick-image, debug access), so + # that POST becomes possible in cleartext for a client that + # chains over http. What bounds it is that the PXE chain has no + # confidentiality at its root anyway: DHCP hands out the next + # server and default.ipxe arrives over TFTP, both unencrypted. + # Anyone positioned to downgrade a client to http already + # controls the boot chain and can serve their own iPXE. The + # management UI, the API and fog-client are untouched and stay + # redirected. + # + # Conditions guard only the NEXT RewriteRule, and multiple + # RewriteConds are ANDed -- so this pair means "redirect only + # when the request is not for the netboot directory and is not + # already HTTPS". working-1.6 carries the same guard over three + # directories; service/secureboot/ and service/uboot/ do not + # exist on this line. + echo " RewriteCond %{REQUEST_URI} !^${webrootre}service/ipxe/" >> "$etcconf" echo " RewriteCond %{HTTPS} off" >> "$etcconf" # GH-978: ^/?(.*)$ rather than (.*). In vhost context a # RewriteRule pattern is matched against the URL-path WITH its @@ -4354,6 +5030,37 @@ EOF if [[ -n $phpsessdir && $phpsessdir == /* && $phpsessdir != "/" && -d $phpsessdir && $phpsessdir == *session* ]]; then chown -R ${apacheuser}:${apacheuser} "$phpsessdir" >>$error_log 2>&1 fi + # The pool's error log is orphaned by the same user change, + # and it fails more quietly than the session directory: the + # pool cannot open it, so every error_log() call from FOG's + # PHP is discarded and the file stays zero bytes forever. + # Nothing reports it -- not the browser, not the master's own + # error.log -- so it reads as an install with no errors. + # + # Measured on a Fedora nginx install: the RPM ships + # /var/log/php-fpm owned apache:root and www-error.log owned + # apache:apache, the pool runs as $apacheuser after the + # rewrite above, and `test -w` says no to both. + # + # The file is chowned unconditionally; the directory only when + # its own name marks it as php-fpm's. On Debian the log sits + # directly in /var/log, and chowning that to the web user + # would be a far worse bug than the one being fixed. + # + # logrotate keeps the ownership: the packaged php-fpm rule + # carries no `create` line, so a rotated file inherits the + # attributes of the one it replaced. + phpfpmlog=$(sed -n "s/^[;[:space:]]*php_admin_value\[error_log\][[:space:]]*=[[:space:]]*//p" $phpfpmconf | tail -1 | tr -d '"') + if [[ -n $phpfpmlog && $phpfpmlog == /* && $phpfpmlog != "/" && -d $(dirname "$phpfpmlog") ]]; then + [[ -f $phpfpmlog ]] || touch "$phpfpmlog" >>$error_log 2>&1 + chown ${apacheuser}:${apacheuser} "$phpfpmlog" >>$error_log 2>&1 + phpfpmlogdir=$(dirname "$phpfpmlog") + case "$(basename "$phpfpmlogdir")" in + *fpm*|*php*) + chown ${apacheuser}:${apacheuser} "$phpfpmlogdir" >>$error_log 2>&1 + ;; + esac + fi sed -i 's/listen = .*/listen = 127.0.0.1:9000/g' $phpfpmconf >>$error_log 2>&1 sed -i 's/^[;]pm\.max_requests = .*/pm.max_requests = 2000/g' $phpfpmconf >>$error_log 2>&1 sed -i 's/^[;]php_admin_value\[memory_limit\] = .*/php_admin_value[memory_limit] = 256M/g' $phpfpmconf >>$error_log 2>&1 @@ -4506,21 +5213,57 @@ configureHttpd() { sed -i 's/.*max_input_vars\ \=.*$/max_input_vars\ \=\ 250000/g' $phpini >>$error_log 2>&1 errorStat $? dots "Testing and removing symbolic links if found" + # GH-1146: $webdirdest IS ${docroot}fog/, so unlinking it here left the + # "Backing up old data" test below with nothing to find. No + # fog_web_.BACKUP was written, and the management/other/ carry-forward + # further down -- which reads that directory -- silently did nothing, on + # every install whose web root is a symlink. The only trace was a find(1) + # complaint in the error log. Remember where the link pointed so the tree + # is still reachable once the link itself is gone. + priorwebdir="" if [[ -h ${docroot}fog ]]; then + priorwebdir=$(readlink -f "${docroot}fog" 2>>$error_log) rm -f ${docroot}fog >>$error_log 2>&1 fi if [[ -h ${docroot}${webroot} ]]; then + [[ -z $priorwebdir ]] && priorwebdir=$(readlink -f "${docroot}${webroot}" 2>>$error_log) rm -f ${docroot}${webroot} >>$error_log 2>&1 fi + # A link pointing at the document root itself, or at one of its parents, + # is not a FOG tree to copy aside -- it is somebody's whole web server. + # GH-953 is the standing reminder of what taking a path like that at face + # value costs. Nothing below reads $priorwebdir once it is cleared. + if [[ -n $priorwebdir ]]; then + case "${docroot%/}/" in + "${priorwebdir%/}/"*) + priorwebdir="" + ;; + esac + fi errorStat $? dots "Backing up old data" + # Whether either branch below actually copied anything. Both can be false: + # $webdirdest may not exist at all, and $priorwebdir is only set when + # ${docroot}fog was a symlink this run removed. See the report at the end + # of this step for why that has to be said out loud. + webbackedup="" if [[ -d $backupPath/fog_web_${version}.BACKUP ]]; then rm -rf $backupPath/fog_web_${version}.BACKUP >>$error_log 2>&1 fi if [[ -d $webdirdest ]]; then cp -RT "$webdirdest" "${backupPath}/fog_web_${version}.BACKUP" >>$error_log 2>&1 + webbackedup=1 rm -rf ${backupPath}/fog_web_${version}.BACKUP/lib/plugins/accesscontrol rm -rf "$webdirdest" >>$error_log 2>&1 + elif [[ -n $priorwebdir && -d $priorwebdir ]]; then + # Copy only, no removal. The branch above deletes $webdirdest because + # the new tree is about to be written over that exact path. + # $priorwebdir is somewhere else the admin chose, and it was already + # being left behind before this fix -- backing it up is the gain here, + # and deleting it would be a new behaviour nobody asked for. + cp -RT "$priorwebdir" "${backupPath}/fog_web_${version}.BACKUP" >>$error_log 2>&1 + webbackedup=1 + rm -rf ${backupPath}/fog_web_${version}.BACKUP/lib/plugins/accesscontrol fi if [[ $osid -eq 2 ]]; then # GH-953: this removed ${docroot} -- the whole document root, taking any @@ -4543,7 +5286,21 @@ configureHttpd() { if [[ ${docroot%/}/${webrootbare} != ${webdirdest%/} && -n $webrootbare ]]; then linkIfAbsent "${webdirdest%/}" "${docroot%/}/${webrootbare}" fi - errorStat $? + # This step printed "OK" whether or not a fog_web_.BACKUP was written. + # errorStat is reached forty lines after the copy and reports the status of + # the link work above it, so an install that found nothing to preserve + # still told the admin their web root had been backed up. It is reachable + # on any server whose webroot is moved aside between installs -- the tree + # is neither at $webdirdest nor behind a symlink this run removed, so both + # branches are skipped -- and the only trace was the absence of a directory + # nobody looks for until they need it. + webbackupstat=$? + errorStat $webbackupstat skip + if [[ -n $webbackedup ]]; then + echo "OK" + else + echo "Skipped" + fi if [[ $copybackold -gt 0 ]]; then if [[ -d ${backupPath}/fog_web_${version}.BACKUP ]]; then dots "Copying back old web folder as is"; @@ -4673,7 +5430,7 @@ class Config define('PXE_KERNEL', 'bzImage'); define('PXE_KERNEL_RAMDISK', 275000); define('USE_SLOPPY_NAME_LOOKUPS', true); - define('MEMTEST_KERNEL', 'memtest.bin'); + define('MEMTEST_KERNEL', 'mt86plus_x86_64'); define('PXE_IMAGE', 'init.xz'); define('STORAGE_HOST', \"${confighostip}\"); define('STORAGE_FTP_USERNAME', \"${username}\"); @@ -4703,6 +5460,34 @@ class Config define('FOG_THEME', 'default/fog.css'); } }" > "${webdirdest}/lib/fog/config.class.php" + # "skipOk", because this step is not finished until the permissions below + # are set: the OK belongs to the errorStat after them, not to this one. + # A failure here still aborts loudly -- that is the half skipOk does not + # touch. + errorStat $? "skipOk" + # This file holds ${DB_password}, both FTP passwords (${SVC_password}, and + # the storage node account the same value backs) and the schema bootstrap + # token. It is written by a plain redirect, so without this it lands at + # whatever umask root is carrying -- 0644 on every distro we support -- and + # every local account on the server can read all of them. The FTP + # credential is fleet-wide, not per-server. + # + # Same reasoning as .fogsettings, which is 0600 for two of the same + # secrets. This one is not 0600 because it is read by PHP rather than by + # the installer: the web tier includes it on every request, and the chown + # below hands it to ${apacheuser}. + # + # Kept at 0640 to match the 1.6 line rather than tightened to 0600, which + # would also work here -- every 1.5 daemon runs as root, so the web user is + # the only non-root reader. Divergence between the two installers over one + # bit is not worth an unnoticed reader (a site script, a plugin's cron) + # breaking on the line people actually run in production. + # + # Set here rather than left to the chown -R at the end of this function: a + # mode is only meaningful once the group is right, and a failure between + # the two should not leave a window where it is neither. + chown ${apacheuser}:${apacheuser} "${webdirdest}/lib/fog/config.class.php" >>$error_log 2>&1 + chmod 0640 "${webdirdest}/lib/fog/config.class.php" >>$error_log 2>&1 errorStat $? dots "Creating paths file" # GH-850: hand the installer's $fogprogramdir to the PHP runtime so @@ -4813,14 +5598,31 @@ downloadfiles() { # make sure we download the most recent hash file to start with if [[ -f $hashfile ]]; then rm -f $hashfile - curl --silent -kOL $hashurl >>$error_log 2>&1 + curl --silent -OL --connect-timeout $inetConnectTimeout \ + --speed-time 30 --speed-limit 1024 $hashurl >>$error_log 2>&1 fi - while [[ $checksum -ne 0 && $cnt -lt 10 ]]; do + # Eight URLs, ten rounds, two curls each: 160 connects, none of them + # bounded, all of them silent under one "Downloading kernel, init and + # fog-client binaries" line. On a host with no route out that was the + # single longest stall the installer could produce. --connect-timeout + # bounds an unreachable host and --speed-time/--speed-limit a transfer + # that opens and then stops; --max-time is deliberately absent, because + # these are multi-megabyte kernels and a slow link must still finish. + # When checkInternetConnection has already established the host is + # unreachable there is nothing to retry FOR, so make one attempt. + tries=10 + [[ $internet_ok -ne 1 ]] && tries=1 + while [[ $checksum -ne 0 && $cnt -lt $tries ]]; do [[ -f $hashfile ]] && sha256sum --check $hashfile >>$error_log 2>&1 checksum=$? if [[ $checksum -ne 0 ]]; then - curl --silent -kOL $url >>$error_log - curl --silent -kOL $hashurl >>$error_log + # No -k, same reasoning as fetchipxeasset(): the hash file + # travels the same connection as the payload, so skipping + # verification here voids the checksum too. + curl --silent -OL --connect-timeout $inetConnectTimeout \ + --speed-time 30 --speed-limit 1024 $url >>$error_log + curl --silent -OL --connect-timeout $inetConnectTimeout \ + --speed-time 30 --speed-limit 1024 $hashurl >>$error_log fi let cnt+=1 done @@ -5350,7 +6152,7 @@ _resignKernels() { echo " and re-run the installer, or Secure Boot clients will not boot." return 0 fi - dots "Signing FOS kernels for Secure Boot" + dots "Signing FOS kernels and Memtest86+ for Secure Boot" local kernel kpath failed=0 certpem # sbsign/sbverify take PEM only; the admin may well have handed us the DER # copy that mokutil wanted. See _secureBootCertPem(). @@ -5360,7 +6162,10 @@ _resignKernels() { echo " Secure Boot clients will not boot until this is fixed." return 0 } - for kernel in bzImage bzImage32 arm_Image; do + # The two Memtest86+ binaries ride along: each is a bzImage that is also + # a PE, and on a UEFI client iPXE chains it as a PE, which under Secure + # Boot needs the same countersignature the kernels get (#321). + for kernel in bzImage bzImage32 arm_Image mt86plus_x86_64 mt86plus_i586; do kpath="${webdirdest}/service/ipxe/${kernel}" [[ -f $kpath ]] || continue # Already carrying our signature means nothing was re-downloaded since @@ -5488,6 +6293,42 @@ _keaAppleClass() { } EOFAPL } +_keaRunAs() { + # Print the account "kea-dhcp4 -t" must run as to read $1, or nothing for root. + # + # Debian/Ubuntu ship /etc/kea as 0750 owned by the service account (_kea), + # and their AppArmor profile grants kea-dhcp4 "/etc/kea/** r" while + # deliberately withholding cap_dac_read_search and cap_dac_override. Root is + # neither the directory's owner nor in its group, so a root-run validation + # cannot even traverse the directory: Kea reports "Unable to open file + # " for a file that plainly exists, and the install aborts (#1039). + # The daemon itself was never affected because systemd runs it as _kea. + # + # Validating as the directory's owner needs no DAC bypass at all, so it + # succeeds with the AppArmor profile intact. Do NOT "fix" this by putting the + # profile into complain mode or deleting it -- that disables a protection the + # distro shipped on purpose, on a box the admin did not ask us to weaken. + # Where /etc/kea is root-owned (RedHat, Arch, Alpine) this returns nothing + # and the validation runs as root exactly as before. + local owner + owner=$(stat -c '%U' "$(dirname "$1")" 2>/dev/null) + [[ -z $owner || $owner == root || $owner == UNKNOWN ]] && return 0 + id -u "$owner" >/dev/null 2>&1 || return 0 + printf '%s' "$owner" +} +_keaValidate() { + # Syntax-check $1, dropping to the config directory's owner when root cannot + # read it (see _keaRunAs). Returns kea-dhcp4's exit status. + local runas + runas=$(_keaRunAs "$1") + if [[ -z $runas ]]; then + kea-dhcp4 -t "$1" >>$error_log 2>&1 + elif command -v runuser >/dev/null 2>&1; then + runuser -u "$runas" -- kea-dhcp4 -t "$1" >>$error_log 2>&1 + else + su -s /bin/sh -c "kea-dhcp4 -t '$1'" "$runas" >>$error_log 2>&1 + fi +} _writeKeaConfig() { # $1 = target file, $2 = client-classes block. Reads $interface, $ipaddress, # $network, $cidr, $startrange, $endrange and $optdata from the caller's scope. @@ -5518,6 +6359,12 @@ $2 } } EOFKEA + # The service account has to be able to read this, and a hardened root umask + # (027/077) would otherwise leave it unreadable to anyone but root -- which + # breaks the daemon, not just the syntax check. 0644 is the mode the distro + # packages ship this file with; the generated config holds no credentials + # (the lease database is memfile). + chmod 0644 "$1" >>$error_log 2>&1 } configureKeaDHCP() { local cidr=$(mask2cidr $submask) @@ -5542,15 +6389,28 @@ configureKeaDHCP() { return 1 fi if command -v kea-dhcp4 >/dev/null 2>&1; then - if ! kea-dhcp4 -t "$target" >>$error_log 2>&1; then + if ! _keaValidate "$target"; then echo "Failed" echo "Kea base configuration failed validation (kea-dhcp4 -t); see $error_log" + # "Unable to open file" against a file we just wrote and can stat is + # never a syntax error -- it is a mandatory access control denial + # (AppArmor on Debian/Ubuntu, SELinux on RedHat) stopping kea-dhcp4 + # from reading it. Say so, because the generic message sends people + # hunting for a JSON typo that isn't there (#1039). + if [[ -s $target ]] && tail -n 20 "$error_log" 2>/dev/null | grep -q 'Unable to open file'; then + echo "" + echo " * $target exists and is readable, so this is not a syntax error." + echo " Something is denying kea-dhcp4 access to it. Check:" + echo " dmesg | grep -i 'apparmor.*kea' (Debian/Ubuntu)" + echo " ausearch -m avc -c kea-dhcp4 (RedHat/Rocky)" + echo " Please report this with that output rather than disabling AppArmor." + fi return 1 fi # Tier 2: best-effort Apple BSDP; drop if Kea rejects it. _writeKeaConfig "$tmp" "${baseclasses}, ${appleclass}" - if kea-dhcp4 -t "$tmp" >>$error_log 2>&1; then + if _keaValidate "$tmp"; then mv -f "$tmp" "$target" else rm -f "$tmp" @@ -5571,18 +6431,18 @@ writeKeaSample() { local target="${webdirdest%/}/kea-dhcp4.conf.fog-sample" [[ -z $webdirdest ]] && target="/etc/kea/kea-dhcp4.conf.fog-sample" [[ -d $(dirname "$target") ]] || return 0 - local sampleip - sampleip=$(ip -4 -o addr show $interface | awk -F'([ /])+' '/global/ {print $4}') - [[ -z $sampleip ]] && sampleip="$ipaddress" - [[ -z $submask ]] && submask=$(cidr2mask $(getCidr $interface)) - local network=$(mask2network $sampleip $submask) + # GH-1747: the subnet comes from $ipaddress, the address FOG advertises. + # Every global address on the interface used to land here, unquoted, so a + # second address became the mask. + [[ -z $submask ]] && submask=$(cidr2mask $(getCidr $interface $ipaddress)) + local network=$(mask2network $ipaddress $submask) local cidr=$(mask2cidr $submask) local startrange=$(addToAddress $network 10) # GH-667: an interface with no brd flag, or any failure inside these # helpers, used to leave endrange holding an error string that went # straight into the generated config. Fall back to the broadcast computed # from the network and mask we already have. - local broadcast=$(interface2broadcast $interface) + local broadcast=$(interface2broadcast $interface $ipaddress) [[ $(validip $broadcast) -ne 0 ]] && broadcast=$(mask2broadcast $network $submask) local endrange=$(subtract1fromAddress $broadcast) [[ $(validip $endrange) -ne 0 ]] && endrange=$(subtract1fromAddress $(mask2broadcast $network $submask)) @@ -5625,15 +6485,16 @@ configureDHCP() { fi case $bldhcp in 1) - serverip=$(ip -4 -o addr show $interface | awk -F'([ /])+' '/global/ {print $4}') - [[ -z $serverip ]] && serverip=$(/sbin/ifconfig $interface | grep -oE 'inet[:]? addr[:]?([0-9]{1,3}\.){3}[0-9]{1,3}' | awk -F'(inet[:]? ?addr[:]?)' '{print $2}') - [[ -z $submask ]] && submask=$(cidr2mask $(getCidr $interface)) - network=$(mask2network $serverip $submask) + # GH-1747: the subnet comes from $ipaddress, the address handed out + # as next-server. Every global address on the interface used to land + # here, unquoted, so a second address became the mask. + [[ -z $submask ]] && submask=$(cidr2mask $(getCidr $interface $ipaddress)) + network=$(mask2network $ipaddress $submask) [[ -z $startrange ]] && startrange=$(addToAddress $network 10) # GH-667: same guard -- never let a helper's failure become the # value that lands in dhcpd.conf. if [[ -z $endrange ]]; then - broadcast=$(interface2broadcast $interface) + broadcast=$(interface2broadcast $interface $ipaddress) [[ $(validip $broadcast) -ne 0 ]] && broadcast=$(mask2broadcast $network $submask) endrange=$(subtract1fromAddress $broadcast) [[ $(validip $endrange) -ne 0 ]] && endrange=$(subtract1fromAddress $(mask2broadcast $network $submask)) diff --git a/lib/common/input.sh b/lib/common/input.sh index 32149ed81d..c5d5869f7a 100755 --- a/lib/common/input.sh +++ b/lib/common/input.sh @@ -125,19 +125,26 @@ while [[ -z $interface ]]; do # want every address -- the certificate SANs and the apache ServerAlias -- # and normalizeIpAddress() then reduces $ipaddress to the primary, which is # what every other consumer has always assumed it was. - ipaddress=$(ip -4 addr show $interface | awk '$1 == "inet" {gsub(/\/.*$/, "", $2); print $2}') + # + # GH-1747: global addresses only, and never link-local 169.254.0.0/16. A + # link-local address appears when DHCP gets no answer on the deployment NIC. + # No client can reach FOG there, and listed first it became the primary. + ipaddress=$(ip -4 addr show $interface | awk '$1 == "inet" && / scope global / && $2 !~ /^169\.254\./ {gsub(/\/.*$/, "", $2); print $2}') ipaddresses="$ipaddress" if [[ $(validip $ipaddress) -ne 0 ]]; then echo echo " * The interface $interface does not seem to have a valid IP configured to it." + # With -y nothing can pick another interface, so this loop would + # repeat the same answer forever. + [[ -n $autoaccept ]] && exit 1 interface="" continue fi - submask=$(cidr2mask $(getCidr $interface)) - if [[ -z $submask ]]; then - submask=$(/sbin/ifconfig -a | grep $ipaddress -B1 | awk -F'[netmask ]+' '{print $4}' | head -n2) - submask=$(mask2cidr $submask) - fi + # The mask of the primary address (the first one listed), not of whichever + # address getCidr found. The ifconfig fallback that followed stored + # mask2cidr's prefix length in $submask, and ifconfig is not installed + # before the package step. An empty mask is derived again in configureDHCP. + submask=$(cidr2mask $(getCidr $interface ${ipaddress%%[[:space:]]*})) done if [[ $strSuggestedHostname == $ipaddress ]]; then strSuggestedHostname=$(hostnamectl --static) diff --git a/packages/pki/fog-mint-web-ca b/packages/pki/fog-mint-web-ca new file mode 100755 index 0000000000..9721cbbef7 --- /dev/null +++ b/packages/pki/fog-mint-web-ca @@ -0,0 +1,206 @@ +#!/bin/bash +# +# FOG is a computer imaging solution. +# Copyright (C) 2007 Chuck Syperski & Jian Zhang +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Issue a Web CA for ANOTHER FOG server from this server's root. +# +# The case this exists for: several independent FOG installs -- not storage +# nodes, separate servers with their own databases -- that an admin wants to +# roll up to a single trust anchor, so one certificate in a browser or a system +# trust store covers all of them instead of one per server. +# +# Storage nodes on this line still generate their own self-signed CA -- the +# automatic issuance from the master is a 1.6 feature and is not present here -- +# so this applies to nodes just as it does to separate servers. +# +# Why a script rather than a documented openssl invocation: the permitted name +# set is easy to get wrong in a way that is invisible until the far server's +# web tier refuses to start. _defaultServerNames() puts `fogserver` and +# `fog-server` on EVERY FOG leaf whatever the host is called, so a CA +# constrained to only the satellite's own hostname cannot sign the certificate +# it was minted for. This gets that right, and refuses to emit a CA whose +# constraints would reject its own server's certificate. + +set -u + +CONF="/opt/fog/.fogsettings" + +usage() { + cat < [extra-dns-name ...] + +Issues a name-constrained Web CA for the FOG server at /, signed +by THIS server's root, and writes a tarball holding the three files that +server's installer needs. + + the far server's hostname, exactly as \`hostname\` reports + it there. It goes into the certificate's name constraints, + so a guess that does not match will produce a CA that + cannot sign that server's certificate. + the far server's IP address. + extra-dns-name ... any additional names that server is installed with + (--extra-server-name / --internal-domain). Omitting one + means its certificate will not verify. + +Install the result on the far server with: + + tar -xzf -webca.tar.gz + ./installfog.sh --web-ca-cert webca.pem \\ + --web-ca-key webca.key \\ + --web-ca-root fog-root.pem + +Environment overrides: ROOT_CERT, ROOT_KEY, OUTDIR. +See docs/MULTI_SERVER_CA.md for the full procedure and the alternatives. +EOF +} + +[[ ${1:-} == -h || ${1:-} == --help ]] && { usage; exit 0; } +[[ $# -ge 2 ]] || { usage; exit 1; } + +host="$1"; ip="$2"; shift 2 +extras=("$@") + +[[ $EUID -eq 0 ]] || { echo "This must be run as root -- the root CA key is root-only by design." >&2; exit 1; } + +# .fogsettings is the source of truth for where this install keeps things, the +# same as it is for the installer and for fog-offline-ca-key. Sourcing it rather +# than hardcoding /opt/fog is what makes this work on a server installed with +# --fogprogramdir. +if [[ -r $CONF ]]; then + # shellcheck disable=SC1090 + . "$CONF" +elif [[ -r /etc/fog/fog.conf ]]; then + . /etc/fog/fog.conf + [[ -r "${fogprogramdir}/.fogsettings" ]] && . "${fogprogramdir}/.fogsettings" +fi +: "${fogprogramdir:=/opt/fog}" +: "${sslpath:=${fogprogramdir}/snapins/ssl}" + +# $rootCAKey is never persisted to .fogsettings -- installfog.sh re-derives it +# every run -- so this falls to the canonical path, the same as fog-offline-ca-key. +ROOT_CERT="${ROOT_CERT:-${rootCAPem:-${sslpath}/CA/.fogCA.pem}}" +ROOT_KEY="${ROOT_KEY:-${rootCAKey:-${fogprogramdir}/pki/root/ca/.fogCA.key}}" +OUTDIR="${OUTDIR:-/root/fog-web-cas}" + +[[ -r $ROOT_CERT ]] || { echo "Cannot read the root certificate at ${ROOT_CERT}" >&2; exit 1; } +if [[ ! -r $ROOT_KEY ]]; then + echo "Cannot read the root private key at ${ROOT_KEY}" >&2 + echo "If it is offline (fog-offline-ca-key), restore it, run this, then take it away again." >&2 + exit 1 +fi + +# A root carrying pathlen:0 cannot anchor an intermediate. Signing anyway would +# produce a CA that verifies nowhere, which is the failure the installer's own +# _rootCACanIssue() exists to avoid -- so refuse here for the same reason. +if openssl x509 -in "$ROOT_CERT" -noout -ext basicConstraints 2>/dev/null | grep -q "pathlen:0"; then + echo "Refusing to continue: the root at ${ROOT_CERT} carries pathlen:0," >&2 + echo "which forbids any CA beneath it. Nothing signed by it would verify." >&2 + exit 1 +fi + +short="${host%%.*}" +work="$(mktemp -d)" || exit 1 +trap 'rm -rf "$work"' EXIT +mkdir -p "$OUTDIR" || exit 1 +chmod 0700 "$OUTDIR" + +# The permitted name set. `fogserver` and `fog-server` are NOT optional: +# _defaultServerNames() puts both on every FOG leaf regardless of hostname, so a +# CA that cannot sign them cannot sign the certificate it exists for. +# +# The parent domain is deliberately NOT granted. "fog2.lan" would grant DNS:lan +# -- the whole .lan space -- and this key lives on a satellite server, so the +# blast radius if it is stolen is exactly what these constraints bound. The FQDN +# and the short name are both permitted, which is all _defaultServerNames emits +# unless the admin added names, and those are what the extra arguments are for. +perm="" +add_dns() { [[ -n ${1:-} ]] && perm="${perm},permitted;DNS:$1"; } +add_dns "$host" +[[ $short != "$host" ]] && add_dns "$short" +add_dns fogserver +add_dns fog-server +for e in ${extras+"${extras[@]}"}; do add_dns "$e"; done +perm="${perm},permitted;IP:${ip}/255.255.255.255" +perm="${perm},permitted;IP:127.0.0.0/255.0.0.0" + +cat > "$work/int.cnf" </dev/null || st=1 +openssl req -new -sha512 -key "$work/webca.key" -out "$work/webca.csr" \ + -config "$work/int.cnf" 2>/dev/null || st=1 +# 30 years, matching _issueIntermediateCA: an intermediate is a CA too, and +# renewing it means re-issuing the leaf beneath it, not a routine rotation. +openssl x509 -req -in "$work/webca.csr" -CA "$ROOT_CERT" -CAkey "$ROOT_KEY" \ + -CAcreateserial -sha512 -days 10950 -extensions v3_int \ + -extfile "$work/int.cnf" -out "$work/webca.pem" 2>/dev/null || st=1 +[[ $st -eq 0 ]] || { echo "Failed to issue the intermediate." >&2; exit 1; } + +openssl verify -CAfile "$ROOT_CERT" "$work/webca.pem" >/dev/null 2>&1 || { + echo "The issued intermediate does not verify against ${ROOT_CERT}." >&2 + exit 1 +} + +# Prove the constraints permit the certificate the far server will actually +# build for itself, rather than discovering it there as a web tier that will not +# start. This is the check that makes the script worth having. +probe_sans="IP:${ip},DNS:${host},DNS:fogserver,DNS:fog-server" +[[ $short != "$host" ]] && probe_sans="${probe_sans},DNS:${short}" +for e in ${extras+"${extras[@]}"}; do probe_sans="${probe_sans},DNS:${e}"; done +openssl req -newkey rsa:2048 -nodes -keyout "$work/probe.key" -out "$work/probe.csr" \ + -subj "/CN=${host}" 2>/dev/null +printf 'subjectAltName=%s\n' "$probe_sans" > "$work/probe.ext" +openssl x509 -req -in "$work/probe.csr" -CA "$work/webca.pem" -CAkey "$work/webca.key" \ + -CAcreateserial -days 1 -extfile "$work/probe.ext" -out "$work/probe.pem" 2>/dev/null +cat "$ROOT_CERT" "$work/webca.pem" > "$work/probe-chain.pem" +if ! openssl verify -CAfile "$work/probe-chain.pem" "$work/probe.pem" >/dev/null 2>&1; then + echo "Refusing to emit this CA: a certificate carrying the names that server" >&2 + echo "will actually request does not verify under it." >&2 + echo " names probed: ${probe_sans}" >&2 + echo "Check matches what that server reports, and pass any" >&2 + echo "--extra-server-name/--internal-domain values as extra arguments." >&2 + exit 1 +fi + +cp "$ROOT_CERT" "$work/fog-root.pem" +chmod 0600 "$work/webca.key" +chmod 0644 "$work/webca.pem" "$work/fog-root.pem" +tar -C "$work" -czf "${OUTDIR}/${short}-webca.tar.gz" webca.pem webca.key fog-root.pem || exit 1 +chmod 0600 "${OUTDIR}/${short}-webca.tar.gz" + +echo "Issued: CN=FOG Web CA - ${host}" +echo " bundle: ${OUTDIR}/${short}-webca.tar.gz" +echo " permits: ${perm#,}" +echo +echo "Copy it to ${host}, then there:" +echo " tar -xzf ${short}-webca.tar.gz" +echo " ./installfog.sh --web-ca-cert webca.pem --web-ca-key webca.key --web-ca-root fog-root.pem" diff --git a/packages/secureboot/fog-enroll-mok.sh b/packages/secureboot/fog-enroll-mok.sh index 14e1a5f2f0..85ad9a0792 100755 --- a/packages/secureboot/fog-enroll-mok.sh +++ b/packages/secureboot/fog-enroll-mok.sh @@ -69,9 +69,24 @@ case "$answer" in *) echo; echo " Aborted -- nothing was changed."; pause; exit 1 ;; esac +# --test-key interrogates MokList -- shim's trust store -- and nothing else. The +# check is right for what this script does: it IS the MOK enroller, and +# re-enrolling a MOK genuinely is a no-op. The MESSAGE used to be wrong, drawing +# a machine-wide "nothing to do" from a store-specific test (GH-1266). It is not +# the same question: a MOK only helps where shim is in the boot chain -- firmware +# never reads MokList -- so booting a FOG-signed binary DIRECTLY needs the +# certificate in db, which this script neither reads nor writes. An admin setting +# up the shim-less route was being told to stop. if mokutil --test-key "$cert" 2>/dev/null | grep -qi "already enrolled"; then echo - echo " This key is already enrolled on this machine. Nothing to do." + echo " This key is already enrolled in MokList, so this script has nothing" + echo " left to do. MokList is shim's own trust store: it covers anything" + echo " booted through shim, which is the normal FOG PXE path." + echo + echo " It does NOT cover the firmware's db. This script does not look at db" + echo " and does not write to it. If you are setting up a shim-less boot --" + echo " firmware loading a FOG-signed binary directly -- MOK.der still has to" + echo " go into db, which is a separate step in the Secure Boot guide." pause exit 0 fi diff --git a/packages/service/lib/service_lib.php b/packages/service/lib/service_lib.php index f44c2a7ebf..8bcf0e546f 100644 --- a/packages/service/lib/service_lib.php +++ b/packages/service/lib/service_lib.php @@ -2,7 +2,7 @@ /** * Service library * - * PHP version 5 + * PHP version 7.4+ * * @category Service_Lib * @package FOGProject diff --git a/packages/web/api/index.php b/packages/web/api/index.php index 7d717c6d18..60c068602c 100644 --- a/packages/web/api/index.php +++ b/packages/web/api/index.php @@ -2,7 +2,7 @@ /** * Index/handler for api subsystem. * - * PHP Version 5 + * PHP version 7.4+ * * @category APIHandler * @package FOGProject diff --git a/packages/web/client/download.php b/packages/web/client/download.php index 6d751d25f3..9efc72f2d4 100644 --- a/packages/web/client/download.php +++ b/packages/web/client/download.php @@ -2,7 +2,7 @@ /** * Downloads fog client and utilitie files. * - * PHP version 5 + * PHP version 7.4+ * * @category Download * @package FOGProject diff --git a/packages/web/client/index.php b/packages/web/client/index.php index 07bbc45ada..2b1a2ad394 100644 --- a/packages/web/client/index.php +++ b/packages/web/client/index.php @@ -2,7 +2,7 @@ /** * Redirects calls to client/index.php to main page. * - * PHP version 5 + * PHP version 7.4+ * * @category Redirect * @package FOGProject diff --git a/packages/web/commons/index.php b/packages/web/commons/index.php index 9c5f111645..7b597971c2 100644 --- a/packages/web/commons/index.php +++ b/packages/web/commons/index.php @@ -2,7 +2,7 @@ /** * Redirects calls to commons/index.php to main page. * - * PHP version 5 + * PHP version 7.4+ * * @category Redirect * @package FOGProject diff --git a/packages/web/commons/init.php b/packages/web/commons/init.php index 12b9b0e571..89b26e41b1 100644 --- a/packages/web/commons/init.php +++ b/packages/web/commons/init.php @@ -78,6 +78,21 @@ public function __construct() if (is_readable($fogPaths)) { require_once $fogPaths; } + // _configureSessionStorage() needs the base path below, and it has to + // run before session_start() at the end of this constructor -- which is + // long before System::__construct applies its own /opt/fog fallback. So + // apply the identical fallback here, guarded, and System then defers to + // it: both sites use if (!defined(...)), so whichever runs first wins + // and they cannot disagree. + if (!defined('FOG_BASE_DIR')) { + define('FOG_BASE_DIR', '/opt/fog'); + } + // FOG's own PHP session store. Created 0700 by the installer and + // pointed at by _configureSessionStorage() below; see that method for + // why FOG stops sharing the distro's session directory. + if (!defined('FOG_SESSION_DIR')) { + define('FOG_SESSION_DIR', FOG_BASE_DIR . DS . 'sessions'); + } $regext = '#^.*\.(report|event|class|hook|page)\.php$#'; $paths = new RegexIterator( @@ -93,11 +108,171 @@ public function __construct() spl_autoload_extensions('.class.php,.page.php,.event.php,.hook.php,.report.php'); spl_autoload_register(); - if (session_status() !== PHP_SESSION_ACTIVE) { + self::_configureSessionStorage(); + + /* + * Start a session only when there is one to resume or something has + * asked for one. + * + * This used to be unconditional, and 59 entry points reach it through + * commons/base.inc.php -- including every file under service/ and + * status/, the API and the iPXE endpoints. None of those can carry a + * cookie back, so each request allocated a session that was written + * once, never read again, and left for gc to clean up. A PXE boot or + * a fleet of fog-clients polling on a timer is a steady stream of + * them. + * + * Browser flows are unaffected: the login page GET declares + * FOG_WANTS_SESSION and creates the session, and every request after + * that presents the cookie, so the first arm matches. The API accepts + * a session cookie when a browser sends one and falls back to token + * auth when it does not -- both still work, because the cookie is + * exactly the signal being tested. + * + * Safe because nothing on the browser-less paths uses session state: + * there is not one $_SESSION write under service/, status/, lib/ + * reg-task, lib/client or lib/service, and setMessage() already + * returns early when no session is active (fogbase.class.php). + */ + /* + * An EMPTY cookie value is not a session to resume. isset() is true for + * "PHPSESSID=", so the gate opened for it, session.use_strict_mode found + * no id to resume, and PHP minted a fresh empty session -- exactly the + * per-request throwaway this gate exists to prevent, just arriving + * through a header instead of through the unconditional session_start() + * it replaced. + * + * FOGURLRequests was sending precisely that from every session-less + * caller (fixed at source there too, but the check belongs here: this is + * the chokepoint every entry point reaches, and a sender is easy to + * reintroduce). Browsers are unaffected -- a real session presents a + * non-empty id -- and a browser holding an emptied cookie now gets the + * same treatment as one holding none, which is the correct answer. + */ + $hasSession = ($_COOKIE[session_name()] ?? '') !== ''; + if (session_status() !== PHP_SESSION_ACTIVE + && ($hasSession || defined('FOG_WANTS_SESSION')) + ) { session_start(); } } + /** + * Point PHP's session store at FOG's own directory and give it a lifetime + * that matches FOG's own session policy. + * + * WHY THIS EXISTS + * + * FOG offers FOG_ALWAYS_LOGGED_IN and FOG_INACTIVITY_TIMEOUT ("Between 1 + * and 24 by hours") and enforces them itself in User::_isLoggedIn(). But + * PHP deletes the session FILE out from under that on its own schedule, + * and the stock session.gc_maxlifetime is 1440 seconds -- 24 minutes. + * Verified identical on Fedora 44, Rocky 9.8, Rocky 10.2 and Ubuntu 26.04. + * + * So on every supported distro, any FOG idle policy longer than 24 minutes + * was a lie: the collector reaped the file, session.use_strict_mode + * (set above) then refused the browser's now-orphaned cookie and issued a + * fresh empty session, and the user was bounced to the login page. Silently + * -- the "You were logged out due to inactivity" toast comes from FOG's own + * inactivity branch, which had not fired and, with FOG_ALWAYS_LOGGED_IN on, + * cannot fire at all. Absence of that toast is the tell that PHP did it. + * + * WHY A PRIVATE DIRECTORY AND NOT JUST A LONGER LIFETIME + * + * gc_maxlifetime is a property of the save_path, not of the application: + * PHP's collector scans the whole directory. Raising it while sharing the + * distro's session directory imposes FOG's retention on every other PHP + * application on the box. Taking our own directory makes the setting mean + * what it says and confines the blast radius to FOG. + * + * It also fixes the ownership problem the installer works around today. + * Session files ARE authentication tokens, so this directory is 0700 and + * owned by the pool user -- deliberately NOT the sticky 1777 that + * FOG_CACHE_DIR uses, because world-readable session files would let any + * local account steal an admin session. + * + * WHY THE COLLECTOR SETTINGS COME WITH IT + * + * Owning the path means owning its garbage collection, and this is the + * trap: Debian and Ubuntu ship session.gc_probability = 0 and clean up + * from cron instead, via /usr/lib/php/sessionclean. That script discovers + * what to clean by running `php -c /php.ini` and reading + * session.save_path out of the INI -- so a path set at runtime here is + * invisible to it. With PHP's in-process collector disabled and the cron + * job looking elsewhere, nothing would ever clean this directory and + * expired session files would accumulate forever. Re-enabling the + * in-process collector is what makes the directory self-maintaining on + * every distro. (Confirmed on Ubuntu 26.04: gc_probability = 0, timer + * phpsessionclean.timer present.) + * + * 86400 is the ceiling of FOG_INACTIVITY_TIMEOUT's own documented range, + * chosen so PHP can never contradict whatever policy is set in the UI. + * FOG_INACTIVITY_TIMEOUT stays the authority on when a user is logged out; + * this is only the floor that stops PHP pre-empting it. + * + * @return void + */ + private static function _configureSessionStorage(): void + { + // Nothing to do once a session is running -- these are all read at + // session_start() and ini_set() on them would silently do nothing. + if (session_status() === PHP_SESSION_ACTIVE) { + return; + } + // Created by the installer, but created here too so an install whose + // installer predates this still gets a working private store rather + // than falling back and keeping the 24-minute bug. 0700 before any + // session file can land in it, never afterwards. + if (!is_dir(FOG_SESSION_DIR)) { + @mkdir(FOG_SESSION_DIR, 0700, true); + } + // A directory we cannot write is worse than the shared one: every + // session_start() would fail and nobody could log in at all. Fall back + // to PHP's configured path, which is exactly today's behaviour, and say + // why -- a silent fallback here would look identical to the bug being + // fixed. + if (!is_dir(FOG_SESSION_DIR) || !is_writable(FOG_SESSION_DIR)) { + error_log( + sprintf( + 'FOG: session directory %s is missing or not writable by %s;' + . ' falling back to the system session store, where PHP will' + . ' expire sessions after session.gc_maxlifetime (%ss)' + . ' regardless of FOG_INACTIVITY_TIMEOUT.', + FOG_SESSION_DIR, + (get_current_user() ?: 'the web user'), + ini_get('session.gc_maxlifetime') + ) + ); + return; + } + ini_set('session.save_path', FOG_SESSION_DIR); + ini_set('session.gc_maxlifetime', '86400'); + // See the docblock: Debian/Ubuntu disable the in-process collector. + // These are PHP's own upstream defaults, restored for our path. + ini_set('session.gc_probability', '1'); + ini_set('session.gc_divisor', '1000'); + // ini_set() on session.* returns false and changes nothing once output + // has been flushed, and a pool declaring session.save_path with + // php_admin_value rather than php_value makes it unsettable outright. + // Either way PHP keeps the system store and its 24-minute lifetime -- + // which is the bug this method exists to fix, so it must not pass + // unremarked. Read the value back rather than trusting the return. + if (ini_get('session.save_path') !== FOG_SESSION_DIR) { + error_log( + sprintf( + 'FOG: could not point session.save_path at %s (it is still' + . ' %s). A php_admin_value in the php-fpm pool, or output' + . ' already flushed this request, will do this. Sessions' + . ' will expire after %ss regardless of' + . ' FOG_INACTIVITY_TIMEOUT.', + FOG_SESSION_DIR, + (ini_get('session.save_path') ?: 'the system default'), + ini_get('session.gc_maxlifetime') + ) + ); + } + } + public static function language(string $lang = 'en'): void { $validLangs = ['de' => 'DE', 'en' => 'US', 'es' => 'ES', 'eu' => 'ES', 'fr' => 'FR', 'it' => 'IT', 'pt' => 'BR', 'zh' => 'CN', 'ja' => 'JP']; @@ -110,6 +285,18 @@ public static function language(string $lang = 'en'): void $apppath = realpath(__DIR__ . '/../management/languages'); setlocale(LC_MESSAGES, $lang . ".UTF-8"); bindtextdomain($domain, $apppath); + // The catalogs are UTF-8 and every page declares UTF-8, but gettext + // converts a translation from the catalog's charset to the codeset of + // LC_CTYPE -- not LC_MESSAGES -- and replaces anything the target + // cannot represent with a literal `?`. php-fpm commonly starts with + // LC_CTYPE=C, whose codeset is ASCII, so the Chinese dashboard title + // reached the browser as `???` -- three bytes 3f3f3f -- even though + // the right catalog had been selected and the response declared + // charset=UTF-8 (GH-1720). + // Binding the domain's output codeset fixes that without depending on + // a zh_CN.UTF-8/ja_JP.UTF-8/... locale being generated on the host, + // which setting LC_CTYPE would. + bind_textdomain_codeset($domain, 'UTF-8'); textdomain($domain); } diff --git a/packages/web/commons/schema.php b/packages/web/commons/schema.php index e0ad9cb672..e0fd113430 100644 --- a/packages/web/commons/schema.php +++ b/packages/web/commons/schema.php @@ -2,7 +2,7 @@ /** * Schema layout for creating the database. * - * PHP version 5 + * PHP version 7.4+ * * @category Redirect * @package FOGProject @@ -3968,3 +3968,695 @@ . "VALUES " . "(14, 'fog.enrollsecureboot', 'Enroll Secure Boot Key', '0', '2', NULL)", ); +// 280 +$this->schema[] = array( + // taskLog gains a type and a body, so a task can log something that is + // not a state change. Ported from 1.6 schema 338 (GH-1206/#1208), which + // is where the feature this serves lives. + // + // Every row in this table so far is one state transition: taskID, + // taskStateID, who, when, from where. There has never been anywhere to + // put WHAT happened, which is why FOS reporting a failure had nowhere to + // land -- and on 1.5 that gap is not academic: FOS is shared between the + // two lines, so a FOS carrying FOGProject/fos#152 posts a failure report + // to every server it boots from, 1.5 included. + // + // `logType` defaults to 'state' and the ALTER backfills every existing + // row with it, which is what those rows are. TaskingElement::taskLog() is + // deliberately left alone: the default is the correct value for it, so a + // state row costs no extra column. + // + // `logText` is NULL, not '', so "no body" and "an empty body" stay + // distinguishable -- a state row has no body at all. + // + // A closure rather than a bare ALTER because ADD COLUMN has no + // IF NOT EXISTS below MariaDB 10.0.2/MySQL 8.0.29, so a re-run has to + // converge on its own rather than error. + function () { + $have = self::$DB->query( + "SELECT `COLUMN_NAME` AS `c` FROM `information_schema`.`COLUMNS` " + . "WHERE `TABLE_SCHEMA` = DATABASE() AND `TABLE_NAME` = 'taskLog' " + . "AND `COLUMN_NAME` IN ('logType','logText')" + )->fetch(PDO::FETCH_ASSOC, 'fetch_all')->get(); + $cols = array(); + foreach ((array)$have as $row) { + if (isset($row['c'])) { + $cols[] = $row['c']; + } + } + $adds = array(); + if (!in_array('logType', $cols)) { + $adds[] = "ADD `logType` VARCHAR(16) NOT NULL DEFAULT 'state'"; + } + if (!in_array('logText', $cols)) { + $adds[] = "ADD `logText` TEXT NULL DEFAULT NULL"; + } + if (count($adds) < 1) { + return true; + } + self::$DB->query( + "ALTER TABLE `taskLog` " . implode(', ', $adds) + ); + + return true; + }, +); +// 281 +$this->schema[] = array( + // A task the host reported dead on gets a state of its own. Ported from + // 1.6 schema 339 (GH-1206/#1211), following step 280 which gave taskLog + // somewhere to record the report in the first place. + // + // Until now such a task stayed Queued or In-Progress forever: the report + // was recorded, but the task list still said the machine was working on + // it, and the host could not be re-tasked because it still held an + // active task. Somebody had to notice and cancel it by hand. + // + // Not reusing Cancelled (5), which was the alternative. Cancelled means + // an administrator stopped it; losing the difference between "somebody + // stopped this" and "this broke" costs the operator the one fact they are + // looking at the task list to find. + // + // INSERT IGNORE, so a re-run converges and a server that somehow already + // has a row 6 keeps whatever it has rather than having it rewritten. + "INSERT IGNORE INTO `taskStates` " + . "(`tsID`,`tsName`,`tsDescription`,`tsOrder`,`tsIcon`) " + . "VALUES " + . "(6,'Failed','Host reported that the task could not be completed.'," + . "6,'exclamation-triangle')", +); +// 282 +$this->schema[] = array( + // Retype the rows that landed untyped between step 280 and the model + // learning to type them. Ported from 1.6 schema 340 (#1213). + // + // Step 280 gave `logType` a DEFAULT of 'state', which reads as though a + // writer that sets no type gets one. It does not: a column default + // applies only when the column is absent from the INSERT, and + // FOGController::save() writes every declared field -- so + // TaskingElement::taskLog(), which has recorded state changes since long + // before this column existed, has been writing '' ever since the field + // was declared. TaskLog::__construct() now supplies the type, and this + // repairs what the gap produced. + "UPDATE `taskLog` " + . "SET `logType` = 'state' " + . "WHERE `logType` = '' OR `logType` IS NULL", +); +// 283 +$this->schema[] = array( + // A report keeps enough identity to be read after its task is gone. + // Ported from 1.6 schema 341 (#1236). + // + // taskLog stores no host and no task type of its own, and reaches both + // through `tasks`. Nothing deletes taskLog rows -- but Host::destroy() + // calls TaskManager->destroy() and taskLog is in no cascade at all, so + // deleting a host destroys its tasks and leaves the reports behind with + // nothing to join to, losing the host name at the same moment the host + // row that could supply it goes. + // + // Host name is the first thing anyone searches a failure by, and this + // branch has no Task Management log pane, so the REST API is the only + // reader there is -- it hands back a report whose taskID points at + // nothing and no way at all to learn which machine it came from. The + // point of GH-1206 is that a failure message is findable later instead + // of arriving as a phone photo of a wrapped console, and a foreign key + // to a routinely-deleted row cannot deliver that. + // + // Blocking deletion of a task that has reports was the alternative. It + // inverts the dependency -- a diagnostic artifact would then constrain + // operational cleanup -- and to be consistent it would have to block + // HOST deletion too, since that is the path that actually removes tasks. + // + // The state a row records is NOT copied: taskLog already stores + // taskStateID itself, so that lookup survives the task. + // + // Written only by the FOS report endpoint. Every other row in this table + // is a state transition written by TaskingElement::taskLog() on every + // transition; they are meaningless without their task anyway, and making + // that path do three extra lookups buys nothing. Same reasoning that + // gave logText no value on a state row in step 280. + // + // Two column shapes on purpose, and they follow what the writer can + // actually produce. FOGController::save() omits an unset OPTIONAL column + // whose key ends in "id" -- so logHostID gets its DEFAULT of NULL -- but + // for every other key an unset value is written as '', never NULL (the + // trap step 282 had to repair for logType). Declaring logHostName NOT + // NULL DEFAULT '' says what the ORM will really store rather than + // describing a NULL the writer cannot produce. + // + // A closure rather than a bare ALTER for the same reason step 280 is: + // ADD COLUMN has no IF NOT EXISTS below MariaDB 10.0.2/MySQL 8.0.29, so + // a re-run has to converge on its own rather than error. + function () { + $have = self::$DB->query( + "SELECT `COLUMN_NAME` AS `c` FROM `information_schema`.`COLUMNS` " + . "WHERE `TABLE_SCHEMA` = DATABASE() AND `TABLE_NAME` = 'taskLog' " + . "AND `COLUMN_NAME` IN " + . "('logHostID','logHostName','logTaskTypeName')" + )->fetch(\PDO::FETCH_ASSOC, 'fetch_all')->get(); + $cols = array(); + foreach ((array)$have as $row) { + if (isset($row['c'])) { + $cols[] = $row['c']; + } + } + $adds = array(); + if (!in_array('logHostID', $cols)) { + $adds[] = "ADD `logHostID` INT(11) NULL DEFAULT NULL"; + } + if (!in_array('logHostName', $cols)) { + // varchar(16) matches hosts.hostName, which is capped at the + // NetBIOS limit and cannot outgrow this copy. + $adds[] = "ADD `logHostName` VARCHAR(16) NOT NULL DEFAULT ''"; + } + if (!in_array('logTaskTypeName', $cols)) { + // varchar(30) matches taskTypes.ttName. + $adds[] = "ADD `logTaskTypeName` VARCHAR(30) NOT NULL DEFAULT ''"; + } + if (count($adds) > 0) { + self::$DB->query( + "ALTER TABLE `taskLog` " . implode(', ', $adds) + ); + } + + // Backfill the reports whose task is still there, so the history is + // not split between rows that know their host and rows that do not. + // Restricted to report rows and to rows not already filled, so a + // re-run is a no-op and a later hand-correction is not overwritten. + self::$DB->query( + "UPDATE `taskLog` " + . "JOIN `tasks` ON `tasks`.`taskID` = `taskLog`.`taskID` " + . "LEFT JOIN `hosts` " + . "ON `hosts`.`hostID` = `tasks`.`taskHostID` " + . "LEFT JOIN `taskTypes` " + . "ON `taskTypes`.`ttID` = `tasks`.`taskTypeID` " + . "SET `taskLog`.`logHostID` = `tasks`.`taskHostID`, " + . "`taskLog`.`logHostName` = COALESCE(`hosts`.`hostName`, ''), " + . "`taskLog`.`logTaskTypeName` = COALESCE(`taskTypes`.`ttName`, '') " + . "WHERE `taskLog`.`logType` <> 'state' " + . "AND `taskLog`.`logHostID` IS NULL" + ); + + return true; + }, +); +// 284 +$this->schema[] = array( + // GH-1245: "this never happened" is NULL, not a zero date. + // + // FOGController::save() writes '' for any unset optional field whose key + // does not end in "id". A date column cannot hold '': the server either + // refuses it or coerces it to '0000-00-00 00:00:00', and FOG only ever + // sees the second because PDODB::_connect() has issued + // `SET SESSION sql_mode=''` on every connection since 13661edb (May 2016). + // That clear is removed in the same change as this step, so from here the + // server's own checks apply and '' into a date column is an error. + // + // save() now writes a real NULL for an empty date, which these columns + // have to be able to hold. Without this step it is worse than a no-op: + // an explicit NULL into a NOT NULL column errors under a strict mode and + // is coerced straight back to the zero date without one. + // + // Eleven columns, being every date column that is optional, not + // auto-filled by save()'s switch, and without a server-side default -- + // which is exactly the set that can reach the '' arm and keep the result. + // The list was derived from a replay of this branch's own schema.php into + // an empty server, not from reading the file: nine years of ALTERs mean + // the CREATE TABLE a column first appeared in is not its current type. + // See scripts/background_scripts/replay_15_schema_1245.sh. + // + // Two reachable columns are deliberately left NOT NULL. snapinTasks + // .stCheckinDate and userTracking.utDateTime both declare + // DEFAULT current_timestamp(), so the server supplies a real value rather + // than a zero date; save() omits them and that default applies. + // + // No historical step is edited. `DATETIME NOT NULL` is legal DDL on every + // server, so the steps that created these columns still replay cleanly and + // a fresh install simply arrives here and is corrected. + // + // ALTER before UPDATE: the rows cannot be set NULL until the column can + // hold it. YEAR() rather than the literal '0000-00-00 00:00:00', because a + // strict server rejects that literal in the comparison too. + "ALTER TABLE `hosts` " + . "MODIFY COLUMN `hostLastDeploy` DATETIME NULL DEFAULT NULL", + "UPDATE `hosts` SET `hostLastDeploy` = NULL " + . "WHERE `hostLastDeploy` IS NOT NULL AND YEAR(`hostLastDeploy`) = 0", + "ALTER TABLE `hosts` " + . "MODIFY COLUMN `hostSecTime` TIMESTAMP NULL DEFAULT NULL", + "UPDATE `hosts` SET `hostSecTime` = NULL " + . "WHERE `hostSecTime` IS NOT NULL AND YEAR(`hostSecTime`) = 0", + "ALTER TABLE `images` " + . "MODIFY COLUMN `imageLastDeploy` DATETIME NULL DEFAULT NULL", + "UPDATE `images` SET `imageLastDeploy` = NULL " + . "WHERE `imageLastDeploy` IS NOT NULL AND YEAR(`imageLastDeploy`) = 0", + "ALTER TABLE `imagingLog` " + . "MODIFY COLUMN `ilFinishTime` DATETIME NULL DEFAULT NULL", + "UPDATE `imagingLog` SET `ilFinishTime` = NULL " + . "WHERE `ilFinishTime` IS NOT NULL AND YEAR(`ilFinishTime`) = 0", + "ALTER TABLE `inventory` " + . "MODIFY COLUMN `iDeleteDate` DATETIME NULL DEFAULT NULL", + "UPDATE `inventory` SET `iDeleteDate` = NULL " + . "WHERE `iDeleteDate` IS NOT NULL AND YEAR(`iDeleteDate`) = 0", + "ALTER TABLE `multicastSessions` " + . "MODIFY COLUMN `msStartDateTime` DATETIME NULL DEFAULT NULL", + "UPDATE `multicastSessions` SET `msStartDateTime` = NULL " + . "WHERE `msStartDateTime` IS NOT NULL AND YEAR(`msStartDateTime`) = 0", + "ALTER TABLE `multicastSessions` " + . "MODIFY COLUMN `msCompleteDateTime` DATETIME NULL DEFAULT NULL", + "UPDATE `multicastSessions` SET `msCompleteDateTime` = NULL " + . "WHERE `msCompleteDateTime` IS NOT NULL AND YEAR(`msCompleteDateTime`) = 0", + "ALTER TABLE `snapinTasks` " + . "MODIFY COLUMN `stCompleteDate` DATETIME NULL DEFAULT NULL", + "UPDATE `snapinTasks` SET `stCompleteDate` = NULL " + . "WHERE `stCompleteDate` IS NOT NULL AND YEAR(`stCompleteDate`) = 0", + "ALTER TABLE `tasks` " + . "MODIFY COLUMN `taskCheckIn` DATETIME NULL DEFAULT NULL", + "UPDATE `tasks` SET `taskCheckIn` = NULL " + . "WHERE `taskCheckIn` IS NOT NULL AND YEAR(`taskCheckIn`) = 0", + "ALTER TABLE `tasks` " + . "MODIFY COLUMN `taskScheduledStartTime` DATETIME NULL DEFAULT NULL", + "UPDATE `tasks` SET `taskScheduledStartTime` = NULL " + . "WHERE `taskScheduledStartTime` IS NOT NULL AND YEAR(`taskScheduledStartTime`) = 0", + "ALTER TABLE `userTracking` " + . "MODIFY COLUMN `utDate` DATE NULL DEFAULT NULL", + "UPDATE `userTracking` SET `utDate` = NULL " + . "WHERE `utDate` IS NOT NULL AND YEAR(`utDate`) = 0", +); +// 285 +$this->schema[] = array( + // GH-1245: repair the ENUM error value. + // + // save() wrote '' for every unset optional field whose key does not end + // in "id". Into an ENUM that is not a member, so the server stored the + // special error value at index 0 -- which reads back as '' and is illegal + // to write under any strict sql_mode. FOG never saw the error because + // PDODB::_connect() cleared sql_mode on every connection. + // + // Each column lands on its FIRST member, which is what save() now writes + // for an empty value and what MySQL uses as a NOT NULL enum's implicit + // default. Deliberately not the column's declared DEFAULT: `hostEnforce` + // declares DEFAULT '1', so honouring it here would silently turn + // enforcement ON for every host holding the error value, as a side effect + // of a storage repair. '' and '0' are both falsey in PHP, so every + // consumer sees what it saw before. + // + // Every enum column in the schema, not only the ones a model can leave + // empty today: the error value is illegal wherever it got in, and a + // column that stops being written by one path may still hold it. + "UPDATE `hostMAC` SET `hmPrimary` = '0' WHERE `hmPrimary` = ''", + "UPDATE `hostMAC` SET `hmPending` = '0' WHERE `hmPending` = ''", + "UPDATE `hostMAC` SET `hmIgnoreClient` = '0' WHERE `hmIgnoreClient` = ''", + "UPDATE `hostMAC` SET `hmIgnoreImaging` = '0' WHERE `hmIgnoreImaging` = ''", + "UPDATE `hosts` SET `hostPending` = '0' WHERE `hostPending` = ''", + "UPDATE `hosts` SET `hostEnforce` = '0' WHERE `hostEnforce` = ''", + "UPDATE `imageGroupAssoc` SET `igaPrimary` = '0' WHERE `igaPrimary` = ''", + "UPDATE `images` SET `imageEnabled` = '0' WHERE `imageEnabled` = ''", + "UPDATE `images` SET `imageReplicate` = '0' WHERE `imageReplicate` = ''", + "UPDATE `nfsGroupMembers` SET `ngmGraphEnabled` = '0' WHERE `ngmGraphEnabled` = ''", + "UPDATE `powerManagement` SET `pmAction` = 'shutdown' WHERE `pmAction` = ''", + "UPDATE `powerManagement` SET `pmOndemand` = '0' WHERE `pmOndemand` = ''", + "UPDATE `pxeMenu` SET `pxeHotKeyEnable` = '0' WHERE `pxeHotKeyEnable` = ''", + "UPDATE `snapinGroupAssoc` SET `sgaPrimary` = '0' WHERE `sgaPrimary` = ''", + "UPDATE `snapins` SET `sEnabled` = '0' WHERE `sEnabled` = ''", + "UPDATE `snapins` SET `sReplicate` = '0' WHERE `sReplicate` = ''", + "UPDATE `snapins` SET `sShutdown` = '0' WHERE `sShutdown` = ''", + "UPDATE `snapins` SET `sHideLog` = '0' WHERE `sHideLog` = ''", + "UPDATE `snapins` SET `sPackType` = '0' WHERE `sPackType` = ''", + "UPDATE `tasks` SET `taskWOL` = '0' WHERE `taskWOL` = ''", + "UPDATE `taskTypes` SET `ttType` = 'fog' WHERE `ttType` = ''", + "UPDATE `taskTypes` SET `ttIsAdvanced` = '0' WHERE `ttIsAdvanced` = ''", + "UPDATE `taskTypes` SET `ttIsAccess` = 'both' WHERE `ttIsAccess` = ''", + "UPDATE `users` SET `uAllowAPI` = '0' WHERE `uAllowAPI` = ''", +); + +// 286 +$this->schema[] = array( + // GH-1245, the third instalment: make the schema SAY which columns are + // optional, instead of leaving it to be inferred. + // + // A column declared NOT NULL with no DEFAULT is only mandatory if + // something enforces it. Under a non-strict sql_mode the server does not + // -- it downgrades the error to a warning and substitutes an implicit + // zero value -- so for the nine years PDODB cleared sql_mode, the + // declaration was a comment rather than a constraint. Removing the clear + // turned every one of those columns into a real constraint at once, which + // is how saving FOG settings started failing with error 1364. + // + // For the TEXT columns it was never even a decision: MySQL could not + // attach a DEFAULT to a TEXT or BLOB column until 8.0.13, MariaDB until + // 10.2.1. `longtext NOT NULL` was the only phrasing the schema language + // offered, so those columns are mandatory by accident of syntax. + // + // WHICH COLUMNS. Not a judgement call: FOG already states its intent in + // each model's $databaseFieldsRequired, and this is that statement made + // true in the database. Of the 312 core columns that are NOT NULL, carry + // no DEFAULT and are not AUTO_INCREMENT, 97 stay exactly as they are -- + // the ones the models declare required, plus every column whose name ends + // in ID, because an INSERT that forgets the row it hangs off should fail + // rather than make a silent orphan. The 215 below are the rest. + // + // WHY THIS CANNOT BREAK A WORKING WRITE. An INSERT that names the column + // is unaffected; a default applies only to an omitted column. An INSERT + // that omits it currently FAILS outright on a strict server, so there is + // no working behaviour to change. On a non-strict server it currently + // gets the server's implicit coercion -- and the defaults chosen here are + // exactly that coercion ('' for text, 0 for integers, the first member + // for an enum), which is the same rule save() applies for an empty value. + // So both kinds of server end up where they already were, with the + // difference that the schema now says so. + // + // users.uCreateDate is the one column given a live default rather than a + // zero: a user record created without a date wants now, and writing a + // zero date is the GH-1245 bug in a different costume. Existing rows are + // untouched either way -- a DEFAULT never rewrites stored data. + // + // PLUGIN TABLES ARE DELIBERATELY ABSENT. A plugin's table is not built + // here: it is built by Schema::createTable() when the plugin installs, + // with every column NOT NULL and no defaults at all, and install() calls + // uninstall() first -- which DROPS the table. An ALTER applied here would + // therefore be erased by the next plugin install, so it would be false + // comfort rather than a fix. The runtime paths still cover those tables: + // save() writes an empty value explicitly and insertBatch() backfills the + // columns the caller omitted. + function () { + $optional = array( + 'clientUpdates' => array( + 'cuMD5', 'cuType' + ), + 'globalSettings' => array( + 'settingCategory', 'settingDesc', 'settingValue' + ), + 'greenFog' => array( + 'gfAction', 'gfDays', 'gfHour', 'gfMin' + ), + 'groups' => array( + 'groupBuilding', 'groupCreateBy', 'groupDesc', + 'groupKernel', 'groupKernelArgs', 'groupPrimaryDisk' + ), + 'history' => array( + 'hIP', 'hText', 'hUser' + ), + 'hostMAC' => array( + 'hmDesc', 'hmIgnoreClient', 'hmIgnoreImaging', + 'hmPending', 'hmPrimary' + ), + 'hosts' => array( + 'hostADDomain', 'hostADOU', 'hostADPass', + 'hostADPassLegacy', 'hostADUser', 'hostBuilding', + 'hostCreateBy', 'hostDesc', 'hostDevice', 'hostImage', + 'hostIP', 'hostKernel', 'hostKernelArgs', + 'hostPending', 'hostPrinterLevel', 'hostPubKey', + 'hostSecToken', 'hostSecTokenPrev', 'hostUseAD' + ), + 'hostScreenSettings' => array( + 'hssHeight', 'hssOrientation', 'hssOther1', + 'hssOther2', 'hssRefresh', 'hssWidth' + ), + 'imageGroupAssoc' => array( + 'igaPrimary' + ), + 'images' => array( + 'imageBuilding', 'imageCreateBy', 'imageDesc', + 'imageMagnetUri', 'imageProtect', 'imageSize' + ), + 'imagingLog' => array( + 'ilCreatedBy', 'ilType' + ), + 'inventory' => array( + 'iBiosdate', 'iBiosvendor', 'iBiosversion', + 'iCaseasset', 'iCaseman', 'iCaseserial', 'iCasever', + 'iCpucurrent', 'iCpuman', 'iCpumax', 'iCpuversion', + 'iGpuproducts', 'iGpuvendors', 'iHdfirmware', + 'iHdmodel', 'iHdserial', 'iMbasset', 'iMbman', + 'iMbproductname', 'iMbserial', 'iMbversion', 'iMem', + 'iOtherTag', 'iOtherTag1', 'iPrimaryUser', 'iSysman', + 'iSysproduct', 'iSysserial', 'iSystype', 'iSysversion' + ), + 'ipxeTable' => array( + 'ipxeFailure', 'ipxeFilename', 'ipxeMAC', + 'ipxeManufacturer', 'ipxeProduct', 'ipxeSuccess', + 'ipxeVersion' + ), + 'modules' => array( + 'description' + ), + 'moduleStatusByHost' => array( + 'msState' + ), + 'multicastSessions' => array( + 'msAnon3', 'msAnon4', 'msAnon5', 'msBasePort', + 'msClients', 'msImage', 'msInterface', 'msIsDD', + 'msLogPath', 'msName', 'msPercent', 'msSessClients', + 'msState' + ), + 'nfsGroupMembers' => array( + 'ngmBandwidthLimit', 'ngmIsEnabled', 'ngmIsMasterNode', + 'ngmKey', 'ngmMaxClients', 'ngmMemberDescription', + 'ngmMemberName', 'ngmSnapinPath', 'ngmSSLPath', + 'ngmWebroot' + ), + 'nfsGroups' => array( + 'ngDesc' + ), + 'os' => array( + 'osDescription' + ), + 'plugins' => array( + 'pAnon1', 'pAnon2', 'pAnon3', 'pAnon4', 'pAnon5', + 'pInstalled', 'pState', 'pVersion' + ), + 'powerManagement' => array( + 'pmDom', 'pmDow', 'pmHour', 'pmMin', 'pmMonth', + 'pmOndemand' + ), + 'printerAssoc' => array( + 'paAnon1', 'paAnon2', 'paAnon3', 'paAnon4', 'paAnon5', + 'paIsDefault' + ), + 'printers' => array( + 'pAnon2', 'pAnon3', 'pAnon4', 'pAnon5', 'pConfig', + 'pConfigFile', 'pDefFile', 'pIP', 'pModel', 'pPort' + ), + 'pxeMenu' => array( + 'pxeDesc', 'pxeHotKeyEnable', 'pxeKeySequence', + 'pxeParams' + ), + 'scheduledTasks' => array( + 'stDesc', 'stDOM', 'stDOW', 'stHour', 'stMinute', + 'stMonth', 'stName', 'stOther1', 'stOther2', + 'stOther3', 'stOther4', 'stOther5', 'stShutDown' + ), + 'schemaVersion' => array( + 'vValue' + ), + 'snapinGroupAssoc' => array( + 'sgaPrimary' + ), + 'snapins' => array( + 'sAnon3', 'sArgs', 'sCreator', 'sDesc', + 'snapinProtect', 'sReboot', 'sRunWith', 'sRunWithArgs' + ), + 'snapinTasks' => array( + 'stReturnCode', 'stReturnDetails', 'stState' + ), + 'supportedOS' => array( + 'osName', 'osValue' + ), + 'taskLog' => array( + 'createdBy', 'ip' + ), + 'tasks' => array( + 'taskBPM', 'taskCreateBy', 'taskDataCopied', + 'taskDataTotal', 'taskForce', 'taskIsDebug', + 'taskNFSFailures', 'taskPassreset', 'taskPCT', + 'taskPercentText', 'taskShutdown', 'taskTimeElapsed', + 'taskTimeRemaining', 'taskWOL' + ), + 'taskStates' => array( + 'tsDescription', 'tsIcon' + ), + 'taskTypes' => array( + 'ttDescription', 'ttInitrd', 'ttKernel', 'ttKernelArgs' + ), + 'users' => array( + 'uAPIToken', 'uCreateBy', 'uCreateDate', 'uDisplay', + 'uType' + ), + 'userTracking' => array( + 'utAction', 'utAnon3', 'utDesc' + ), + 'virus' => array( + 'vAnon2', 'vMode' + ), + ); + + // MySQL and MariaDB spell a TEXT/BLOB default differently: MariaDB + // takes the literal, MySQL requires it parenthesised as an + // expression and rejects it outright below 8.0.13. + $version = (string)self::$DB->query('SELECT VERSION() AS `v`') + ->fetch()->get('v'); + $maria = false !== stripos($version, 'mariadb'); + $lobDefaults = $maria; + if (!$maria) { + preg_match('/^(\d+)\.(\d+)\.(\d+)/', $version, $m); + $lobDefaults = count($m) === 4 + && (int)$m[1] * 10000 + (int)$m[2] * 100 + (int)$m[3] + >= 80013; + } + + foreach ($optional as $table => $columns) { + // Only columns that are actually still missing a default, so a + // re-run is a read and nothing else. A table that does not exist + // on this install returns nothing and is skipped rather than + // erroring. + $rows = self::$DB->query( + "SELECT `COLUMN_NAME` AS `c`, `COLUMN_TYPE` AS `ty` " + . "FROM `information_schema`.`COLUMNS` " + . "WHERE `TABLE_SCHEMA` = DATABASE() " + . "AND LOWER(`TABLE_NAME`) = :table " + . "AND `IS_NULLABLE` = 'NO' " + . "AND `COLUMN_DEFAULT` IS NULL " + . "AND `EXTRA` NOT LIKE '%auto_increment%'", + array(), + array(':table' => strtolower($table)) + )->fetch(\PDO::FETCH_ASSOC, 'fetch_all')->get(); + + $want = array_map('strtolower', $columns); + foreach ((array)$rows as $row) { + if (!isset($row['c'], $row['ty']) + || !in_array(strtolower($row['c']), $want, true) + ) { + continue; + } + $type = trim($row['ty']); + $lob = (bool)preg_match( + '/^(tiny|medium|long)?(text|blob)\b/i', + $type + ); + if ($lob && !$lobDefaults) { + // Nothing sensible to do on MySQL below 8.0.13, and + // nothing broken by skipping: insertBatch() backfills + // the column and save() writes it explicitly. + continue; + } + if (preg_match('/^datetime\b/i', $type)) { + $default = 'current_timestamp()'; + } elseif (preg_match( + '/^(tiny|small|medium|big)?int\b/i', + $type + )) { + $default = '0'; + } elseif (preg_match( + "/^(enum|set)\\s*\\(\\s*'((?:[^']|'')*)'/i", + $type, + $member + )) { + $default = "'" . $member[2] . "'"; + } elseif ($lob) { + $default = $maria ? "''" : "('')"; + } else { + $default = "''"; + } + self::$DB->query( + sprintf( + 'ALTER TABLE `%s` MODIFY COLUMN `%s` %s NOT NULL ' + . 'DEFAULT %s', + $table, + $row['c'], + $type, + $default + ) + ); + } + } + + return true; + }, +); + +// 287 +$this->schema[] = array( + // Widen the stored pxeMenu param blocks past three NICs. + // + // The mac0/mac1/mac2 enumeration is not only in code -- six of these + // blocks ship as `pxeMenu`.`pxeParams` DATA, and _menuOpt() emits whatever + // the row says verbatim. So fixing bootmenu.class.php and the installer's + // default.ipxe leaves every existing site's menu items still posting at + // most three MACs, which is what made a host registered under only its + // fourth NIC unfindable. + // + // Two additions per row, matching bootmenu.class.php: + // - macboot, ${netX/mac}, the NIC iPXE actually booted from. An + // ADDITION to mac0, not a replacement: netX is a pointer at one of + // net0..netN, so substituting it would drop net0 on a machine that + // booted off net1. boot.php unions every mac* field and array_unique()s + // the result, so the overlap costs nothing. It goes ABOVE the chain + // because the chain short-circuits to :bootme on the first absent + // interface, which on a single-NIC machine is net1. + // - net3..net7, so the enumeration reaches eight interfaces. + // + // Guarded on the row still matching what we shipped, byte for byte. These + // rows are user-writable from iPXE Menu Customization, and a site that has + // edited one has made a deliberate choice; an untouched row provably has + // not. A customized row keeps its three NICs rather than losing the edit, + // and re-running is a no-op because the old value no longer matches. + // + // A closure rather than seven literal statements: the old and the new + // value differ by one line in the middle of a nine-line blob, and writing + // both out per menu entry is fourteen near-identical paragraphs in which a + // single wrong character silently means "match nothing, change nothing". + function () { + // pxeName => the boolean flag that row's params block carries. + $menus = array( + 'fog.deployimage' => 'qihost', + 'fog.quickdel' => 'delhost', + 'fog.keyreg' => 'keyreg', + 'fog.debug' => 'debugAccess', + 'fog.multijoin' => 'sessionJoin', + 'fog.advancedlogin' => 'advLog', + 'fog.approvehost' => 'approveHost' + ); + $head = "login\n" + . "params\n" + . 'param mac0 ${net0/mac}' . "\n" + . 'param arch ${arch}' . "\n" + . 'param username ${username}' . "\n" + . 'param password ${password}' . "\n"; + $oldTail = 'isset ${net1/mac} && param mac1 ${net1/mac} || goto bootme' + . "\n" + . 'isset ${net2/mac} && param mac2 ${net2/mac} || goto bootme'; + $newTail = 'isset ${netX/mac} && param macboot ${netX/mac} ||'; + for ($nic = 1; $nic <= 7; $nic++) { + $newTail .= "\n" . sprintf( + 'isset ${net%1$d/mac} && param mac%1$d ${net%1$d/mac}' + . ' || goto bootme', + $nic + ); + } + foreach ($menus as $pxeName => $flag) { + $body = $head . sprintf('param %s 1', $flag) . "\n"; + self::$DB->query( + 'UPDATE `pxeMenu` SET `pxeParams` = :new ' + . 'WHERE `pxeName` = :name AND `pxeParams` = :old', + array(), + array( + ':new' => $body . $newTail, + ':name' => $pxeName, + ':old' => $body . $oldTail + ) + ); + } + + return true; + } +); +// 288 +$this->schema[] = array( + // Memtest86+ 8.10 replaces the 2013 Memtest86+ 5.01 ISO that memdisk + // loaded. The new file boots on both legacy BIOS and UEFI, which the + // memdisk chain never could (#321). Only a value still at the old + // default is moved: a site that pointed this at its own file keeps it. + "UPDATE `globalSettings` SET `settingValue`='mt86plus_x86_64' " + . "WHERE `settingKey`='FOG_MEMTEST_KERNEL' " + . "AND `settingValue`='memtest.bin'", +); diff --git a/packages/web/commons/text.php b/packages/web/commons/text.php index 37b6019468..971330ec7e 100644 --- a/packages/web/commons/text.php +++ b/packages/web/commons/text.php @@ -10,7 +10,7 @@ * then be translated just the one time for all the languages. * Then the element Host or Printer could be translated later. * - * PHP version 5 + * PHP version 7.4+ * * @category Redirect * @package FOGProject diff --git a/packages/web/index.php b/packages/web/index.php index 56d0e04e30..bdfb778d52 100644 --- a/packages/web/index.php +++ b/packages/web/index.php @@ -2,7 +2,7 @@ /** * Redirects calls to index.php to main page. * - * PHP version 5 + * PHP version 7.4+ * * @category Redirect * @package FOGProject diff --git a/packages/web/lib/client/alobg.class.php b/packages/web/lib/client/alobg.class.php index 44f59c27e0..535ed3e2f9 100644 --- a/packages/web/lib/client/alobg.class.php +++ b/packages/web/lib/client/alobg.class.php @@ -3,7 +3,7 @@ * Sends the auto logout background image * NOTE: Only used on legacy client * - * PHP version 5 + * PHP version 7.4+ * * @category ALOGB * @package FOGProject diff --git a/packages/web/lib/client/autologout.class.php b/packages/web/lib/client/autologout.class.php index cca6f0b990..b4dac09b6d 100644 --- a/packages/web/lib/client/autologout.class.php +++ b/packages/web/lib/client/autologout.class.php @@ -2,7 +2,7 @@ /** * Handles auto log information as requested. * - * PHP version 5 + * PHP version 7.4+ * * @category AutoLogout * @package FOGProject diff --git a/packages/web/lib/client/directorycleanup.class.php b/packages/web/lib/client/directorycleanup.class.php index 8a62d3eda4..b28259466f 100644 --- a/packages/web/lib/client/directorycleanup.class.php +++ b/packages/web/lib/client/directorycleanup.class.php @@ -2,7 +2,7 @@ /** * Cleans directories but only for legacy client * - * PHP version 5 + * PHP version 7.4+ * * @category DirectoryCleanup * @package FOGProject diff --git a/packages/web/lib/client/displaymanager.class.php b/packages/web/lib/client/displaymanager.class.php index 0970d61cba..ef9e8271f8 100644 --- a/packages/web/lib/client/displaymanager.class.php +++ b/packages/web/lib/client/displaymanager.class.php @@ -2,7 +2,7 @@ /** * Handles display manager * - * PHP version 5 + * PHP version 7.4+ * * @category DisplayManager * @package FOGProject diff --git a/packages/web/lib/client/fogclient.class.php b/packages/web/lib/client/fogclient.class.php index 189a56cc8c..0bdb634b37 100644 --- a/packages/web/lib/client/fogclient.class.php +++ b/packages/web/lib/client/fogclient.class.php @@ -2,7 +2,7 @@ /** * Base element for client services * - * PHP version 5 + * PHP version 7.4+ * * @category FOGClient * @package FOGProject diff --git a/packages/web/lib/client/fogclientsend.class.php b/packages/web/lib/client/fogclientsend.class.php index f89d6b9385..cfc5d1e97b 100644 --- a/packages/web/lib/client/fogclientsend.class.php +++ b/packages/web/lib/client/fogclientsend.class.php @@ -2,7 +2,7 @@ /** * A basic interface to define how client classes should operate * - * PHP version 5 + * PHP version 7.4+ * * @category FOGClientSend * @package FOGProject diff --git a/packages/web/lib/client/gf.class.php b/packages/web/lib/client/gf.class.php index e2394da597..3343762785 100644 --- a/packages/web/lib/client/gf.class.php +++ b/packages/web/lib/client/gf.class.php @@ -2,7 +2,7 @@ /** * Handles GreenFog, now only for legacy client * - * PHP version 5 + * PHP version 7.4+ * * @category Greenfog * @package FOGProject diff --git a/packages/web/lib/client/hostnamechanger.class.php b/packages/web/lib/client/hostnamechanger.class.php index 3fc88d3f24..3a716ba917 100644 --- a/packages/web/lib/client/hostnamechanger.class.php +++ b/packages/web/lib/client/hostnamechanger.class.php @@ -3,7 +3,7 @@ * Sends the client with the hostname and domain * information needed to perform the client actions. * - * PHP version 5 + * PHP version 7.4+ * * @category HostnameChanger * @package FOGProject diff --git a/packages/web/lib/client/index.php b/packages/web/lib/client/index.php index d499d42de2..c410a98308 100644 --- a/packages/web/lib/client/index.php +++ b/packages/web/lib/client/index.php @@ -2,7 +2,7 @@ /** * Redirects calls to lib/client/index.php to main page. * - * PHP version 5 + * PHP version 7.4+ * * @category Redirect * @package FOGProject diff --git a/packages/web/lib/client/jobs.class.php b/packages/web/lib/client/jobs.class.php index 480289b60e..0251b673bc 100644 --- a/packages/web/lib/client/jobs.class.php +++ b/packages/web/lib/client/jobs.class.php @@ -2,7 +2,7 @@ /** * Tells the client if there's a task waiting for the host * - * PHP version 5 + * PHP version 7.4+ * * @category Jobs * @package FOGProject diff --git a/packages/web/lib/client/pm.class.php b/packages/web/lib/client/pm.class.php index 5d7553fd61..e44142b4c7 100644 --- a/packages/web/lib/client/pm.class.php +++ b/packages/web/lib/client/pm.class.php @@ -2,7 +2,7 @@ /** * Powermanagement Client information * - * PHP version 5 + * PHP version 7.4+ * * @category Powermanagement * @package FOGProject diff --git a/packages/web/lib/client/printerclient.class.php b/packages/web/lib/client/printerclient.class.php index 3c4dc2c50d..cb6beafd87 100644 --- a/packages/web/lib/client/printerclient.class.php +++ b/packages/web/lib/client/printerclient.class.php @@ -2,7 +2,7 @@ /** * Sends the printer information for the FOG Client * - * PHP version 5 + * PHP version 7.4+ * * @category PrinterClient * @package FOGProject diff --git a/packages/web/lib/client/registerclient.class.php b/packages/web/lib/client/registerclient.class.php index 66ba46d0e1..d30e2dd524 100644 --- a/packages/web/lib/client/registerclient.class.php +++ b/packages/web/lib/client/registerclient.class.php @@ -4,7 +4,7 @@ * If using the new client can also register new hosts * into a pending status. * - * PHP version 5 + * PHP version 7.4+ * * @category RegisterClient * @package FOGProject diff --git a/packages/web/lib/client/servicemodule.class.php b/packages/web/lib/client/servicemodule.class.php index d19c6f1b3c..7b060f5fe0 100644 --- a/packages/web/lib/client/servicemodule.class.php +++ b/packages/web/lib/client/servicemodule.class.php @@ -2,7 +2,7 @@ /** * The service module checks * - * PHP version 5 + * PHP version 7.4+ * * @category ServiceModule * @package FOGProject diff --git a/packages/web/lib/client/snapinclient.class.php b/packages/web/lib/client/snapinclient.class.php index c1f75fe4cf..500e7c8aa5 100644 --- a/packages/web/lib/client/snapinclient.class.php +++ b/packages/web/lib/client/snapinclient.class.php @@ -2,7 +2,7 @@ /** * Handles snapins for the host * - * PHP version 5 + * PHP version 7.4+ * * @category SnapinClient * @package FOGProject diff --git a/packages/web/lib/client/updateclient.class.php b/packages/web/lib/client/updateclient.class.php index a792fc1214..617eed8eed 100644 --- a/packages/web/lib/client/updateclient.class.php +++ b/packages/web/lib/client/updateclient.class.php @@ -3,7 +3,7 @@ * Updates client files * NOTE: Only for legacy client relations * - * PHP version 5 + * PHP version 7.4+ * * @category UpdateClient * @package FOGProject diff --git a/packages/web/lib/client/usercleaner.class.php b/packages/web/lib/client/usercleaner.class.php index d00ce81957..1e4bab749e 100644 --- a/packages/web/lib/client/usercleaner.class.php +++ b/packages/web/lib/client/usercleaner.class.php @@ -2,7 +2,7 @@ /** * Legacy client use only just returns the users to cleanup * - * PHP version 5 + * PHP version 7.4+ * * @category UserCleaner * @package FOGProject diff --git a/packages/web/lib/client/usertrack.class.php b/packages/web/lib/client/usertrack.class.php index 387fd80fe1..3c6cb523ff 100644 --- a/packages/web/lib/client/usertrack.class.php +++ b/packages/web/lib/client/usertrack.class.php @@ -2,7 +2,7 @@ /** * Logs the user who logged in * - * PHP version 5 + * PHP version 7.4+ * * @category UserTrack * @package FOGProject @@ -55,7 +55,11 @@ public function json() $user = strtolower( $_REQUEST['user'] ); - if (isset($_REQUEST['date'])) { + // GH-1245: an empty date parameter means the client did not send + // one, so fall back to now. niceDate() used to do that itself for '' + // and no longer does -- it now reads empty as "no value", which here + // would stamp every login with the year zero. + if (isset($_REQUEST['date']) && '' !== trim((string) $_REQUEST['date'])) { $tmpDate = self::niceDate($_REQUEST['date']); } else { $tmpDate = self::niceDate(); @@ -109,8 +113,11 @@ public function send() base64_decode($_REQUEST['user']) ); unset($tmpDate); - if (isset($_REQUEST['date'])) { - $date = base64_decode($_REQUEST['date']); + // GH-1245: as above, and base64_decode('') is '' too. + $date = isset($_REQUEST['date']) + ? base64_decode($_REQUEST['date']) + : ''; + if ('' !== trim((string) $date)) { $tmpDate = self::niceDate($date); } else { $tmpDate = self::niceDate(); diff --git a/packages/web/lib/db/databasemanager.class.php b/packages/web/lib/db/databasemanager.class.php index 536f46d576..cec3f04879 100644 --- a/packages/web/lib/db/databasemanager.class.php +++ b/packages/web/lib/db/databasemanager.class.php @@ -2,7 +2,7 @@ /** * Database Manager Handles communication from fog to db class. * - * PHP version 5 + * PHP version 7.4+ * * This is what communicates with fog to the db class. * diff --git a/packages/web/lib/db/index.php b/packages/web/lib/db/index.php index e4e7245de4..b100178dac 100644 --- a/packages/web/lib/db/index.php +++ b/packages/web/lib/db/index.php @@ -2,7 +2,7 @@ /** * Redirects calls to lib/db/index.php to main page. * - * PHP version 5 + * PHP version 7.4+ * * @category Redirect * @package FOGProject diff --git a/packages/web/lib/db/mysqldump.class.php b/packages/web/lib/db/mysqldump.class.php index b318dc7e2c..016e0d1b16 100644 --- a/packages/web/lib/db/mysqldump.class.php +++ b/packages/web/lib/db/mysqldump.class.php @@ -2,7 +2,7 @@ /** * Mysqldump File Doc Comment * - * PHP version 5 + * PHP version 7.4+ * * @category Library * @package Ifsnop\Mysqldump diff --git a/packages/web/lib/db/pdodb.class.php b/packages/web/lib/db/pdodb.class.php index 0a5d91925a..e6e5137b44 100644 --- a/packages/web/lib/db/pdodb.class.php +++ b/packages/web/lib/db/pdodb.class.php @@ -2,7 +2,7 @@ /** * PDODB, the database connector. * - * PHP version 5 + * PHP version 7.4+ * * This is what communicates between FOG and the Database. * @@ -291,7 +291,29 @@ private function _connect($dbexists = true) self::redirect('../management/index.php?node=schema'); } } - self::query("SET SESSION sql_mode=''"); + /* + * GH-1245: no `SET SESSION sql_mode=''` here. + * + * That line arrived in 13661edb (May 2016) as "try to set sql_mode + * to non-strict which should allow 5.7 mysql to operate", and it + * shipped with a TARGETED mode commented out one line above it -- + * one that kept STRICT_TRANS_TABLES. So even then the intent was + * not to disable validation; the blanket clear was the fallback. + * + * It stayed for nine years and meant every statement FOG issued + * ran with the server's checks off: truncations, out-of-range + * numerics and invalid enum members were all silently coerced and + * reported only as warnings nothing reads. That is how a zero + * `hostLastDeploy` came to sit on servers whose own configuration + * forbids one, and how the ENUM error value got into 24 columns. + * + * What actually needed fixing was FOGController::save(), which + * wrote '' for every unset optional field regardless of the + * column's type. emptyValueFor() now writes the value the server + * was coercing to anyway, so nothing here depends on the checks + * being off. Schema steps 284 and 285 repair the rows that were + * written while they were. + */ } catch (PDOException $e) { if ($dbexists) { self::$_link = false; @@ -534,6 +556,28 @@ public function fetch( $this->sqlerror() ); self::$_result = false; + /* + * $msg used to be built here and dropped on the floor: a failed + * fetch set no ->error, logged nothing, and left $_result false + * -- so get() answered an empty set and the caller could not tell + * "the read failed" from "there are no rows". That is the same + * defect FOGController::save() and load() were carrying, one + * layer down, and it is why those checks alone were not enough. + * + * Only ever ADDS a failure, never overwrites one. query() owns + * clearing ->error -- it runs immediately before every fetch() + * and always sets it to false or to a message -- so when a fetch + * fails BECAUSE the query did ("No query result, use query() + * first"), the guard keeps the original cause rather than + * replacing it with the symptom. + * + * Not logged from here. The callers know which class and table + * they were reading, and this does not; a line naming neither is + * worse than the caller's, and two lines per failure is noise. + */ + if (!$this->error) { + $this->error = $msg; + } if (self::$throwOnQueryError) { throw $e; @@ -902,6 +946,38 @@ private static function _bind($param, $value, $type = null) if (is_null($type)) { $type = PDO::PARAM_STR; } + /* + * A PHP boolean bound as a string is the string cast of it, and + * (string)false is ''. Every caller reaches this method with the + * default PDO::PARAM_STR, so `->set('shutdown', $action == + * 'shutdown')` -- an ordinary comparison, and how Snapin::save() has + * always spelled it -- stored '' into `snapins`.`sShutdown`, an + * enum('0','1'). That is error 1265, "Data truncated for column + * 'sShutdown' at row 1", on any server with STRICT_TRANS_TABLES. + * + * It is the same defect as GH-1245 arriving by a different door. + * save()'s emptyValueFor() only recognises null and '' as empty, so + * a boolean walks straight past it, and the manager UPDATE path + * (HostManager::update() writing `hosts`.`hostInfoLock` from + * ->set('tokenlock', false) at the end of every imaging task) never + * went through save() at all. Normalising here is the only place + * that covers save(), insertBatch(), the manager builders and + * hand-written queries at once. + * + * '0'/'1' rather than PDO::PARAM_BOOL: bound as an integer, 0 + * against an ENUM is an *index*, and index 0 is the error value. + * As strings they are literal enum members, and a numeric column + * coerces them to 0/1. Readers are unaffected either way; '0' is + * falsey in PHP exactly as '' was. + * + * A caller that passes an explicit type is left alone -- it has + * said what it means. + * + * See forum topic 18227. + */ + if (is_bool($value) && $type === PDO::PARAM_STR) { + $value = $value ? '1' : '0'; + } self::$_queryResult->bindParam($param, $value, $type); } } diff --git a/packages/web/lib/events/hostlist.event.php b/packages/web/lib/events/hostlist.event.php index c08a8f477f..a63879ead9 100644 --- a/packages/web/lib/events/hostlist.event.php +++ b/packages/web/lib/events/hostlist.event.php @@ -2,7 +2,7 @@ /** * Host list event * - * PHP version 5 + * PHP version 7.4+ * * @category HostList_Event * @package FOGProject diff --git a/packages/web/lib/events/index.php b/packages/web/lib/events/index.php index 4df96ff455..b87cd9cb57 100644 --- a/packages/web/lib/events/index.php +++ b/packages/web/lib/events/index.php @@ -2,7 +2,7 @@ /** * Redirects calls to lib/events/index.php to main page. * - * PHP version 5 + * PHP version 7.4+ * * @category Redirect * @package FOGProject diff --git a/packages/web/lib/fog/bootmenu.class.php b/packages/web/lib/fog/bootmenu.class.php index 2871d17f11..419ebf12fd 100644 --- a/packages/web/lib/fog/bootmenu.class.php +++ b/packages/web/lib/fog/bootmenu.class.php @@ -2,7 +2,7 @@ /** * Boot menu for the fog pxe system * - * PHP Version 5 + * PHP version 7.4+ * * @category Bootmenu * @package FOGProject @@ -117,6 +117,196 @@ class BootMenu extends FOGBase * @var array */ private static $_exitTypes = array(); + /** + * Lines to show the operator about how this boot was resolved + * + * @var array + */ + private $_notices = array(); + /** + * Is the booting machine an ARM one? + * + * iPXE tells us: default.ipxe posts "param arch ${arch}", derived from + * ${buildarch}, and every chain this class emits carries it forward. + * The value is the architecture of the iPXE binary DHCP handed the + * machine, not a guess. + * + * One place rather than the four open-coded stripos() tests, so the + * kernel selection, the loader selection and the two new guards below + * cannot answer it differently. + * + * @return bool + */ + private static function _archIsArm() + { + return false !== stripos( + isset($_REQUEST['arch']) ? $_REQUEST['arch'] : '', + 'arm' + ); + } + /** + * Makes a value safe to interpolate into an iPXE `echo` line. + * + * boot.php is unauthenticated by necessity -- a booting NIC has no + * credential to present -- and iPXE scripts are newline-delimited + * commands. Initiator::sanitizeOutput() collapses a RUN of whitespace + * to its first character rather than removing newlines, so a lone + * "\n" in a request or stored value survives into the emitted script + * as a command separator and anything past it would be executed. + * + * Whitelist rather than escape: these values are kernel and init + * filenames, so the safe set is small and known, and iPXE's tokenizer + * strips quotes outright so there is no escaping mechanism to lean on. + * + * @param string $value the value to render + * + * @return string + */ + private static function _echoSafe($value) + { + return substr( + preg_replace('/[^A-Za-z0-9._-]/', '', (string)$value), + 0, + 64 + ); + } + /** + * Applies a host's kernel/init override to the arch-selected default. + * + * A host (or the group that wrote to it) can name its own kernel and + * init, and that override deliberately wins over the arch default -- + * but only where it CAN win. The override is a bare filename with no + * architecture in it, and `hosts` stores no architecture at all, so + * nothing at edit time can warn an admin that the kernel they picked + * is wrong for some of the machines it will reach. Setting a kernel on + * a mixed group therefore handed an x86 bzImage to every ARM member, + * silently discarding the arch selection made moments earlier. + * + * Only the arm/non-arm split is policed. i386 code runs on x86_64, so a + * deliberate 32-bit override is a legitimate choice and is left alone; + * aarch64 and x86 are not the same instruction set in either direction, + * so an override across that line can only ever fail to boot. + * + * The test is the `arm` filename prefix -- the convention every kernel + * and init FOG ships follows (arm_Image, arm_init.cpio.gz). + * + * @param string $field 'kernel' or 'init' + * @param string $default what the architecture selected + * + * @return string the filename to boot + */ + private function _hostOverride($field, $default) + { + $override = trim((string)self::$Host->get($field)); + if ('' === $override) { + return $default; + } + $isArmFile = 0 === stripos(basename($override), 'arm'); + if ($isArmFile === self::_archIsArm()) { + return $override; + } + // Say so on screen rather than just ignoring it: from the + // operator's side an ignored override and an honoured one look + // identical, and the machine that is misconfigured is the one + // that needs telling. + $this->_notices[] = sprintf( + 'echo Ignoring host %s %s -- this machine is %s. Using %s.', + $field, + self::_echoSafe($override), + self::_archIsArm() ? '64-bit ARM' : 'x86', + self::_echoSafe($default) + ); + + return $default; + } + /** + * The memtest boot lines, or a refusal on an architecture that has no + * Memtest86+ build. + * + * Memtest86+ 6.0 and later is a single file that boots two ways: a + * legacy BIOS loads it through the Linux boot protocol, so iPXE boots + * it with `kernel`; UEFI firmware loads it as a PE, so iPXE boots it + * with `chain`. Which one this client needs is what ${platform} says. + * The memdisk chain this replaced (memdisk + a Memtest86+ 5.01 ISO) + * was a 16-bit loader that UEFI clients refused with "Exec format + * error" (#321). + * + * `chain` does not return on success, so on UEFI the lines after it + * are only reached on failure. The BIOS branch is jumped over rather + * than left to `||` fall-through, because `iseq ... && chain ... || + * kernel ...` would run the BIOS loader on a UEFI client whose chain + * had just failed. + * + * Upstream publishes no aarch64 build, so on ARM the menu entry and + * the scheduled task could only ever fail -- the entry dropping the + * machine back to the menu with an iPXE error, the task leaving it at + * a bare prompt with nothing said about why. + * + * @param string $onFail what to append when the boot fails or is refused + * + * @return array + */ + private function _memtestChoice($onFail = ' || goto MENU') + { + if (self::_archIsArm()) { + return array( + 'echo Memtest86+ publishes no build for 64-bit ARM, so it ' + . 'cannot run here.', + 'sleep 5' . $onFail, + ); + } + + return array( + 'iseq ${platform} efi && goto fog.memtest.efi ||', + "kernel $this->_memtest", + 'boot' . $onFail, + ':fog.memtest.efi', + "chain $this->_memtest" . $onFail, + ); + } + /** + * Whether a kernel-argument string asks for a shutdown when the task + * ends. + * + * stripos()'s arguments were the wrong way round at every call site + * this replaces: stripos('shutdown=1', $args) searches for the + * ARGUMENTS inside the literal, not the other way about. Two silent + * consequences, in opposite directions. + * + * A real 'shutdown=1 mode=debug' was never detected -- the + * ten-character literal cannot contain it -- so a custom iPXE menu + * entry that asks for a shutdown never produced one. Only an extraargs + * of EXACTLY 'shutdown=1' worked, by accident. + * + * And an empty string matches at offset 0, so the + * `false !== stripos(...)` spelling reported a shutdown for any task + * type with no kernel arguments at all. + * + * @param string $args the argument string to test + * + * @return bool + */ + private static function _wantsShutdown($args) + { + $args = trim((string)$args); + + return '' !== $args && false !== stripos($args, 'shutdown=1'); + } + /** + * The extraargs the chain arrived with, '' when there were none. + * + * Not every chain back into a flow carries them, and passing an unset + * key to stripos() emits a PHP warning straight into the iPXE script + * this class is building. + * + * @return string + */ + private static function _extraArgs() + { + return isset($_REQUEST['extraargs']) + ? (string)$_REQUEST['extraargs'] + : ''; + } /** * Initializes the boot menu class * @@ -133,10 +323,7 @@ public function __construct() . 'chain -ar ${boot-url}/service/ipxe/refind_x64.efi', "\n" ); - $reboot = sprintf( - 'reboot', - "\n" - ); + $reboot = 'reboot'; if (isset($_REQUEST['arch']) && stripos($_REQUEST['arch'], 'i386') !== false) { //user i386 boot loaders instead @@ -213,7 +400,26 @@ public function __construct() */ $bootroot = trim((string)$curroot, '/'); $curroot = '/' . ($bootroot === '' ? '' : $bootroot . '/'); + /** + * BOOT_ITEM_NEW_SETTINGS passes 'webroot' by reference, but no + * $webroot was ever assigned, so PHP created it at the call and + * every plugin reading it saw NULL. Bind it to the bare form that + * accompanies 'webserver' in the same payload -- the value + * 'set fog-webroot' emits -- so the argument means what its name + * says. + */ + $webroot = $bootroot; $this->_web = sprintf('%s://%s%s', self::$httpproto, $webserver, $curroot); + /** + * setmacto is the MAC FOS forces onto whichever interface it manages + * to reach us on, so it has to be the MAC iPXE actually booted with. + * ${net0/mac} was wrong on any machine whose first enumerated NIC has + * no link: iPXE gets its lease over the NIC that does, FOS then + * rewrites that NIC to the unplugged one's MAC and the re-DHCP fails. + * ${netX} is iPXE's alias for the last opened network device, so it + * follows the interface that got us here and still resolves to net0 + * on a single-NIC machine. + */ $Send['booturl'] = array( '#!ipxe', "set fog-ip $webserver", @@ -221,7 +427,7 @@ public function __construct() 'set boot-url ' . self::$httpproto . '://${fog-ip}/${fog-webroot}', - 'set setmacto ${net0/mac}', + 'set setmacto ${netX/mac}', ); if (self::$Host->isValid()) { $sysuuid = filter_input(INPUT_POST, 'sysuuid') @@ -331,24 +537,38 @@ public function __construct() $keySequence : '' ); - if (($_REQUEST['arch'] ?? '') == 'i386') { + $rawArch = (string)($_REQUEST['arch'] ?? ''); + $archId = 'x86_64'; + if ('i386' === $rawArch) { + $archId = 'i386'; $bzImage = $bzImage32; + // The 32-bit Memtest86+ build. Set before HOST_EDIT_SETTINGS so + // a plugin that repoints $memtest at its own node (Location + // does) repoints the right file. Not a setting: nothing else + // about the i386 profile is configurable either (#321). + $memtest = 'mt86plus_i586'; $imagefile = $init_32; - } elseif (false !== stripos(($_REQUEST['arch'] ?? ''), 'arm')) { + } elseif (false !== stripos($rawArch, 'arm')) { + $archId = 'arm64'; $bzImage = $bzImageArm; $imagefile = $init_arm; } - $kernel = $bzImage; - if (self::$Host->get('kernel')) { - $bzImage = trim( - self::$Host->get('kernel') - ); - } - if (self::$Host->get('init')) { - $imagefile = trim( - self::$Host->get('init') + // Anything that is not one of the three FOG builds a kernel for. + // arm32 lands here too: it matches on 'arm' and so is handed the + // aarch64 files, which is the same thing that used to happen + // silently -- the difference is that the operator is now told, on + // screen, why the boot is about to fail. + if ('' !== $rawArch + && !in_array($rawArch, array('x86_64', 'i386', 'arm64'), true) + ) { + $this->_notices[] = sprintf( + 'echo FOG ships no boot kernel for %s -- trying the %s one.', + self::_echoSafe($rawArch), + $archId ); } + $bzImage = $this->_hostOverride('kernel', $bzImage); + $imagefile = $this->_hostOverride('init', $imagefile); $StorageGroup = $StorageNode->getStorageGroup(); $exit = trim( ( @@ -360,6 +580,7 @@ public function __construct() $exit = 'sanboot'; } $initrd = $imagefile; + $hookInitrd = $initrd; if (self::$Host->isValid()) { self::$HookManager->processEvent( 'BOOT_ITEM_NEW_SETTINGS', @@ -382,8 +603,22 @@ public function __construct() ) ); } - $kernel = $bzImage; - $initrd = $imagefile; + /** + * 'initrd' and 'imagefile' are both passed to the hook by + * reference, and this used to reassign $initrd = $imagefile + * unconditionally -- so a plugin that set 'initrd' had its value + * discarded on the very next line, while one that set 'imagefile' + * was honoured. Nothing said which of the two to write to, and + * the one named after the thing being chosen was the dead one. + * + * Follow 'imagefile' only when the hook left 'initrd' alone, so + * the working argument keeps working and the documented one + * starts to. With no plugin listening both are equal here and + * this is a no-op, which is what the golden file pins. + */ + if ($initrd === $hookInitrd) { + $initrd = $imagefile; + } $this->_timeout = $timeout; $this->_hiddenmenu = ($hiddenmenu && !(isset($_REQUEST['menuAccess']) && $_REQUEST['menuAccess'])); $this->_bootexittype = self::$_exitTypes[$exit]; @@ -392,7 +627,11 @@ public function __construct() $this->_booturl = self::$httpproto . "://{$webserver}/fog/service"; $this->_memdisk = "kernel $memdisk initrd=$memtest"; - $this->_memtest = "initrd $memtest"; + // The bare file, not "initrd $memtest": _memtestChoice() boots it + // with kernel or chain, and memdisk is no longer part of that path. + // _memdisk is still built and still handed to IPXE_EDIT because a + // custom entry may chain a floppy or ISO image through it. + $this->_memtest = $memtest; $StorageNodes = (array)self::getClass('StorageNodeManager') ->find( array( @@ -479,7 +718,7 @@ public function __construct() ), $this->_storage ); - $this->_initrd = "imgfetch $imagefile"; + $this->_initrd = "imgfetch $initrd"; self::$HookManager ->processEvent('BOOT_MENU_ITEM'); $PXEMenuID = self::maxId( @@ -511,7 +750,9 @@ public function __construct() 'ALTERNATE_BOOT_CHECKS' ); if (isset($_REQUEST['username']) && isset($_REQUEST['password'])) { - $tmpUser = self::attemptLogin( + // authenticateOnly: iPXE holds no cookie, so a session + // established here could never be presented back. + $tmpUser = self::authenticateOnly( $_REQUEST['username'], $_REQUEST['password'] ); @@ -591,20 +832,29 @@ private function _ipxeLog() */ private function _chainBoot($debug = false, $shortCircuit = false) { - $debug = $debug; if (!(isset($this->_hiddenmenu) && $this->_hiddenmenu) || $shortCircuit) { $Send['chainnohide'] = array( 'set arch ${buildarch}', 'iseq ${arch} i386 && cpuid --ext 29 && set arch x86_64 ||', 'params', 'param mac0 ${net0/mac}', + 'param product ${product}', + 'param manufacturer ${manufacturer}', + 'param ipxever ${version}', + 'param filename ${filename}', 'param arch ${arch}', 'param platform ${platform}', 'param menuAccess 1', "param debug $debug", 'param sysuuid ${uuid}', + 'isset ${netX/mac} && param macboot ${netX/mac} ||', 'isset ${net1/mac} && param mac1 ${net1/mac} || goto bootme', 'isset ${net2/mac} && param mac2 ${net2/mac} || goto bootme', + 'isset ${net3/mac} && param mac3 ${net3/mac} || goto bootme', + 'isset ${net4/mac} && param mac4 ${net4/mac} || goto bootme', + 'isset ${net5/mac} && param mac5 ${net5/mac} || goto bootme', + 'isset ${net6/mac} && param mac6 ${net6/mac} || goto bootme', + 'isset ${net7/mac} && param mac7 ${net7/mac} || goto bootme', ':bootme', "chain -ar $this->_booturl/ipxe/boot.php##params", ); @@ -632,6 +882,10 @@ private function _chainBoot($debug = false, $shortCircuit = false) 'login', 'params', 'param mac0 ${net0/mac}', + 'param product ${product}', + 'param manufacturer ${manufacturer}', + 'param ipxever ${version}', + 'param filename ${filename}', 'param arch ${arch}', 'param platform ${platform}', 'param username ${username}', @@ -639,8 +893,14 @@ private function _chainBoot($debug = false, $shortCircuit = false) 'param menuaccess 1', "param debug $debug", 'param sysuuid ${uuid}', + 'isset ${netX/mac} && param macboot ${netX/mac} ||', 'isset ${net1/mac} && param mac1 ${net1/mac} || goto bootme', 'isset ${net2/mac} && param mac2 ${net2/mac} || goto bootme', + 'isset ${net3/mac} && param mac3 ${net3/mac} || goto bootme', + 'isset ${net4/mac} && param mac4 ${net4/mac} || goto bootme', + 'isset ${net5/mac} && param mac5 ${net5/mac} || goto bootme', + 'isset ${net6/mac} && param mac6 ${net6/mac} || goto bootme', + 'isset ${net7/mac} && param mac7 ${net7/mac} || goto bootme', ':bootme', "chain -ar $this->_booturl/ipxe/boot.php##params", ); @@ -694,10 +954,7 @@ private function _approveHost() 'echo Host approved successfully', 'sleep 3' ); - $shutdown = stripos( - 'shutdown=1', - isset($_REQUEST['extraargs']) ? $_REQUEST['extraargs'] : '' - ); + $shutdown = self::_wantsShutdown(self::_extraArgs()); $isdebug = preg_match( '#isdebug=yes|mode=debug|mode=onlydebug#i', isset($_REQUEST['extraargs']) ? $_REQUEST['extraargs'] : '' @@ -786,13 +1043,23 @@ public function delConf() 'param delconf 1', ':deleteno', 'param mac0 ${net0/mac}', + 'param product ${product}', + 'param manufacturer ${manufacturer}', + 'param ipxever ${version}', + 'param filename ${filename}', 'param arch ${arch}', 'param platform ${platform}', 'param sysuuid ${uuid}', 'param username ${username}', 'param password ${password}', + 'isset ${netX/mac} && param macboot ${netX/mac} ||', 'isset ${net1/mac} && param mac1 ${net1/mac} || goto bootme', 'isset ${net2/mac} && param mac2 ${net2/mac} || goto bootme', + 'isset ${net3/mac} && param mac3 ${net3/mac} || goto bootme', + 'isset ${net4/mac} && param mac4 ${net4/mac} || goto bootme', + 'isset ${net5/mac} && param mac5 ${net5/mac} || goto bootme', + 'isset ${net6/mac} && param mac6 ${net6/mac} || goto bootme', + 'isset ${net7/mac} && param mac7 ${net7/mac} || goto bootme', ':bootme', "chain -ar $this->_booturl/ipxe/boot.php##params", ); @@ -814,13 +1081,23 @@ public function aprvConf() 'param aprvconf 1', ':answerno', 'param mac0 ${net0/mac}', + 'param product ${product}', + 'param manufacturer ${manufacturer}', + 'param ipxever ${version}', + 'param filename ${filename}', 'param arch ${arch}', 'param platform ${platform}', 'param sysuuid ${uuid}', 'param username ${username}', 'param password ${password}', + 'isset ${netX/mac} && param macboot ${netX/mac} ||', 'isset ${net1/mac} && param mac1 ${net1/mac} || goto bootme', 'isset ${net2/mac} && param mac2 ${net2/mac} || goto bootme', + 'isset ${net3/mac} && param mac3 ${net3/mac} || goto bootme', + 'isset ${net4/mac} && param mac4 ${net4/mac} || goto bootme', + 'isset ${net5/mac} && param mac5 ${net5/mac} || goto bootme', + 'isset ${net6/mac} && param mac6 ${net6/mac} || goto bootme', + 'isset ${net7/mac} && param mac7 ${net7/mac} || goto bootme', ':bootme', "chain -ar $this->_booturl/ipxe/boot.php##params", ); @@ -840,14 +1117,24 @@ public function keyreg() 'read key', 'params', 'param mac0 ${net0/mac}', + 'param product ${product}', + 'param manufacturer ${manufacturer}', + 'param ipxever ${version}', + 'param filename ${filename}', 'param arch ${arch}', 'param platform ${platform}', 'param key ${key}', 'param sysuuid ${uuid}', 'param username ${username}', 'param password ${password}', + 'isset ${netX/mac} && param macboot ${netX/mac} ||', 'isset ${net1/mac} && param mac1 ${net1/mac} || goto bootme', 'isset ${net2/mac} && param mac2 ${net2/mac} || goto bootme', + 'isset ${net3/mac} && param mac3 ${net3/mac} || goto bootme', + 'isset ${net4/mac} && param mac4 ${net4/mac} || goto bootme', + 'isset ${net5/mac} && param mac5 ${net5/mac} || goto bootme', + 'isset ${net6/mac} && param mac6 ${net6/mac} || goto bootme', + 'isset ${net7/mac} && param mac7 ${net7/mac} || goto bootme', ':bootme', "chain -ar $this->_booturl/ipxe/boot.php##params", ); @@ -893,12 +1180,22 @@ public function sesscheck() 'iseq ${arch} i386 && cpuid --ext 29 && set arch x86_64 ||', 'params', 'param mac0 ${net0/mac}', + 'param product ${product}', + 'param manufacturer ${manufacturer}', + 'param ipxever ${version}', + 'param filename ${filename}', 'param arch ${arch}', 'param platform ${platform}', 'param sessionJoin 1', 'param sysuuid ${uuid}', + 'isset ${netX/mac} && param macboot ${netX/mac} ||', 'isset ${net1/mac} && param mac1 ${net1/mac} || goto bootme', 'isset ${net2/mac} && param mac2 ${net2/mac} || goto bootme', + 'isset ${net3/mac} && param mac3 ${net3/mac} || goto bootme', + 'isset ${net4/mac} && param mac4 ${net4/mac} || goto bootme', + 'isset ${net5/mac} && param mac5 ${net5/mac} || goto bootme', + 'isset ${net6/mac} && param mac6 ${net6/mac} || goto bootme', + 'isset ${net7/mac} && param mac7 ${net7/mac} || goto bootme', ':bootme', "chain -ar $this->_booturl/ipxe/boot.php##params", ); @@ -919,12 +1216,22 @@ public function sesscheck() 'iseq ${arch} i386 && cpuid --ext 29 && set arch x86_64 ||', 'params', 'param mac0 ${net0/mac}', + 'param product ${product}', + 'param manufacturer ${manufacturer}', + 'param ipxever ${version}', + 'param filename ${filename}', 'param arch ${arch}', 'param platform ${platform}', 'param sessionJoin 1', 'param sysuuid ${uuid}', + 'isset ${netX/mac} && param macboot ${netX/mac} ||', 'isset ${net1/mac} && param mac1 ${net1/mac} || goto bootme', 'isset ${net2/mac} && param mac2 ${net2/mac} || goto bootme', + 'isset ${net3/mac} && param mac3 ${net3/mac} || goto bootme', + 'isset ${net4/mac} && param mac4 ${net4/mac} || goto bootme', + 'isset ${net5/mac} && param mac5 ${net5/mac} || goto bootme', + 'isset ${net6/mac} && param mac6 ${net6/mac} || goto bootme', + 'isset ${net7/mac} && param mac7 ${net7/mac} || goto bootme', ':bootme', "chain -ar $this->_booturl/ipxe/boot.php##params", ); @@ -947,14 +1254,24 @@ public function sessjoin() 'read sessname', 'params', 'param mac0 ${net0/mac}', + 'param product ${product}', + 'param manufacturer ${manufacturer}', + 'param ipxever ${version}', + 'param filename ${filename}', 'param arch ${arch}', 'param platform ${platform}', 'param sessname ${sessname}', 'param sysuuid ${uuid}', 'param username ${username}', 'param password ${password}', + 'isset ${netX/mac} && param macboot ${netX/mac} ||', 'isset ${net1/mac} && param mac1 ${net1/mac} || goto bootme', 'isset ${net2/mac} && param mac2 ${net2/mac} || goto bootme', + 'isset ${net3/mac} && param mac3 ${net3/mac} || goto bootme', + 'isset ${net4/mac} && param mac4 ${net4/mac} || goto bootme', + 'isset ${net5/mac} && param mac5 ${net5/mac} || goto bootme', + 'isset ${net6/mac} && param mac6 ${net6/mac} || goto bootme', + 'isset ${net7/mac} && param mac7 ${net7/mac} || goto bootme', ':bootme', "chain -ar $this->_booturl/ipxe/boot.php##params", ); @@ -1009,16 +1326,8 @@ public function falseTasking($mc = false, $Image = false) false, '' ); - $shutdown = false !== stripos( - 'shutdown=1', - $TaskType->get('kernelArgs') - ); - if (!$shutdown && isset($_REQUEST['extraargs'])) { - $shutdown = false !== stripos( - 'shutdown=1', - $_REQUEST['extraargs'] - ); - } + $shutdown = self::_wantsShutdown($TaskType->get('kernelArgs')) + || self::_wantsShutdown(self::_extraArgs()); if (!is_numeric($mcastmaxwait)) { $mcastmaxwait = 10; } @@ -1183,14 +1492,25 @@ function ($Image) use (&$Send) { ), 'params', 'param mac0 ${net0/mac}', + 'param product ${product}', + 'param manufacturer ${manufacturer}', + 'param ipxever ${version}', + 'param filename ${filename}', 'param arch ${arch}', + 'param platform ${platform}', 'param imageID ${imageID}', 'param qihost 1', 'param username ${username}', 'param password ${password}', 'param sysuuid ${uuid}', + 'isset ${netX/mac} && param macboot ${netX/mac} ||', 'isset ${net1/mac} && param mac1 ${net1/mac} || goto bootme', 'isset ${net2/mac} && param mac2 ${net2/mac} || goto bootme', + 'isset ${net3/mac} && param mac3 ${net3/mac} || goto bootme', + 'isset ${net4/mac} && param mac4 ${net4/mac} || goto bootme', + 'isset ${net5/mac} && param mac5 ${net5/mac} || goto bootme', + 'isset ${net6/mac} && param mac6 ${net6/mac} || goto bootme', + 'isset ${net7/mac} && param mac7 ${net7/mac} || goto bootme', 'goto bootme', ); unset($Image); @@ -1201,10 +1521,21 @@ function ($Image) use (&$Send) { ':return', 'params', 'param mac0 ${net0/mac}', + 'param product ${product}', + 'param manufacturer ${manufacturer}', + 'param ipxever ${version}', + 'param filename ${filename}', 'param arch ${arch}', + 'param platform ${platform}', 'param sysuuid ${uuid}', + 'isset ${netX/mac} && param macboot ${netX/mac} ||', 'isset ${net1/mac} && param mac1 ${net1/mac} || goto bootme', 'isset ${net2/mac} && param mac2 ${net2/mac} || goto bootme', + 'isset ${net3/mac} && param mac3 ${net3/mac} || goto bootme', + 'isset ${net4/mac} && param mac4 ${net4/mac} || goto bootme', + 'isset ${net5/mac} && param mac5 ${net5/mac} || goto bootme', + 'isset ${net6/mac} && param mac6 ${net6/mac} || goto bootme', + 'isset ${net7/mac} && param mac7 ${net7/mac} || goto bootme', 'goto bootme', ); $Send['bootmefunc'] = array( @@ -1241,10 +1572,7 @@ public function multijoin($msid) ->set('imageID', $msImage); } } - $shutdown = stripos( - 'shutdown=1', - isset($_REQUEST['extraargs']) ? $_REQUEST['extraargs'] : '' - ); + $shutdown = self::_wantsShutdown(self::_extraArgs()); $isdebug = preg_match( '#isdebug=yes|mode=debug|mode=onlydebug#i', isset($_REQUEST['extraargs']) ? $_REQUEST['extraargs'] : '' @@ -1308,6 +1636,17 @@ public function keyset() */ private function _parseMe($Send) { + /** + * Anything _hostOverride() decided the operator needs to know, + * emitted once, on whichever path actually runs. Appended rather + * than prepended because the very first batch through here opens + * with '#!ipxe', which has to stay the first line of the script; + * and drained so a later batch does not repeat them. + */ + if (count($this->_notices) > 0) { + $Send['archnotices'] = $this->_notices; + $this->_notices = array(); + } self::$HookManager->processEvent( 'IPXE_EDIT', array( @@ -1389,9 +1728,12 @@ public function verifyCreds() if ($noMenu) { $this->noMenu(); } - $tmpUser = self::attemptLogin( - $_REQUEST['username'], - $_REQUEST['password'] + // authenticateOnly: iPXE holds no cookie, so a session established + // here could never be presented back -- it would just be an + // authenticated session nobody owns. isValid() below is the point. + $tmpUser = self::authenticateOnly( + $_REQUEST['username'] ?? '', + $_REQUEST['password'] ?? '' ); if ($tmpUser->isValid()) { self::$HookManager @@ -1437,10 +1779,7 @@ public function verifyCreds() */ public function setTasking($imgID = '') { - $shutdown = stripos( - 'shutdown=1', - isset($_REQUEST['extraargs']) ? $_REQUEST['extraargs'] : '' - ); + $shutdown = self::_wantsShutdown(self::_extraArgs()); $isdebug = preg_match( '#isdebug=yes|mode=debug|mode=onlydebug#i', isset($_REQUEST['extraargs']) ? $_REQUEST['extraargs'] : '' @@ -1618,16 +1957,8 @@ public function getTasking() false, '' ); - $shutdown = false !== stripos( - 'shutdown=1', - $TaskType->get('kernelArgs') - ); - if (!$shutdown && isset($_REQUEST['extraargs'])) { - $shutdown = false !== stripos( - 'shutdown=1', - $_REQUEST['extraargs'] - ); - } + $shutdown = self::_wantsShutdown($TaskType->get('kernelArgs')) + || self::_wantsShutdown(self::_extraArgs()); if (!is_numeric($mcastmaxwait)) { $mcastmaxwait = 10; } @@ -1849,11 +2180,11 @@ public function getTasking() self::$Host->get('kernelArgs'), ); if ($Task->get('typeID') == 4) { - $Send['memtest'] = array( - "$this->_memdisk iso raw", - "$this->_memtest", - "boot", - ); + // No '|| goto MENU' tail: a tasked boot has no menu to + // return to. On an architecture without memdisk this says + // why and stops, rather than dropping the machine to a + // bare iPXE prompt with no explanation. + $Send['memtest'] = $this->_memtestChoice(''); $this->_parseMe($Send); } else { $this->_printTasking($kernelArgsArray); @@ -1880,7 +2211,7 @@ private function _menuItem($option, $desc) ); } } - return array("item${hotkey}${name} ${desc}"); + return array("item{$hotkey}{$name} {$desc}"); } /** * The options of the menu @@ -1930,14 +2261,7 @@ private function _menuOpt($option, $type) ); break; case 2: - $Send = self::fastmerge( - $Send, - array( - "$this->_memdisk iso raw", - $this->_memtest, - 'boot || goto MENU' - ) - ); + $Send = self::fastmerge($Send, $this->_memtestChoice()); break; case 11: $Send = self::fastmerge( @@ -2051,7 +2375,6 @@ public function printDefault() $this->_chainBoot(true); return; } - $Menus = self::getClass('PXEMenuOptionsManager')->find('', '', 'id'); $ipxeGrabs = array( 'FOG_ADVANCED_MENU_LOGIN', 'FOG_IPXE_BG_FILE', diff --git a/packages/web/lib/fog/clientupdater.class.php b/packages/web/lib/fog/clientupdater.class.php index c66dc3cc0a..b1a77d9d84 100644 --- a/packages/web/lib/fog/clientupdater.class.php +++ b/packages/web/lib/fog/clientupdater.class.php @@ -2,7 +2,7 @@ /** * Deals with the client updater files * - * PHP version 5 + * PHP version 7.4+ * * @category ClientUpdater * @package FOGProject diff --git a/packages/web/lib/fog/clientupdatermanager.class.php b/packages/web/lib/fog/clientupdatermanager.class.php index 94c8a01e4e..004f99a6b1 100644 --- a/packages/web/lib/fog/clientupdatermanager.class.php +++ b/packages/web/lib/fog/clientupdatermanager.class.php @@ -2,7 +2,7 @@ /** * Client Update Manager handles the mass client update stuff. * - * PHP version 5 + * PHP version 7.4+ * * @category ClientUpdaterManager * @package FOGProject diff --git a/packages/web/lib/fog/csrf.class.php b/packages/web/lib/fog/csrf.class.php index 84d87b5ab1..bd2b87bbc2 100644 --- a/packages/web/lib/fog/csrf.class.php +++ b/packages/web/lib/fog/csrf.class.php @@ -2,7 +2,7 @@ /** * CSRF, hopefully more secure handling centralized. * - * PHP version 5 + * PHP version 7.4+ * * For setting/checking CSRF tokens. * diff --git a/packages/web/lib/fog/dircleaner.class.php b/packages/web/lib/fog/dircleaner.class.php index bc167a33a3..32891cb254 100644 --- a/packages/web/lib/fog/dircleaner.class.php +++ b/packages/web/lib/fog/dircleaner.class.php @@ -2,7 +2,7 @@ /** * Dir Cleaner handles directory cleanup * - * PHP version 5 + * PHP version 7.4+ * * @category DirCleaner * @package FOGProject diff --git a/packages/web/lib/fog/dircleanermanager.class.php b/packages/web/lib/fog/dircleanermanager.class.php index bd83cdbd3f..c2a531632a 100644 --- a/packages/web/lib/fog/dircleanermanager.class.php +++ b/packages/web/lib/fog/dircleanermanager.class.php @@ -2,7 +2,7 @@ /** * Directory Cleaner Manager deals with mass Dir Cleaner items. * - * PHP version 5 + * PHP version 7.4+ * * @category DirCleanerManager * @package FOGProject diff --git a/packages/web/lib/fog/event.class.php b/packages/web/lib/fog/event.class.php index 1ffdc93004..06dd1caea6 100644 --- a/packages/web/lib/fog/event.class.php +++ b/packages/web/lib/fog/event.class.php @@ -4,7 +4,7 @@ * Because of the similarities of use for events and hooks * the event class here is the hook base model as well. * - * PHP version 5 + * PHP version 7.4+ * * @category Event * @package FOGProject diff --git a/packages/web/lib/fog/eventmanager.class.php b/packages/web/lib/fog/eventmanager.class.php index 616ed211ef..19cbb5ca5e 100644 --- a/packages/web/lib/fog/eventmanager.class.php +++ b/packages/web/lib/fog/eventmanager.class.php @@ -3,7 +3,7 @@ * EventManager handles registering and loading * events and hooks. * - * PHP version 5 + * PHP version 7.4+ * * @category EventManager * @package FOGProject @@ -41,6 +41,19 @@ class EventManager extends FOGBase * @var mixed */ public $events; + /** + * Names already present in notifyEvents, as a lookup set. + * + * notify() used to ask the database whether the event name was already + * recorded on EVERY call, and notify() is called from the snapin client + * protocol and from taskqueue, so that was a round trip per snapin per + * check-in. This is the same mistake HookManager::processEvent() had and + * the same fix; see HookManager::$knownEvents for why remembering the + * answer for the life of the process is safe. + * + * @var array|null + */ + private static $knownNotifyEvents = null; /** * Registers events and listeners within the system. * @@ -98,14 +111,14 @@ public function register($event, $listener) } } catch (Exception $e) { $string = sprintf( - '%s: %s: %s, $s: %s, %s: %s', + '%s: %s: %s, %s: %s, %s: %s', _('Could not register'), _('Error'), $e->getMessage(), _('Event'), - $event, + self::_describeEvent($event), _('Class'), - $listener[0] + self::_describeListener($listener) ); self::log( $string, @@ -117,6 +130,82 @@ public function register($event, $listener) } return $this; } + /** + * Names a listener for a log line, whatever shape it arrived in. + * + * This used to be a bare `$listener[0]`, written inside the very catch + * that exists to swallow a bad listener. Handing register() an object that + * is not an array -- a closure, say -- therefore raised "Cannot use object + * of type X as array" from the error handler itself, which is an Error and + * not caught by catch (Exception). Registration runs in a hook constructor + * during LoadGlobals, so that escaped to the top and the whole application + * answered 500 with an empty body, on every entry point, until the file was + * deleted from disk. An error handler must not be able to fail harder than + * the error it is reporting. + * + * @param mixed $listener The listener as the caller supplied it. + * + * @return string + */ + private static function _describeListener($listener) + { + if (is_array($listener)) { + $first = reset($listener); + return is_object($first) ? get_class($first) : gettype($first); + } + if (is_object($listener)) { + return get_class($listener); + } + return gettype($listener); + } + /** + * Renders an event name for a log line. + * + * One of the conditions that reaches the catch below is "$event is not a + * string", and %s on an object with no __toString is an Error, which + * catch (Exception) does not catch. Same defect as _describeListener() + * covers for the listener: an error handler must not be able to fail + * harder than the error it is reporting. + * + * @param mixed $event The event as the caller supplied it. + * + * @return string + */ + private static function _describeEvent($event) + { + return is_string($event) ? $event : gettype($event); + } + /** + * Records an event name in notifyEvents if it is not already there. + * + * The table is a discovery aid -- it is what the notify event list is + * built from -- so it is written opportunistically rather than being + * authoritative. Marked known before the save, not after, so an event + * fired from inside save() could not recurse into saving the same name. + * + * @param string $event the event name to record + * + * @return void + */ + private static function _recordEventName($event) + { + if (self::$knownNotifyEvents === null) { + self::$knownNotifyEvents = array_flip( + (array) self::getSubObjectIDs( + 'NotifyEvent', + array(), + 'name' + ) + ); + } + if (isset(self::$knownNotifyEvents[$event])) { + return; + } + self::$knownNotifyEvents[$event] = true; + self::getClass('NotifyEvent') + ->set('name', $event) + ->save(); + } /** * Notifies the system of events. * @@ -129,16 +218,6 @@ public function register($event, $listener) */ public function notify($event, $eventData = array()) { - $exists = self::getClass('NotifyEventManager')->exists( - $event, - '', - 'name' - ); - if (!$exists) { - self::getClass('NotifyEvent') - ->set('name', $event) - ->save(); - } try { if (!is_string($event)) { throw new Exception(_('Event must be a string')); @@ -146,6 +225,13 @@ public function notify($event, $eventData = array()) if (!is_array($eventData)) { throw new Exception(_('Event Data must be an array')); } + // Recorded here rather than above the try, which is where it used + // to sit. Running before the guard meant a caller that passed an + // array or an object still got that value handed to + // NotifyEvent::set('name') and saved, so the discovery table + // collected rows for things that were never event names -- and + // the guard then rejected the same value one line later. + self::_recordEventName($event); if (!isset($this->data[$event])) { throw new Exception(_('Event and data are not set')); } @@ -161,12 +247,12 @@ public function notify($event, $eventData = array()) } } catch (Exception $e) { $string = sprintf( - '%s: %s: %s, $s: %s', + '%s: %s: %s, %s: %s', _('Could not notify'), _('Error'), $e->getMessage(), _('Event'), - $event + self::_describeEvent($event) ); self::log( $string, @@ -188,34 +274,33 @@ public function notify($event, $eventData = array()) */ public function load() { - // Sets up regex and paths to scan for - if ($this instanceof self) { - $regext = sprintf( - '#^.+%sevents%s.*\.event\.php$#', - DS, - DS - ); - ; - $dirpath = sprintf( - '%sevents%s', - DS, - DS - ); - $strlen = -strlen('.event.php'); - } + // Sets up regex and paths to scan for. + // + // HookManager extends EventManager, so a HookManager satisfies + // `instanceof self` too. This used to be two sequential ifs and the + // hook branch was reached only because it ran second and overwrote + // what the event branch had just assigned -- reordering the two + // blocks silently made every hook load as an event and find nothing. + // One decision, taken most-specific first, cannot be reordered wrong. if ($this instanceof HookManager) { - $regext = sprintf( - '#^.+%shooks%s.*\.hook\.php$#', - DS, - DS - ); - $dirpath = sprintf( - '%shooks%s', - DS, - DS - ); - $strlen = -strlen('.hook.php'); + $type = 'hook'; + } else { + $type = 'event'; } + $regext = sprintf( + '#^.+%s%ss%s.*\.%s\.php$#', + DS, + $type, + DS, + $type + ); + $dirpath = sprintf( + '%s%ss%s', + DS, + $type, + DS + ); + $strlen = -strlen(sprintf('.%s.php', $type)); // Initiates plugins used in fileitems function $plugins = ''; // Function simply returns the files based on the regex and data passed. @@ -288,11 +373,10 @@ public function load() $strlen ) ); - $decClasses = get_declared_classes(); - foreach ((array)$decClasses as $key => &$classExist) { - $exists[$classExist] = 1; - unset($classExist); - } + // There used to be a loop building a lookup of every declared + // class into $exists here, immediately before $exists was + // overwritten by the class_exists() call below. It ran once per + // hook or event file and its result was never read. $exists = class_exists( $className, false @@ -307,7 +391,7 @@ public function load() $className ) ); - unset($element, $key); + unset($element); }; // Plugins should be established first so menus and what not are setup. array_map( diff --git a/packages/web/lib/fog/fogbase.class.php b/packages/web/lib/fog/fogbase.class.php index 7369f7aeab..ced47e4d1e 100644 --- a/packages/web/lib/fog/fogbase.class.php +++ b/packages/web/lib/fog/fogbase.class.php @@ -2,7 +2,7 @@ /** * FOGBase, the base class for pretty much all of fog. * - * PHP version 5 + * PHP version 7.4+ * * This gives all the rest of the classes a common frame to work from. * @@ -534,7 +534,7 @@ public static function getHostItem( file_get_contents('php://input'), $vars ); - $mac = $vars['mac']; + $mac = $vars['mac'] ?? ''; } } // disabling sysuuid detection code for now as it is causing @@ -545,11 +545,12 @@ public static function getHostItem( $sysuuid = filter_input(INPUT_GET, 'sysuuid'); } */ - // If encoded decode and store value - if ($encoded === true) { - $mac = base64_decode($mac); - // $sysuuid = base64_decode($sysuuid); - } + // Normalize the mac. stripAndDecode() rewrites $_REQUEST, but the mac + // is read here from the raw request via filter_input() (or passed in + // explicitly), which that rewrite never touches, so the encoding has + // to be resolved here. The legacy $encoded flag is now redundant but + // kept for call-signature compatibility. + $mac = self::stripAndDecodeMac($mac); // See if we can find the host by system uuid rather than by mac's first. /* if ($sysuuid) { $Inventory = self::getClass('Inventory') @@ -766,6 +767,212 @@ protected static function debug($txt, $data = array()) ); printf('
%s
', $string); } + /** + * FOG's log directory. + * + * 1.5 has no FOG_LOG_DIR constant -- every service spells this path out + * -- so this one does too, matching TaskError::LOG_DIR and the + * installer's $servicelogs default. FOG_LOG_DIR is still preferred when + * something HAS defined it, which costs nothing, keeps this method the + * same shape as the 1.6 original it was ported from, and is what lets a + * test point it somewhere writable. + * + * @var string + */ + const FAULT_LOG_DIR = '/opt/fog/log'; + /** + * The subdirectory of that directory fault lines are written to. + * + * Its own subdirectory rather than the top level, for the reason + * TaskError gives for the FOS report log: rotation renames and unlinks, + * and the top level is root's -- the eight daemons' logs live there and + * nothing running as the web user should be able to remove them. + * + * @var string + */ + const FAULT_LOG_SUBDIR = 'faults'; + /** + * How big a fault log may get before one old copy is kept, in bytes. + * + * A literal, NOT the SERVICE_LOG_SIZE setting the daemons rotate on, and + * that is the whole point: getSetting() issues a query, and the thing + * being reported here is a query that just failed. + * + * @var int + */ + const FAULT_LOG_MAX = 10485760; + /** + * How long a single fault line may get, in bytes, before it is cut. + * + * A backstop, not the main defence: logFault() drops PDODB's debug tail + * outright (see there). This catches what has no tail to drop -- a + * driver message that is itself enormous, or a caller that built its + * own. One fault stays one readable line either way. + * + * @var int + */ + const FAULT_LINE_MAX = 2048; + /** + * Records that something FOG needed to write or read did not happen. + * + * The failure sink of last resort, and deliberately the only logger here + * that asks nobody's permission to run. + * + * WHY THIS EXISTS AT ALL. FOGController::save(), destroy() and load() + * recorded a failure by calling logHistory(), which returns without doing + * anything unless self::$FOGUser is a valid User. Nothing on a machine + * -facing path ever sets one -- packages/web/service/, lib/reg-task/ and + * the daemons are matched to a HOST by MAC or token, and the daemons have + * no request at all -- so on every one of those paths the failure branch + * ran and wrote nowhere. + * + * debug() was not a second chance. On this branch it writes to no file at + * ALL -- it printf()s into the page and returns immediately when + * self::$service or self::$ajax is set, which on a machine endpoint is + * always. So a failed write on a service path had literally no possible + * output. (1.6's debug() at least reaches a file, behind a globalSetting + * that ships off.) + * + * WHY A FILE AND NOT A TABLE. logHistory() writes a row, so it shares its + * failure mode with the thing it is reporting on: a lost connection, a + * locked table or a full disk takes out the report along with the write. + * A sink for a failed database operation cannot itself be a database + * write. That -- not the user gate -- is the structural reason this is + * not simply logHistory() with the gate widened. The user gate is correct + * where it is: `history` is the audit trail, "who did what", and nobody + * did this. + * + * IT MUST NOT call getSetting(), for the path or the rotation size or + * anything else: getSetting() issues a query, and a logger that queries + * in order to report a failed query is the recursion that has already + * cost this project a silently dying worker. FAULT_LOG_MAX is a literal + * for exactly that reason. + * + * error_log() is the fallback, not the destination, and it is + * load-bearing rather than tidiness. The directory is the installer's, so + * a server whose web tree has been updated but which has not been + * re-installed has nowhere to write yet -- and PHP's own channel is + * already pointed somewhere useful in both tiers. + * + * @param string $message what did not happen, and why + * + * @return void + */ + public static function logFault($message) + { + // Nothing here can re-enter through the database; this covers the one + // real case, which is logFault() failing on its own file write. + static $inFault = false; + if ($inFault) { + return; + } + $inFault = true; + + try { + /* + * Drop PDODB's debug tail BEFORE anything else looks at the + * message. Its error text always appends + * "\nSQL: ...\nParams: ...\nErrorInfo: ...\nDebug: ..." + * (pdodb.class.php, both sqlerror() formats), and the Params and + * Debug sections print every BOUND VALUE of the statement that + * failed. On `users` that is the password hash, on `hosts` the + * client security token, on `nfsGroupMembers` the storage node's + * FTP password -- the credential GHSA-2hqx turns into root. + * + * That was survivable while this text only ever reached + * logHistory(), which is user-gated and so dropped it on exactly + * the machine paths that fail most, and debug(), which ships + * off. It is NOT survivable in a file written unconditionally on + * every failed write, and readable by any local account. What an + * operator actually needs is the part before the tail: the + * driver, the SQLSTATE and the message. + */ + $raw = (string) $message; + foreach (array("\nSQL: ", "\nParams: ", "\nErrorInfo: ", "\nDebug: ") as $marker) { + $tail = strpos($raw, $marker); + if (false !== $tail) { + $raw = substr($raw, 0, $tail); + } + } + // One line per fault, so `tail -f` stays readable and a + // multi-line message cannot be mistaken for several faults. + $flat = preg_replace('#\s+#', ' ', $raw); + // Never let a message become an empty one. preg_replace returns + // null when it gives up, and a (string) cast of that is '', which + // the guard below would then throw away -- losing the one record + // this method exists to keep. + $line = trim(null === $flat ? $raw : $flat); + if ('' === $line) { + return; + } + if (strlen($line) > self::FAULT_LINE_MAX) { + $line = substr($line, 0, self::FAULT_LINE_MAX) . ' [truncated]'; + } + $stamped = sprintf( + '[%s] %s%s', + date('Y-m-d H:i:s'), + $line, + PHP_EOL + ); + $file = self::_faultLogPath(); + if ('' !== $file) { + self::_rotateFaultLog($file); + if (false !== @file_put_contents($file, $stamped, FILE_APPEND)) { + return; + } + } + error_log($line); + } finally { + $inFault = false; + } + } + /** + * The fault log's path, or '' if there is nowhere to write. + * + * Split by SAPI, into faults-web.log and faults-service.log. This is the + * one FOG log directory written by BOTH tiers -- the web user, and root + * for the daemons -- and a single shared file would be owned by whichever + * wrote first. A root-owned file appears the moment any daemon hits a + * failed write, and from then on every web-tier fault would fall silently + * to error_log(). Silently diverting to a worse destination is the exact + * failure this whole path exists to end, so the two writers get two files. + * + * The directory is never created here. It is the installer's, which gives + * it to the web user with the right SELinux label (GH-964: /opt/fog + * inherits usr_t and httpd_t may read it but not write it, so an + * unlabelled mkdir would produce a directory that looks right and + * silently swallows every write on an enforcing host). + * + * @return string + */ + private static function _faultLogPath() + { + $base = defined('FOG_LOG_DIR') ? FOG_LOG_DIR : self::FAULT_LOG_DIR; + $dir = rtrim($base, DS) . DS . self::FAULT_LOG_SUBDIR; + if (!is_dir($dir) || !is_writable($dir)) { + return ''; + } + + return $dir . DS . sprintf( + 'faults-%s.log', + 'cli' === PHP_SAPI ? 'service' : 'web' + ); + } + /** + * Keeps one old copy once the fault log passes FAULT_LOG_MAX. + * + * @param string $file the fault log + * + * @return void + */ + private static function _rotateFaultLog($file) + { + $size = @filesize($file); + if (false === $size || $size < self::FAULT_LOG_MAX) { + return; + } + @rename($file, $file . '.1'); + } /** * Prints info. * @@ -1210,6 +1417,31 @@ protected static function getGlobalModuleStatus($names = false, $keys = false) */ public static function niceDate($date = 'now', $utc = false) { + /* + * GH-1245: an empty value means "this never happened", not "now". + * + * new DateTime('') and new DateTime(null) both return the CURRENT + * time, so a date column holding no value renders as a real + * timestamp. That has stayed hidden because FOGController::save() + * writes '' into date columns and PDODB clears sql_mode on every + * connection, so the server coerces it to '0000-00-00 00:00:00' -- + * and THAT parses to year -0001, which validDate() rejects and + * formatTime() renders as "No Data". The empty case is only reached + * by the columns that are already nullable, where it is wrong today. + * + * Mapping empty onto the same zero date makes the two spellings of + * "no value" render identically, which is also what lets the columns + * move to NULL without the display changing -- FOGController::get() + * hands back null for a NULL column. + * + * Callers that genuinely want the current time pass 'now', which is + * this method's own default. The ten call sites in this branch that + * relied on '' meaning now were changed to say 'now' in the same + * commit. + */ + if (null === $date || (is_string($date) && '' === trim($date))) { + $date = '0000-00-00 00:00:00'; + } if ($utc || empty(self::$TimeZone)) { $tz = new DateTimeZone('UTC'); } else { @@ -2329,6 +2561,15 @@ public static function getCancelledState() { return TaskState::getCancelledState(); } + /** + * Get failed state id. + * + * @return int + */ + public static function getFailedState() + { + return TaskState::getFailedState(); + } /** * Safe min() over a collection that may be empty. * @@ -2381,6 +2622,55 @@ public static function stringBetween($string, $start, $end) return substr($string, $ini, $len); } + /** + * Decodes a credential FOS sent base64-encoded. + * + * NOT stripAndDecode(), which is what the registration path used to use. + * That helper finishes with Initiator::e() -- HTML escaping, which is + * right for a value about to be rendered into a page and wrong for one + * about to be compared against a password hash. A password containing + * & < > " or ' arrived at password_verify() as its entity form and could + * never match, so those accounts could not register-with-deploy while + * working perfectly in the web UI, which does not go through that helper. + * Forums topic 18228. + * + * STRICT decoding, unlike stripAndDecode()'s. base64_decode() without + * $strict silently drops every character outside the alphabet and always + * "succeeds", so a corrupted field became a plausible wrong credential + * rather than a refused one. + * + * Shared rather than written out twice because service/checkcredentials.php + * validates the SAME credential for the SAME caller. The two disagreeing is + * the bug: that endpoint answered '#!ok' for a password registration then + * rejected. + * + * @param mixed $value the raw request value + * + * @return string|bool the decoded credential, or false if it was not + * valid base64 + */ + public static function decodeCredential($value) + { + /* + * Restore '+' from ' ' before decoding, exactly as stripAndDecode() + * has always done. '+' is in the base64 alphabet and a bare '+' in a + * urlencoded body decodes back to a space, so a credential whose + * encoding contains one arrives corrupted. A space is never valid + * base64, so the swap is lossless -- and without it the strict decode + * below would REFUSE those credentials rather than mangle them, which + * is a worse failure than the one being fixed. + */ + $value = str_replace(' ', '+', trim((string) ($value ?? ''))); + $decoded = base64_decode($value, true); + if (!is_string($decoded)) { + return false; + } + + // Trimmed to match checkcredentials.php. Both ends must agree, and a + // credential that differs only by surrounding whitespace is not one + // anybody can type reliably at the FOS prompt anyway. + return trim($decoded); + } /** * Strips and decodes items. * @@ -2404,6 +2694,65 @@ public static function stripAndDecode(&$item) return $item; } + /** + * Strips and decodes a mac, or a '|' separated list of macs. + * + * FOS base64-encodes the mac on some paths (registration, deploy) and + * sends it plain on others (checkin, the standalone inventory task), so + * the encoding has to be sniffed. The sniff cannot be the one + * stripAndDecode() uses -- "do the decoded bytes happen to be valid + * UTF-8" -- because a hex mac is built entirely out of base64 alphabet + * characters, so a plain mac decodes to accidentally-valid UTF-8 roughly + * once in every few hundred (measured: 0.26% lowercase, 0.84% upper) and + * that host would then silently fail to resolve, intermittently and per + * mac. Sniff on shape instead: keep the plain value when it is already a + * well formed mac list, and accept the decoded value only when it is one. + * + * @param mixed $mac the raw mac value + * + * @return string + */ + public static function stripAndDecodeMac($mac) + { + $mac = trim((string) ($mac ?? '')); + if ($mac === '' || self::isMacList($mac)) { + return Initiator::e($mac); + } + $decoded = trim(base64_decode(str_replace(' ', '+', $mac))); + if (self::isMacList($decoded)) { + return Initiator::e($decoded); + } + + // Neither shape matched; hand back the plain value so the caller + // reports the mac it was actually sent. + return Initiator::e($mac); + } + /** + * Tests whether a string is a '|' separated list of mac addresses. + * + * @param string $macs the string to test + * + * @return bool + */ + private static function isMacList($macs) + { + $parts = array_filter( + array_map( + 'trim', + explode('|', $macs) + ) + ); + if (count($parts) < 1) { + return false; + } + foreach ($parts as $part) { + if (!preg_match(MACAddress::PATTERN, $part)) { + return false; + } + } + + return true; + } /** * Gets the master interface based on the ip found. * @@ -2703,6 +3052,27 @@ public static function attemptLogin($username, $password) return self::getClass('User') ->validatePw($username, $password); } + /** + * Proves a credential without establishing a session. + * + * For callers with no browser to carry one -- the iPXE boot menu and + * service/ipxe/advanced.php -- where attemptLogin() would otherwise + * stamp $_SESSION['FOG_USER'] for a request that can never present the + * cookie back. + * + * Returns a User either way, exactly like attemptLogin(), so callers + * MUST test isValid(). A returned object is never itself the answer. + * + * @param string $username the username to attempt + * @param string $password the password to attempt + * + * @return object + */ + public static function authenticateOnly($username, $password) + { + return self::getClass('User') + ->authenticate($username, $password); + } /** * Clears the mac lookup table * @@ -2938,6 +3308,333 @@ public static function validSchemaBootstrap() return self::installTokenHeader() || (self::schemaNeedsDeploy() && self::installTokenParam()); } + /** + * The globalSettings key holding the shared node-signing secret. + * + * Deliberately NOT a config.class.php constant like + * FOG_SCHEMA_INSTALL_TOKEN: every storage node's installer generates its + * own config.class.php with its own random values, so a constant would + * differ on every machine and never verify. globalSettings is the one + * store master and node genuinely share -- functions.sh points a node's + * DATABASE_HOST at the master (see `[[ -z ${DB_host} ]] && + * DB_host="$snmysqlhost"`), so a row written once is readable everywhere + * with nothing to distribute. + * + * That last clause holds for a TRUE storage node and only for one. A + * peer that is itself a full FOG server -- its own DATABASE_HOST, its + * own globalSettings -- shares no row with the master, mints its own + * key here, and cannot verify anything the master signs. Nothing in the + * installer distributes this value, and validNodeSignature() must never + * mint one, so a pure receiver cannot heal itself either. + * + * nodeSigningKeyFor() is the answer for that topology: a per-peer key + * on the master's storage node record, which the administrator also + * sets as that peer's own FOG_NODE_API_KEY. Same model as ngmUser and + * ngmPass, which have always had to be kept in step with the account + * that actually exists on the node. + * + * @var string + */ + const NODE_API_KEY_SETTING = 'FOG_NODE_API_KEY'; + /** + * How far, in seconds, a signed request's timestamp may be from ours. + * + * This is the property service/nodecert.php does NOT have: its HMAC + * covers only the payload, so a captured request is replayable forever. + * Node traffic runs with CURLOPT_SSL_VERIFYPEER off (NODE_TLS_OPTIONS in + * FOGURLRequests -- a node's certificate is self-signed and there is no + * chain to check), so a capture is a realistic thing to defend against. + * + * Five minutes rather than something tighter because master and node + * clocks are not disciplined to each other by anything FOG installs, and + * the failure mode of too-tight is a node that silently serves nothing. + * It bounds replay to the same method on the same path -- for the reads + * this authenticates, that is a re-read of a directory listing. + * + * @var int + */ + const NODE_SIGNATURE_WINDOW = 300; + /** + * The shared secret FOG's own components sign inter-node requests with, + * created on first use if it is not there yet. + * + * Purpose-scoped on purpose. The obvious existing secret to reuse was + * FOG_STORAGENODE_MYSQLPASS, which is what service/nodecert.php signs + * with -- but that password is direct database access. Leaking it during + * transport hands an attacker the whole schema; leaking this hands them + * the ability to list directories on a node, which is all it authorises. + * + * INSERT IGNORE rather than setSetting(), for two reasons. setSetting() + * is an UPDATE through ServiceManager and does nothing at all when the + * row is absent, which is exactly the case being healed here. And the + * UNIQUE INDEX schema step 225 put on settingKey makes the INSERT the + * arbiter when two processes race -- both then re-read and agree, + * instead of the loser signing with a key the verifier has replaced. + * + * @return string The key, or '' if one could not be established. + */ + public static function nodeApiKey() + { + $key = trim((string)self::getSetting(self::NODE_API_KEY_SETTING)); + if ($key !== '') { + return $key; + } + try { + $candidate = bin2hex(random_bytes(32)); + } catch (Exception $e) { + // No CSPRNG means no key. Returning '' leaves callers + // unauthenticated, which is the safe direction: an unsigned + // request is refused, a weakly signed one would not be. + return ''; + } + self::$DB->query( + sprintf( + "INSERT IGNORE INTO `globalSettings` (`settingKey`, " + . "`settingDesc`, `settingValue`, `settingCategory`) " + . "VALUES (%s, %s, %s, %s)", + self::$DB->escape(self::NODE_API_KEY_SETTING), + self::$DB->escape( + 'Shared secret FOG signs its own server-to-server ' + . 'requests with. Generated automatically; there is ' + . 'nothing to set here, and the FOG Configuration page ' + . 'does not show it. Delete the row to rotate the key -- ' + . 'every component reads it from this table, so the next ' + . 'request regenerates one and they agree again.' + ), + self::$DB->escape($candidate), + self::$DB->escape('FOG Storage Nodes') + ) + ); + // Re-read rather than trusting $candidate: on a race the INSERT was + // ignored and the row holds the other process's value. + return trim((string)self::getSetting(self::NODE_API_KEY_SETTING)); + } + /** + * The exact bytes both ends run through hash_hmac(). + * + * Method and path are in the signed material so a captured signature + * cannot be lifted onto a different request -- a GET of a directory + * listing must not become a POST of anything. The timestamp is in it so + * it cannot be adjusted to widen the window it was issued for. + * + * @param string $method The HTTP method, upper case. + * @param string $uri Path plus query string, exactly as sent. + * @param string $timestamp Unix seconds, as a decimal string. + * + * @return string + */ + /** + * The signing key for one peer, or '' to fall back to the shared one. + * + * A storage node that shares the master's database verifies with the + * global key and needs nothing here. A peer that is a full FOG server + * has its own globalSettings and cannot see the master's row at all, so + * the two ends have to be given a value in common by hand. + * + * nfsGroupMembers.ngmKey is where it goes. The column has existed since + * 1.5, is declared on StorageNode as `key`, and has never been read or + * written by anything. + * + * Matched on ngmHostname because that is what the caller has: signing + * happens in FOGURLRequests, which knows a URL and not which node it + * belongs to. A host that matches no node, or a node with an empty key, + * returns '' and the caller signs with the installation-wide key -- + * which is what every existing shared-database install keeps doing. + * + * @param string $host The host part of the URL about to be requested. + * + * @return string The peer's key, or '' if it has none. + */ + public static function nodeSigningKeyFor($host) + { + $host = trim((string)$host); + if ($host === '') { + return ''; + } + // fetch_all rather than fetch()->get('ngmKey'): on a host that + // matches no node the single-row form hands back the empty result + // set itself, which casts to the string 'Array' -- a non-empty + // "key" that signs every request to an unknown host with a + // constant nobody can verify. Indexing a list makes "no row" and + // "no key" the same, empty, answer. + $rows = self::$DB->query( + sprintf( + 'SELECT `ngmKey` FROM `nfsGroupMembers` ' + . 'WHERE `ngmHostname` = %s AND `ngmKey` <> %s LIMIT 1', + self::$DB->escape($host), + self::$DB->escape('') + ) + )->fetch(\PDO::FETCH_ASSOC, 'fetch_all')->get(); + $rows = (array)$rows; + if (count($rows) < 1) { + return ''; + } + return trim((string)(isset($rows[0]['ngmKey']) ? $rows[0]['ngmKey'] : '')); + } + /** + * Every key a signature reaching THIS server could legitimately carry. + * + * The global key first, because on a shared-database install that is + * the one the master signed with and the common case should cost one + * comparison. + * + * Then every non-empty ngmKey this server can see. Two topologies need + * it and they need it for opposite reasons: + * + * - Shared database. The master signed with the target node's own + * ngmKey; the node reads the same table, so the key is right there. + * - Standalone peer. The administrator set this server's + * FOG_NODE_API_KEY to match, so the global key already covers it -- + * but this server's OWN node rows are also legitimate signers if it + * is a master in its own right. + * + * The candidate set is bounded by the number of storage nodes and each + * miss is one hash_hmac, so the cost is not worth a cache that could + * then go stale against a rotated key. + * + * @return array Distinct non-empty keys. + */ + private static function _nodeVerificationKeys() + { + $keys = array(); + $global = trim((string)self::getSetting(self::NODE_API_KEY_SETTING)); + if ($global !== '') { + $keys[] = $global; + } + $rows = self::$DB->query( + 'SELECT `ngmKey` FROM `nfsGroupMembers` ' + . "WHERE `ngmKey` <> ''" + )->fetch(\PDO::FETCH_ASSOC, 'fetch_all')->get(); + foreach ((array)$rows as $row) { + $candidate = trim( + (string)(isset($row['ngmKey']) ? $row['ngmKey'] : '') + ); + if ($candidate !== '') { + $keys[] = $candidate; + } + } + return array_values(array_unique($keys)); + } + private static function _nodeSignaturePayload($method, $uri, $timestamp) + { + return $timestamp . "\n" . $method . "\n" . $uri; + } + /** + * Headers proving a request came from this FOG installation. + * + * Header-only, for the reason installTokenHeader() already sets out: a + * header cannot be set by a cross-site form, a link or an , and it + * never lands in browser history, a bookmark, a Referer or an access + * log. A query parameter would put a long-lived shared secret in every + * one of those. + * + * The signature covers path-and-query rather than the whole URL, so the + * http -> https redirect FOG's own vhost issues does not invalidate it. + * + * @param string $url The URL about to be requested. + * @param string $method The HTTP method that will be used. + * + * @return array Header lines, or an empty array when unavailable. + */ + public static function nodeSignatureHeaders($url, $method = 'GET') + { + $parts = parse_url((string)$url); + if ($parts === false) { + return array(); + } + // The peer's own key if it has one, otherwise the installation-wide + // key. Ordered this way round so a shared-database install -- where + // no ngmKey is ever set -- signs exactly as it did before, and a + // full FOG server registered as a peer gets a secret that is only + // good for talking to it. + $key = self::nodeSigningKeyFor( + isset($parts['host']) ? $parts['host'] : '' + ); + if ($key === '') { + $key = self::nodeApiKey(); + } + if ($key === '') { + return array(); + } + $uri = isset($parts['path']) ? $parts['path'] : '/'; + if (isset($parts['query']) && $parts['query'] !== '') { + $uri .= '?' . $parts['query']; + } + $timestamp = (string)time(); + $signature = hash_hmac( + 'sha256', + self::_nodeSignaturePayload( + strtoupper((string)$method), + $uri, + $timestamp + ), + $key + ); + return array( + 'X-Fog-Node-Timestamp: ' . $timestamp, + 'X-Fog-Node-Signature: ' . $signature + ); + } + /** + * Is this request signed by a FOG component that holds the node key? + * + * Authentication, not authorisation: it says the caller is part of this + * installation, and nothing about what it may do. Endpoints accepting it + * must still be ones a node is entitled to reach -- the same split + * service/nodecert.php makes when it checks the HMAC and then separately + * matches the source IP against a registered node. + * + * getSetting() rather than nodeApiKey() deliberately: verification must + * never mint a key. If no key exists there is nothing this request can + * have signed with, and the answer is no. + * + * @return bool + */ + public static function validNodeSignature() + { + $timestamp = isset($_SERVER['HTTP_X_FOG_NODE_TIMESTAMP']) + ? $_SERVER['HTTP_X_FOG_NODE_TIMESTAMP'] + : null; + $signature = isset($_SERVER['HTTP_X_FOG_NODE_SIGNATURE']) + ? $_SERVER['HTTP_X_FOG_NODE_SIGNATURE'] + : null; + if (!is_string($timestamp) + || !is_string($signature) + || $signature === '' + || !ctype_digit($timestamp) + ) { + return false; + } + if (abs(time() - (int)$timestamp) > self::NODE_SIGNATURE_WINDOW) { + return false; + } + $keys = self::_nodeVerificationKeys(); + if (count($keys) < 1) { + return false; + } + $method = isset($_SERVER['REQUEST_METHOD']) + ? $_SERVER['REQUEST_METHOD'] + : 'GET'; + $uri = isset($_SERVER['REQUEST_URI']) ? $_SERVER['REQUEST_URI'] : ''; + $payload = self::_nodeSignaturePayload( + strtoupper((string)$method), + (string)$uri, + $timestamp + ); + // Every candidate is compared, and the result is accumulated rather + // than returned early, so the time taken does not depend on WHICH + // key matched -- an early return would let a caller learn a node's + // position in the list by timing it. hash_equals is already constant + // time for the comparison itself; this keeps the loop from undoing + // that. + $matched = false; + foreach ($keys as $key) { + if (hash_equals(hash_hmac('sha256', $payload, $key), $signature)) { + $matched = true; + } + } + return $matched; + } /** * Is the acting user a FOG administrator (uType 0)? * @@ -3038,6 +3735,242 @@ public static function is_authorized($return_bool = false) exit; } } + /** + * Column type and nullability per table, read once per request. + * + * Lives on FOGBase rather than FOGController because the two write + * paths that need it are SIBLINGS, not parent and child: + * FOGController::save() writes one row, FOGManagerController:: + * insertBatch() writes many, and both extend FOGBase directly. It + * started on FOGController because save() was its only caller, and + * the cost of leaving it there was that GH-1245's fix reached one of + * the two paths -- which is how a strict server came to reject saving + * FOG settings while saving a host worked fine. + * + * Null until first asked for. + * + * @var array|null + */ + private static $columnTypes = null; + + /** + * Loads the column map from the server's own catalog. + * + * Read from information_schema rather than from a committed manifest: + * this branch has no commons/schema-expected.php and no SchemaReconciler, + * so the database itself is the only description of the current schema + * there is. One query per request, and only if something actually asks -- + * a save that supplies every field never gets here. + * + * A failure leaves the map empty, which makes emptyValueFor() answer '' + * for everything. That is exactly the behaviour that shipped before this + * change, so a server that will not answer the catalog query degrades to + * the old code path rather than to a broken one. + * + * @return void + */ + private static function _loadColumnTypes() + { + self::$columnTypes = array(); + try { + $rows = self::$DB->query( + "SELECT `TABLE_NAME` AS `t`, `COLUMN_NAME` AS `c`, " + . "`COLUMN_TYPE` AS `ty`, `IS_NULLABLE` AS `n`, " + . "`COLUMN_DEFAULT` AS `d`, `EXTRA` AS `e` " + . "FROM `information_schema`.`COLUMNS` " + . "WHERE `TABLE_SCHEMA` = DATABASE()" + )->fetch(PDO::FETCH_ASSOC, 'fetch_all')->get(); + } catch (Exception $e) { + $rows = array(); + self::logFault( + sprintf( + '%s: %s: %s, %s', + _('Column type lookup failed'), + _('Error'), + $e->getMessage(), + _('every column will be treated as untyped') + ) + ); + } + /* + * The degradation is deliberate, the SILENCE was not. PDODB swallows + * a rejected statement, so this never reached the catch above on a + * real error -- it cached an empty type map and every column went + * back to being untyped, which is exactly the bug this method exists + * to prevent, reappearing with nothing said. + * + * Behaviour is unchanged: still an empty map. Only now it is written + * down. + */ + if (self::$DB->error) { + self::logFault( + sprintf( + '%s: %s: %s, %s', + _('Column type lookup failed'), + _('Error'), + self::$DB->error, + _('every column will be treated as untyped') + ) + ); + $rows = array(); + } + foreach ((array)$rows as $row) { + if (!isset($row['t'], $row['c'], $row['ty'])) { + continue; + } + $nullable = isset($row['n']) && strtoupper($row['n']) === 'YES'; + $auto = isset($row['e']) + && false !== stripos($row['e'], 'auto_increment'); + self::$columnTypes[strtolower($row['t'])][strtolower($row['c'])] = array( + 'type' => trim($row['ty']), + 'nullable' => $nullable, + /* + * "An INSERT must name this column or the server rejects the + * row." True when it is NOT NULL, carries no DEFAULT, and is + * not AUTO_INCREMENT -- see columnsRequiringValue() below for + * why all three parts matter. Carried here rather than asked + * for separately because this query already visits every + * column of every table exactly once. + */ + 'required' => !$nullable + && !$auto + && (!isset($row['d']) || null === $row['d']), + ); + } + } + + /** + * The declared SQL type of a column, or '' when it is not known. + * + * @param string $table the database table + * @param string $column the database column + * + * @return string + */ + protected static function columnType($table, $column) + { + if (null === self::$columnTypes) { + self::_loadColumnTypes(); + } + $t = strtolower($table); + $c = strtolower($column); + return isset(self::$columnTypes[$t][$c]) + ? self::$columnTypes[$t][$c]['type'] + : ''; + } + + /** + * Can this column hold NULL? + * + * @param string $table the database table + * @param string $column the database column + * + * @return bool + */ + protected static function columnIsNullable($table, $column) + { + if (null === self::$columnTypes) { + self::_loadColumnTypes(); + } + $t = strtolower($table); + $c = strtolower($column); + return isset(self::$columnTypes[$t][$c]) + && self::$columnTypes[$t][$c]['nullable']; + } + + /** + * What an unset optional field should actually be written as. + * + * GH-1245. save() used to write '' for every unset optional field whose + * key does not end in "id". '' is a value only a string column can hold. + * Everywhere else the server either refuses it under a strict sql_mode or + * coerces it without one, and FOG only ever saw the second, because + * PDODB::_connect() cleared sql_mode on every connection. So this is not + * new behaviour being introduced -- it is the coercion the server was + * already performing, written down and made legal: + * + * date/time -> NULL (was '0000-00-00 00:00:00') + * integer -> 0 (was 0, via error 1366 downgraded) + * enum/set -> first member (was '', the error value at index 0) + * anything -> '' (unchanged; '' is a real value here) + * + * The integer and enum choices deliberately match the coercion rather + * than the column's DEFAULT. `hosts.hostEnforce` is declared + * DEFAULT '1' and rows across the field hold '' -- so honouring the + * default would silently turn enforcement ON for those hosts as a side + * effect of a storage fix. '' and '0' are both falsey in PHP, so the + * first enum member behaves as the error value already did. + * + * The column's TYPE is the only reliable way to tell these apart; the + * key's name is not, which is the lesson $databaseFieldsNotInt already + * exists for. + * + * @param string $table the database table + * @param string $column the database column + * + * @return mixed the value to write; null means a real SQL NULL + */ + protected static function emptyValueFor($table, $column) + { + $type = self::columnType($table, $column); + if ('' === $type) { + return ''; + } + if (preg_match('/^(datetime|timestamp|date)\b/i', $type)) { + return null; + } + if (preg_match('/^(tiny|small|medium|big)?int\b/i', $type)) { + return 0; + } + if (preg_match("/^(enum|set)\\s*\\(\\s*'((?:[^']|'')*)'/i", $type, $match)) { + return str_replace("''", "'", $match[2]); + } + + return ''; + } + /** + * Columns this table will not accept an INSERT without. + * + * A column qualifies when it is NOT NULL, carries no DEFAULT, and is not + * AUTO_INCREMENT. Under a strict sql_mode, omitting one of those from an + * INSERT is error 1364 -- "Field 'x' doesn't have a default value" -- and + * the row is rejected outright. Without a strict mode the server invents + * a zero value and says nothing, which is what FOG saw for nine years + * because PDODB cleared sql_mode on every connection. + * + * All three parts matter. NOT NULL alone is not enough: a column with a + * DEFAULT is happily omitted, and filling it would override the default + * the schema chose. Nor is "NOT NULL and no DEFAULT" enough: that also + * describes an AUTO_INCREMENT primary key, and filling `stID` with 0 + * would write over the value the server was about to generate. + * + * Answered from the map _loadColumnTypes() already builds, so this costs + * no extra query. A catalog that could not be read leaves the map empty, + * which reports nothing required -- the caller then builds exactly the + * statement it built before this existed, rather than a different one. + * + * @param string $table the database table + * + * @return array column name (lowercased) => declared SQL type + */ + protected static function columnsRequiringValue($table) + { + if (null === self::$columnTypes) { + self::_loadColumnTypes(); + } + $t = strtolower((string)$table); + $out = array(); + if (!isset(self::$columnTypes[$t])) { + return $out; + } + foreach (self::$columnTypes[$t] as $column => $meta) { + if (!empty($meta['required'])) { + $out[$column] = isset($meta['type']) ? $meta['type'] : ''; + } + } + return $out; + } + /** * Output var_dump for logging * diff --git a/packages/web/lib/fog/fogcontroller.class.php b/packages/web/lib/fog/fogcontroller.class.php index c5f291387c..fe1aa6dc7d 100644 --- a/packages/web/lib/fog/fogcontroller.class.php +++ b/packages/web/lib/fog/fogcontroller.class.php @@ -2,7 +2,7 @@ /** * FOGController, individual SQL getters/setters. * - * PHP Version 5 + * PHP version 7.4+ * * Gets and sets data for an individual object. * Generates the SQL Statements more specifically. @@ -57,6 +57,19 @@ abstract class FOGController extends FOGBase * @var array */ protected $databaseFieldsRequired = array(); + /** + * Keys that end in "id" but do not hold a foreign key. + * + * save() and isValid() both infer "this is an integer id" from the key's + * name, which is right for every real foreign key in the tree and wrong + * for a string identifier that happens to end the same way -- a system + * UUID, a task id kept in a text column. The name is a proxy for the + * column's type, and the model is the only thing that knows the actual + * type, so it says so here. + * + * @var array + */ + protected $databaseFieldsNotInt = array(); /** * Additional elements unrelated to DB side directly for object. * @@ -406,6 +419,13 @@ public function save() $required[$reqKeyNorm] = true; } + // Keys the model has declared are NOT foreign keys, normalized the + // same way, so the branch below can ask about $key directly. + $notInt = []; + foreach ($this->databaseFieldsNotInt as $strKey) { + $notInt[$this->key($strKey)] = true; + } + foreach ($this->databaseFields as $rawKey => $column) { $key = $this->key($rawKey); $column = trim($column); @@ -417,6 +437,10 @@ public function save() $eColumn = sprintf('`%s`', $column); $paramInsert = sprintf(':%s_insert', $column); + // GH-1245: set when the column is to be written as a real + // SQL NULL rather than left out of the statement. + $writeNull = false; + $val = $this->get($key); // Primary key 'id': allow null/empty/0 so DB auto-increments. @@ -429,8 +453,11 @@ public function save() $val = (int)$validId; } - // Keys ending with "id" (case-insensitive) - elseif (strtolower(substr($key, -2)) === 'id') { + // Keys ending with "id" (case-insensitive), unless the model + // has said this one is a string rather than a foreign key. + elseif (strtolower(substr($key, -2)) === 'id' + && !isset($notInt[$key]) + ) { $isRequired = isset($required[$key]); $isEmpty = ($val === null) || (is_string($val) && trim($val) === ''); @@ -460,7 +487,22 @@ public function save() if ($isRequired) { throw new Exception(self::$foglang['RequiredDB'] . ": " . $key); } - $val = ''; + // GH-1245: '' is a value only a string column can + // hold. Everywhere else the server was coercing it; + // emptyValueFor() writes down what to. + $val = self::emptyValueFor($this->databaseTable, $column); + /* + * A NULL for a column that cannot hold one means + * "leave it out and let the server's DEFAULT apply". + * Binding it is error 1048 -- snapinTasks + * .stCheckinDate and userTracking.utDateTime are + * NOT NULL DEFAULT current_timestamp(), which is why + * schema step 284 leaves them alone, and MySQL 8 ships + * explicit_defaults_for_timestamp=ON so an explicit + * NULL is refused rather than turned into "now". + */ + $writeNull = (null === $val) + && self::columnIsNullable($this->databaseTable, $column); } } @@ -490,7 +532,13 @@ public function save() // Don't make an entry if the value isn't set (null = truly unset). // Empty string is a valid user-supplied value and must be written. - if ($val === null) { + // + // GH-1245: an emptied DATE column is the exception. Omitting + // it would leave ON DUPLICATE KEY UPDATE with nothing to say + // about that column, so an existing date could never be + // cleared -- the write would report success and change + // nothing. It is bound as a real NULL instead. + if ($val === null && !$writeNull) { continue; } @@ -521,6 +569,31 @@ public function save() self::info($msg); self::$DB->query($query, [], $queryArray); + /* + * PDODB swallows a rejected statement, so ASK it. + * + * PDO runs in ERRMODE_EXCEPTION, but PDODB::query() catches the + * PDOException, records the message on ->error and returns + * normally; it rethrows only when $throwOnQueryError is true, + * which nothing sets and which must not be set globally -- it + * would turn every already-tolerated failure across the codebase + * into an uncaught 500 at once. + * + * So without this check the catch below never runs on a real SQL + * error. For a NEW row that was survivable by accident: insertId() + * comes back 0 and the "no valid ID was assigned" throw further + * down catches it. For an EXISTING row -- every progress update, + * every task state change, every inventory write against a known + * host -- there was nothing to catch on, so save() went on to log + * the SUCCESS message and return $this. `if (!$obj->save())` was + * not merely unrecorded on those paths, it was answered "fine". + * + * Truthy rather than `false !== ...`: PDODB declares $error with + * no default, so it is null until the first statement runs. + */ + if (self::$DB->error) { + throw new Exception((string) self::$DB->error); + } $lastInsertID = self::$DB->insertId(); // Force ID correctness: if we still don't have a valid ID, this wasn't created properly. @@ -588,14 +661,26 @@ public function save() } $msg = sprintf( - '%s: %s: %s, %s: %s', + '%s: %s: %s, %s: %s, %s: %s, %s: %s', _('Database save failed'), + _('Class'), + get_class($this), + _('Table'), + $this->databaseTable, _('ID'), $this->get('id'), _('Error'), $e->getMessage() ); self::debug($msg); + /* + * The line that actually gets written. debug() on this branch + * writes to no file at all and returns immediately on a service + * or ajax request, and logHistory() needs somebody signed in -- + * neither is true on the paths that generate most of these. + * See FOGBase::logFault(). + */ + self::logFault($msg); return false; } @@ -675,6 +760,51 @@ public function load($key = 'id') $queryArray ); $vals = self::$DB->fetch()->get(); + /* + * A rejected SELECT is swallowed the same way a rejected INSERT + * is -- see save(). fetch()->get() then hands back nothing, and + * an object that could not be read is indistinguishable from a + * row that genuinely holds no data. That is the read half of the + * same defect: not a wrong answer anybody can see, a plausible + * empty one. + * + * AFTER the fetch, not between it and the query, so that ONE + * check covers both halves of the read. fetch() records its own + * failure on ->error and never clears one, and query() always + * sets ->error immediately before -- so a fetch that failed + * because the query did still reports the query's message here, + * not "No query result, use query() first". + * + * Recorded HERE rather than in the catch below, and that split is + * the point. This catch also handles the method's ORDINARY + * control flow -- "Operation field not set" fires on every + * `new Host()` built without an id, which is constant traffic -- + * so faulting the whole catch would bury the one line that + * matters under thousands that do not. + * + * Throwing after logging costs nothing and buys the debug line + * below: setQuery() merges (fastmerge, never clears), so skipping + * it with nothing to merge leaves the object exactly as it was. + * load() still returns $this either way -- `new Host(42)` must + * not become fatal because a read failed. + */ + if (self::$DB->error) { + self::logFault( + sprintf( + '%s: %s: %s, %s: %s, %s: %s, %s: %s', + _('Database load failed'), + _('Class'), + get_class($this), + _('Table'), + $this->databaseTable, + _('Key'), + $key, + _('Error'), + self::$DB->error + ) + ); + throw new Exception((string) self::$DB->error); + } $this->setQuery($vals); } catch (Exception $e) { $str = sprintf( @@ -772,6 +902,13 @@ public function destroy($key = 'id') (array) $val ); self::$DB->query($query, array(), $queryArray); + // Same reason as save()'s, above: a rejected DELETE is swallowed + // by PDODB, so destroy() reported success for a row still there. + // A DELETE matching nothing is not an error and does not land + // here -- only a statement the server actually rejected does. + if (self::$DB->error) { + throw new Exception((string) self::$DB->error); + } if (!$this instanceof History) { if ($this->get('name')) { $msg = sprintf( @@ -822,14 +959,26 @@ public function destroy($key = 'id') self::logHistory($msg); } $msg = sprintf( - '%s: %s: %s, %s: %s', + '%s: %s: %s, %s: %s, %s: %s, %s: %s', _('Destroy failed'), + _('Class'), + get_class($this), + _('Table'), + $this->databaseTable, _('ID'), $this->get('id'), _('Error'), $e->getMessage() ); self::debug($msg); + /* + * The line that actually gets written. debug() on this branch + * writes to no file at all and returns immediately on a service + * or ajax request, and logHistory() needs somebody signed in -- + * neither is true on the paths that generate most of these. + * See FOGBase::logFault(). + */ + self::logFault($msg); return false; } @@ -947,12 +1096,23 @@ protected function addRemItem($key, $array, $array_type) public function isValid() { try { + // The same opt-out save() honors. Both methods carry their own + // copy of the "ends in id, so it is a foreign key" inference, so + // both need the exclusion: fixing only save() lets an object save + // its string identifier and then fail validation forever after. + $notInt = []; + foreach ($this->databaseFieldsNotInt as $strKey) { + $notInt[$this->key($strKey)] = true; + } + foreach ($this->databaseFieldsRequired as $reqKey) { $key = $this->key($reqKey); $val = $this->get($key); // If key ends with ID (case-insensitive), require integer >= 1 - if (strtolower(substr($key, -2)) === 'id') { + if (strtolower(substr($key, -2)) === 'id' + && !isset($notInt[$key]) + ) { if (filter_var($val, FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]) === false) { throw new Exception(self::$foglang['RequiredDB'] . ": " . $key); } @@ -1001,42 +1161,6 @@ public function buildQuery( $not = false, $compare = '=' ) { - /** - * Lambda function to build the where array additionals. - * - * @param string $field the field to work from - * @param mixed $value the value of the field - */ - $whereInfo = function ( - &$value, - $field - ) use ( - &$whereArrayAnd, - &$c, - $not, - $compare - ) { - if (is_array($value)) { - $whereArrayAnd[] = sprintf( - "`%s`.`%s` IN ('%s')", - $c->databaseTable, - $field, - implode("','", $value) - ); - } else { - if (strpos($value, '%')) { - $compare = 'LIKE'; - } - $whereArrayAnd[] = sprintf( - "`%s`.`%s` %s '%s'", - $c->databaseTable, - $c->databaseFields[$field], - $compare, - $value - ); - } - unset($value, $field); - }; /** * Lambda function to build the join of a query. * @@ -1050,25 +1174,48 @@ public function buildQuery( &$join, &$whereArrayAnd, &$c, - $whereInfo, $not, $compare ) { $className = strtolower($class); $c = self::getClass($class); if (!array_key_exists($className, $join)) { + // The relationship's optional 4th element is a filter on the + // joined (optional) table. It must live in the JOIN ON clause, + // not in WHERE: a WHERE condition on the right-hand table of a + // LEFT JOIN silently degrades it to an INNER JOIN, dropping the + // base row entirely when there is no matching joined row (e.g. + // a host with no primary MAC would fail to load at all). + $onExtra = ''; + if (isset($fields[3]) && $fields[3]) { + foreach ((array) $fields[3] as $filterField => $filterValue) { + if (is_array($filterValue)) { + $onExtra .= sprintf( + " AND `%s`.`%s` IN ('%s')", + $c->databaseTable, + $c->databaseFields[$filterField], + implode("','", $filterValue) + ); + } else { + $onExtra .= sprintf( + " AND `%s`.`%s` = '%s'", + $c->databaseTable, + $c->databaseFields[$filterField], + $filterValue + ); + } + } + } $join[$className] = sprintf( - ' LEFT OUTER JOIN `%s` ON `%s`.`%s`=`%s`.`%s` ', + ' LEFT OUTER JOIN `%s` ON `%s`.`%s`=`%s`.`%s`%s ', $c->databaseTable, $c->databaseTable, $c->databaseFields[$fields[0]], $this->databaseTable, - $this->databaseFields[$fields[1]] + $this->databaseFields[$fields[1]], + $onExtra ); } - if (isset($fields[3])) { - array_walk($fields[3], $whereInfo); - } $c->buildQuery($join, $whereArrayAnd, $c, $not, $compare); unset($class, $fields, $c); }; diff --git a/packages/web/lib/fog/fogcore.class.php b/packages/web/lib/fog/fogcore.class.php index 4073109c41..75c6cd3c03 100644 --- a/packages/web/lib/fog/fogcore.class.php +++ b/packages/web/lib/fog/fogcore.class.php @@ -2,7 +2,7 @@ /** * The core elements accessible for all else * - * PHP version 5 + * PHP version 7.4+ * * @category FOGCore * @package FOGProject diff --git a/packages/web/lib/fog/fogcron.class.php b/packages/web/lib/fog/fogcron.class.php index 6c804a3896..8c4e4af935 100644 --- a/packages/web/lib/fog/fogcron.class.php +++ b/packages/web/lib/fog/fogcron.class.php @@ -2,7 +2,7 @@ /** * The cron validation * - * PHP version 5 + * PHP version 7.4+ * * @category FOGCron * @package FOGProject diff --git a/packages/web/lib/fog/fogftp.class.php b/packages/web/lib/fog/fogftp.class.php index c018d8827e..8e13bf4fdd 100644 --- a/packages/web/lib/fog/fogftp.class.php +++ b/packages/web/lib/fog/fogftp.class.php @@ -2,7 +2,7 @@ /** * Handles FTP connections and operations for FOG * - * PHP version 5 + * PHP version 7.4+ * * @category FOGFTP * @package FOGProject @@ -65,6 +65,17 @@ class FOGFTP extends FOGGetSet * @var string */ private $_currentLoginHash; + /** + * Which stage of connect() failed, if one did. + * + * Either 'connect' (the socket never came up) or 'login' (the server + * answered and refused the credentials). connect() throws the same generic + * exception for both, so without this the caller cannot tell a network + * problem from a wrong password -- and every caller guessed "network". + * + * @var string + */ + private $_lastFailure = ''; /** * Destroy the ftp object * @@ -168,6 +179,7 @@ public function connect( $autologin = true, $connectmethod = 'ftp_connect' ) { + $this->_lastFailure = ''; try { $this->_currentConnectionHash = password_hash( print_r($this->data, 1), @@ -214,21 +226,33 @@ public function connect( $timeout = $this->get('timeout'); } } + $this->_lastFailure = 'connect'; $this->_link = $connectmethod($host, $port, $timeout); if ($this->_link === false) { trigger_error(_('FTP connection failed'), E_USER_NOTICE); $this->ftperror($this->data); } if ($autologin) { + $this->_lastFailure = 'login'; $this->login(); $this->pasv($this->get('passive')); } + $this->_lastFailure = ''; } catch (Exception $e) { throw new Exception($e->getMessage()); } $this->_lastConnectionHash = $this->_currentConnectionHash; return $this; } + /** + * Which stage of the last connect() failed. + * + * @return string 'connect', 'login', or '' when the last attempt succeeded + */ + public function lastFailure() + { + return $this->_lastFailure; + } /** * Deletes the item passed * diff --git a/packages/web/lib/fog/foggetset.class.php b/packages/web/lib/fog/foggetset.class.php index 020d35ce36..4b07f759f1 100644 --- a/packages/web/lib/fog/foggetset.class.php +++ b/packages/web/lib/fog/foggetset.class.php @@ -2,7 +2,7 @@ /** * Get/set container for other elements * - * PHP version 5 + * PHP version 7.4+ * * @category FOGGetSet * @package FOGProject diff --git a/packages/web/lib/fog/fogmanagercontroller.class.php b/packages/web/lib/fog/fogmanagercontroller.class.php index dfbb6ccc4f..0e9af5caf8 100644 --- a/packages/web/lib/fog/fogmanagercontroller.class.php +++ b/packages/web/lib/fog/fogmanagercontroller.class.php @@ -2,7 +2,7 @@ /** * FOG Manager Controller, main object mass getter. * - * PHP version 5 + * PHP version 7.4+ * * @category FOGManagerController * @package FOGProject @@ -57,6 +57,17 @@ abstract class FOGManagerController extends FOGBase * @var array */ protected $databaseFieldClassRelationships = array(); + /** + * Fields whose name ends in "id" but which are not foreign keys. + * + * Mirrored from the model for the same reason isValid() carries its own + * copy of the rule save() uses: both write paths infer "ends in id, so it + * is a foreign key", so both need the model's opt-out or one of them + * refuses a string identifier the other accepts. + * + * @var array + */ + protected $databaseFieldsNotInt = array(); /** * The additional fields. * @@ -129,12 +140,14 @@ public function __construct() 'additionalFields', 'databaseFieldsRequired', 'databaseFieldClassRelationships', + 'databaseFieldsNotInt', ); $this->databaseTable = &$classVars[$classGet[0]]; $this->databaseFields = &$classVars[$classGet[1]]; $this->additionalFields = &$classVars[$classGet[2]]; $this->databaseFieldsRequired = &$classVars[$classGet[3]]; $this->databaseFieldClassRelationships = &$classVars[$classGet[4]]; + $this->databaseFieldsNotInt = &$classVars[$classGet[5]]; $this->databaseFieldsFlipped = array_flip($this->databaseFields); unset($classGet); } @@ -151,6 +164,7 @@ public function __construct() * @param mixed $idField what fields to get * @param bool $onecompare second where uses AND * @param string $filter array function for filter + * @param string $scopeWhere an object-boundary SQL fragment to AND on * * @return array */ @@ -164,7 +178,8 @@ public function find( $not = false, $idField = false, $onecompare = true, - $filter = 'array_unique' + $filter = 'array_unique', + $scopeWhere = '' ) { // Fail safe defaults if (empty($findWhere)) { @@ -191,7 +206,27 @@ public function find( $count = 0; foreach ($findWhere as $field => &$value) { $key = trim($field); - if (!$value) { + /* + * GH-1245 gave find() the `null === $value` branch below, but + * left this expansion above it -- and `!null` is true, so an + * explicit null was turned into the array first and that + * branch could never run. The emitted term was + * `col IN ('0',0,NULL,'')`, which is never TRUE for a column + * holding NULL: SQL evaluates `NULL IN (...)` to unknown. So + * every `find()` filtering on null silently matched nothing. + * + * TaskingElement::imageLog() is where it showed: it looks up + * the open imaging log by `finish => null`, missed it on every + * single deployment, and the caller reported "Failed to update + * imaging log" for a machine that had imaged perfectly (forums + * topic 18228). + * + * Only the null case is taken out of the expansion. A filter + * holding 0, '' or false keeps matching the falsey stored + * representations exactly as before -- that list has hundreds + * of callers and this is not the change to alter them in. + */ + if (null !== $value && !$value) { $value = array( '0', 0, @@ -201,9 +236,7 @@ public function find( } if (is_array($value) && count($value) > 0) { foreach ($value as $i => &$val) { - if (is_string($val)) { - $val = trim($val); - } + $val = self::_trimValue($val); // Define the key $k = sprintf( '%s_%d', @@ -227,11 +260,30 @@ public function find( implode(',', $findKeys) ); unset($findKeys); + } elseif (null === $value) { + /* + * GH-1245: a null filter asks for rows where the column + * holds nothing. Bound as a placeholder it becomes + * `col = NULL`, which is never true, so the query + * silently returns nothing -- and it now has callers, + * because the date columns that used to carry + * '0000-00-00 00:00:00' as their "not yet" sentinel hold + * NULL from schema step 284 on. + */ + $whereArray[] = sprintf( + '`%s`.`%s` IS%sNULL', + $this->databaseTable, + $this->databaseFields[$field], + (trim($not) ? ' NOT ' : ' ') + ); } else { if (is_array($value)) { $value = ''; } - $value = trim($value); + // Read side, same rule as the write side: a filter + // holding false has to bind the same literal the column + // now stores, or it silently matches nothing. + $value = self::_trimValue($value); $k = sprintf( '%s', $key @@ -349,6 +401,64 @@ public function find( $idFields = array_filter($idFields); $idField = $idFields; unset($idFields); + $whereClause = ( + count($whereArray) > 0 ? + sprintf( + ' WHERE %s%s', + implode(" $whereOperator ", (array) $whereArray), + ( + $isEnabled ? + sprintf(' AND %s', $isEnabled) : + '' + ) + ) : + ( + $isEnabled ? + sprintf(' WHERE %s', $isEnabled) : + '' + ) + ); + $andClause = ( + count($whereArrayAnd) > 0 ? + ( + count($whereArray) > 0 ? + sprintf( + 'AND %s', + implode(" $whereOperator ", (array) $whereArrayAnd) + ) : + sprintf( + ' WHERE %s', + implode(" $whereOperator ", (array) $whereArrayAnd) + ) + ) : + '' + ); + // The object boundary, when the caller was given one to apply. + // + // Two properties this has to hold that the obvious splice does not. + // It is ANDed on LAST, after everything the caller asked for, with + // the caller's own terms parenthesised: $whereOperator is a parameter + // and 'OR' is a value it takes, so a term merged in beside the + // caller's could be satisfied INSTEAD of the boundary rather than as + // well as it. And it is joined with a literal ' AND ', never through + // $whereOperator, for the same reason. An OR that can reach outside + // the boundary is not a boundary. + // + // Empty means no boundary, which is every caller that does not pass + // this argument. A caller that means "you may see nothing" passes a + // fragment saying so, such as '1=0' -- NOT an empty string, which + // reads here as unrestricted and would hand back the whole table. + $scopeWhere = trim((string)$scopeWhere); + if ('' !== $scopeWhere) { + $inner = trim($whereClause . ' ' . $andClause); + $inner = preg_replace('#^WHERE\s+#i', '', $inner); + $whereClause = ( + '' === $inner ? + sprintf(' WHERE %s', $scopeWhere) : + sprintf(' WHERE (%s) AND (%s)', $inner, $scopeWhere) + ); + $andClause = ''; + } $query = sprintf( $this->loadQueryTemplate, ( @@ -358,38 +468,8 @@ public function find( ), $this->databaseTable, $join, - ( - count($whereArray) > 0 ? - sprintf( - ' WHERE %s%s', - implode(" $whereOperator ", (array) $whereArray), - ( - $isEnabled ? - sprintf(' AND %s', $isEnabled) : - '' - ) - ) : - ( - $isEnabled ? - sprintf(' WHERE %s', $isEnabled) : - '' - ) - ), - ( - count($whereArrayAnd) > 0 ? - ( - count($whereArray) > 0 ? - sprintf( - 'AND %s', - implode(" $whereOperator ", (array) $whereArrayAnd) - ) : - sprintf( - ' WHERE %s', - implode(" $whereOperator ", (array) $whereArrayAnd) - ) - ) : - '' - ), + $whereClause, + $andClause, $groupBy, $orderBy ); @@ -402,6 +482,23 @@ public function find( PDO::FETCH_ASSOC, 'fetch_all' ); + // A rejected read answers an EMPTY set, which every caller reads as + // "there are none" rather than "the question was not asked". The + // return contract is left alone -- there is no catch here and callers + // expect an array -- so the fault line is the whole of the fix. + if (self::$DB->error) { + self::logFault( + sprintf( + '%s: %s: %s, %s: %s, %s', + _('Find failed'), + _('Table'), + $this->databaseTable, + _('Error'), + self::$DB->error, + _('answering an empty set for a read that never ran') + ) + ); + } if ($idField) { $data = (array)self::$DB->get($idField); if ($filter) { @@ -468,6 +565,20 @@ public function count( ); } unset($countKeys); + } elseif (null === $value) { + /* + * GH-1245: a null filter asks for rows where the column + * holds nothing. Bound as a placeholder it becomes + * `col = NULL`, which is never true, so the query + * silently returns nothing -- and it now has callers, + * because the date columns that used to carry + * '0000-00-00 00:00:00' as their "not yet" sentinel hold + * NULL from schema step 284 on. + */ + $whereArray[] = sprintf( + '`%s` IS NULL', + $this->databaseFields[$field] + ); } else { if (is_array($value)) { $value = ''; @@ -542,10 +653,147 @@ public function count( ) ); - return (int)self::$DB - ->query($query, array(), $countVals) - ->fetch() - ->get('total'); + self::$DB->query($query, array(), $countVals); + $total = self::$DB->fetch()->get('total'); + // A rejected count answers 0, which reads as "there are none" rather + // than "nobody asked". After the fetch, so one check covers both + // halves. Contract unchanged; the fault line is the fix. + if (self::$DB->error) { + self::logFault( + sprintf( + '%s: %s: %s, %s: %s, %s', + _('Count failed'), + _('Table'), + $this->databaseTable, + _('Error'), + self::$DB->error, + _('answering 0 for a read that never ran') + ) + ); + } + + return (int)$total; + } + /** + * Trims a value on its way into a bound parameter, and leaves anything + * that is not a string alone. + * + * Trimming is a string operation, but `trim()` casts first, and the cast + * is where the information goes. trim(null) is '' -- and a PHP 8.1 + * deprecation -- which would put the zero date back into a column being + * cleared. trim(false) is also '', which is not how any column in the + * schema spells false: enum('0','1') rejects it outright under + * STRICT_TRANS_TABLES, and so does the tinyint(1) `hosts`.`hostInfoLock` + * that ends every imaging task via ->set('tokenlock', false). And an + * array -- a nested IN () list is one -- is a TypeError on PHP 8. + * + * A boolean is left as a boolean and normalised once, in PDODB::_bind(), + * so save() and the builders here cannot disagree about what false + * stores. See GH-1245 and forum topic 18227. + * + * @param mixed $value the value being bound + * + * @return mixed + */ + private static function _trimValue($value) + { + return is_string($value) ? trim($value) : $value; + } + /** + * Refuses a batch row whose required foreign key points at nothing. + * + * FOGController::save() will not write a row whose required *ID field is + * not an integer >= 1 -- see its "Required *id must be integer >= 1" + * branch. insertBatch() enforced nothing, so the same model validated + * itself when written one row at a time and did not when written a + * hundred at a time. + * + * What gets through is a 0. Since GH-1245 the server's own sql_mode + * rejects a null or an '' bound into a NOT NULL int, but 0 is a perfectly + * legal integer and no layer has an opinion about it -- and a caller + * indexing a positional list past its end, or reading an id off an object + * that did not load, produces exactly that. + * + * In `tasks` such a row is permanent and invisible at the same time. + * taskStateID still resolves, so the task counts as active for every + * "is this live" test in the tree; taskHostID/taskImageID/taskTypeID + * match nothing, and because the Active Tasks list renders from + * buildQuery()'s LEFT OUTER JOINs those columns come back NULL. The row + * shows as "() -" for host and image with no type icon, cannot be + * completed by any host, and nothing ever reaps it. Reported as "null + * tasks" in forum topics 18228 and 18230. + * + * Deliberately narrow, because this is a write path with many call sites: + * + * - Only columns the caller NAMED. A required column the batch is silent + * about is left to columnsRequiringValue() below, which is the + * behaviour every one of those call sites already relies on; turning + * silence into an error is a different change with a different blast + * radius. + * - Only *ID columns. They are the ones whose zero is indistinguishable + * from a value; a required string is caught by the server or by the + * reader either way. + * - Never the model's own primary key. No batch caller supplies it, and + * save() skips it for the same reason. + * - Never a key the model has declared is a string via + * $databaseFieldsNotInt. + * + * @param array $fields the friendly field names the caller named + * @param array $values the rows, positional against $fields + * + * @throws Exception + * + * @return void + */ + private function _assertBatchForeignKeys($fields, $values) + { + $notInt = array_map( + 'strtolower', + (array)$this->databaseFieldsNotInt + ); + $positions = array(); + foreach ((array)$this->databaseFieldsRequired as $friendly) { + $lower = strtolower($friendly); + if ('id' === $lower + || 'id' !== substr($lower, -2) + || in_array($lower, $notInt, true) + ) { + continue; + } + foreach ((array)$fields as $i => $named) { + if (strtolower($named) === $lower) { + $positions[$i] = $friendly; + } + } + } + if (count($positions) < 1) { + return; + } + foreach ((array)$values as $rowIndex => $row) { + foreach ($positions as $i => $friendly) { + $val = isset($row[$i]) ? $row[$i] : null; + $valid = filter_var( + $val, + FILTER_VALIDATE_INT, + array('options' => array('min_range' => 1)) + ); + if (false !== $valid) { + continue; + } + throw new Exception( + sprintf( + '%s: `%s`.%s %s %d, %s: %s', + self::$foglang['RequiredDB'], + $this->databaseTable, + $friendly, + _('in batch row'), + $rowIndex, + _('got'), + var_export($val, true) + ) + ); + } + } } /** * Inserts data in mass to the database. @@ -565,6 +813,9 @@ public function insertBatch($fields, $values) if ($valuelength < 1) { throw new Exception(_('No values passed')); } + // Before the loop below, which rewrites $fields from friendly names + // to column names in place. + $this->_assertBatchForeignKeys($fields, $values); $keys = array(); foreach ((array) $fields as &$key) { $key = $this->databaseFields[$key]; @@ -576,11 +827,62 @@ public function insertBatch($fields, $values) ); unset($key); } + /* + * GH-1245 again, on the other write path. + * + * A caller names the columns it has a value for, which is not the + * same set as the columns the server will accept an INSERT without. + * Under a strict sql_mode a NOT NULL column with no DEFAULT that the + * statement does not name is error 1364 and the whole batch is + * rejected; without one the server invents a zero value and says + * nothing. PDODB cleared sql_mode until GH-1245, so every such call + * site had been relying on the second behaviour without knowing it -- + * saving FOG settings omits settingDesc and settingCategory, and + * tasking a group's snapins omits stReturnCode and stReturnDetails. + * + * So write the coercion down instead of relying on it, exactly as + * FOGController::save() now does for a single row. The values come + * from the same emptyValueFor(), which is why it moved to FOGBase. + * + * They are deliberately NOT added to the ON DUPLICATE KEY UPDATE + * list: this fills a column the caller had nothing to say about, so + * on a row that already exists the stored value must stand. Filling + * settingDesc into the update list would blank the description of + * every setting on the page the moment anyone pressed save. + * + * The filled columns and their values are worked out ONCE here; the + * placeholders that carry them are named PER ROW, down in the loop. + * They were named once too, and the single `:_fill_0` was then + * repeated in every VALUES tuple -- which one row survives and two do + * not: PDODB sets PDO::ATTR_EMULATE_PREPARES => false, and a real + * server-side prepare answers SQLSTATE[HY093] "Invalid parameter + * number" to a named parameter used twice. So every batch of two or + * more rows into a table with an unnamed NOT NULL column failed + * outright, silently to the user: tasking a GROUP of two hosts with + * anything that is not a deploy or a multicast (wipe, virus scan, + * hardware inventory, password reset, snapins) names none of + * `tasks`' NFS/image columns and so hit this every time. + * See background_scripts/prove_batch_fill_duplicate_bind.php. + */ + $fillCols = array(); + $named = array_map('strtolower', $keys); + foreach ((array) self::columnsRequiringValue( + $this->databaseTable + ) as $column => $type) { + if (in_array(strtolower($column), $named, true)) { + continue; + } + $keys[] = $column; + $fillCols[] = self::emptyValueFor( + $this->databaseTable, + $column + ); + } $affectedRows = 0; $vals = array(); - $insertVals = array(); $values = array_chunk($values, 500); foreach ((array) $values as $ind => &$v) { + $insertVals = array(); foreach ((array) $v as $index => &$value) { $insertKeys = array(); foreach ((array) $value as $i => &$val) { @@ -593,11 +895,26 @@ public function insertBatch($fields, $values) ':%s', $key ); - $val = trim($val); + $val = self::_trimValue($val); $insertVals[$key] = $val; unset($val); } - $vals[] = sprintf('(%s)', implode(',', (array) $insertKeys)); + foreach ($fillCols as $fillIndex => $fillVal) { + $key = sprintf( + '_fill_%d_%d', + $fillIndex, + $index + ); + $insertKeys[] = sprintf( + ':%s', + $key + ); + $insertVals[$key] = $fillVal; + } + $vals[] = sprintf( + '(%s)', + implode(',', (array) $insertKeys) + ); unset($value); } if (count($vals) < 1) { @@ -611,6 +928,12 @@ public function insertBatch($fields, $values) implode(',', $dups) ); self::$DB->query($query, array(), $insertVals); + // Same swallowed-error seam as FOGController::save(): without + // this the loop went on to report affectedRows for a batch the + // server rejected. + if (self::$DB->error) { + throw new Exception((string) self::$DB->error); + } if ($ind === 0) { $insertID = (int) self::$DB->insertId(); } @@ -672,7 +995,10 @@ private function perform_update($findWhere, $whereOperator, $insertData) $updateVals = array(); foreach ((array) $insertData as $field => &$value) { $field = trim($field); - $value = trim($value); + // GH-1245: null is a value to write, not a string to trim. + // trim(null) is '' -- and a PHP 8.1 deprecation -- which would + // put the zero date back into a column being cleared. + $value = self::_trimValue($value); $updateKey = sprintf( ':update_%s', $field @@ -697,7 +1023,7 @@ private function perform_update($findWhere, $whereOperator, $insertData) $key = trim($field); if (is_array($value) && count($value) > 0) { foreach ($value as $i => &$val) { - $val = trim($val); + $val = self::_trimValue($val); // Define the key $k = sprintf( '%s_%d', @@ -720,11 +1046,29 @@ private function perform_update($findWhere, $whereOperator, $insertData) implode(',', $findKeys) ); unset($findKeys); + } elseif (null === $value) { + /* + * GH-1245: a null filter asks for rows where the column + * holds nothing. Bound as a placeholder it becomes + * `col = NULL`, which is never true, so the query + * silently returns nothing -- and it now has callers, + * because the date columns that used to carry + * '0000-00-00 00:00:00' as their "not yet" sentinel hold + * NULL from schema step 284 on. + */ + $whereArray[] = sprintf( + '`%s`.`%s` IS NULL', + $this->databaseTable, + $this->databaseFields[$field] + ); } else { if (is_array($value)) { $value = ''; } - $value = trim($value); + // Read side, same rule as the write side: a filter + // holding false has to bind the same literal the column + // now stores, or it silently matches nothing. + $value = self::_trimValue($value); $k = sprintf( '%s', $key @@ -770,7 +1114,33 @@ private function perform_update($findWhere, $whereOperator, $insertData) (array) $findVals ); - return (bool) self::$DB->query($query, array(), $queryVals); + self::$DB->query($query, array(), $queryVals); + /* + * `(bool) self::$DB->query(...)` was ALWAYS true: query() returns + * $this, and an object casts to true whatever the server said. So + * this reported success for every rejected mass update. + * + * Faulted here rather than thrown: this method has no catch and its + * callers expect a bool, so throwing would turn a silently-failed + * bulk edit into an uncaught 500. False is the honest answer they + * were already written to read. + */ + if (self::$DB->error) { + self::logFault( + sprintf( + '%s: %s: %s, %s: %s', + _('Mass update failed'), + _('Table'), + $this->databaseTable, + _('Error'), + self::$DB->error + ) + ); + + return false; + } + + return true; } /** * Destroys items related to the main object. @@ -839,6 +1209,24 @@ public function destroy( ); unset($destroyKeys); self::$DB->query($query, array(), $destroyVals); + // Returned true unconditionally, so a rejected DELETE reported + // every row removed. Faulted and answered false rather than + // thrown, for the same reason update() is: no catch here, and + // callers expect a bool. + if (self::$DB->error) { + self::logFault( + sprintf( + '%s: %s: %s, %s: %s', + _('Mass destroy failed'), + _('Table'), + $this->databaseTable, + _('Error'), + self::$DB->error + ) + ); + + return false; + } unset($destroyVals, $destroyKeys); } @@ -989,10 +1377,37 @@ public function exists( ':id' ); - return (bool)self::$DB - ->query($query, array(), $existVals) - ->fetch() - ->get('total') > 0; + self::$DB->query($query, array(), $existVals); + $total = self::$DB->fetch()->get('total'); + /* + * After the fetch, so one check covers both halves of the read -- + * fetch() records its own failure on ->error and never clears one. + * + * A rejected read here answers "no, it does not exist", which is the + * most expensive wrong answer this class can give: callers use + * exists() to decide whether to CREATE, so an unreadable database + * turns into a duplicate rather than an error. + * + * The contract is left alone -- callers expect a bool and there is no + * catch here -- so the fault line is the whole of the fix. Making + * this throw is a real change to a read contract and belongs in its + * own decision, not smuggled into a logging fix. + */ + if (self::$DB->error) { + self::logFault( + sprintf( + '%s: %s: %s, %s: %s, %s', + _('Existence check failed'), + _('Table'), + $this->databaseTable, + _('Error'), + self::$DB->error, + _('answering "does not exist" for a read that never ran') + ) + ); + } + + return (bool)$total > 0; } /** * Search for items passed to keyword. @@ -1002,7 +1417,7 @@ public function exists( * * @return mixe */ - public function search($keyword = '', $returnObjects = false) + public function search($keyword = '', $returnObjects = false, $scopeWhere = '') { $keyword = trim($keyword); if (!$keyword) { @@ -1030,7 +1445,19 @@ public function search($keyword = '', $returnObjects = false) ) ); if (empty($keyword) || $keyword === '%') { - return $this->find(); + return $this->find( + array(), + 'AND', + 'name', + 'ASC', + '=', + false, + false, + false, + true, + 'array_unique', + $scopeWhere + ); } $keyword = preg_replace( '#[%\+\s\+]#', @@ -1330,7 +1757,19 @@ public function search($keyword = '', $returnObjects = false) array('id' => $itemIDs) ); if ($returnObjects) { - return $this->find(array('id' => $itemIDs)); + return $this->find( + array('id' => $itemIDs), + 'AND', + 'name', + 'ASC', + '=', + false, + false, + false, + true, + 'array_unique', + $scopeWhere + ); } return $itemIDs; @@ -1393,6 +1832,15 @@ function ( $this->databaseFields[$field], implode(',', $inKeys) ); + } elseif (null === $value) { + // GH-1245: as in find() above -- a null filter means + // "the column holds nothing", which is `IS NULL`, not + // a bound `= NULL` that matches no row at all. + $whereArray[] = sprintf( + '`%s`.`%s` IS NULL', + $this->databaseTable, + $this->databaseFields[$field] + ); } else { if (is_array($value)) { $value = ''; @@ -1439,10 +1887,138 @@ function ( ) ); - return (int)self::$DB - ->query($query, array(), $countVals) - ->fetch() - ->get('total'); + self::$DB->query($query, array(), $countVals); + $total = self::$DB->fetch()->get('total'); + // Same as exists(): a rejected distinct count answers 0. After the + // fetch, so one check covers both halves. + if (self::$DB->error) { + self::logFault( + sprintf( + '%s: %s: %s, %s: %s, %s', + _('Count failed'), + _('Table'), + $this->databaseTable, + _('Error'), + self::$DB->error, + _('answering 0 for a read that never ran') + ) + ); + } + + return (int)$total; + } + /** + * Builds the CREATE TABLE for this manager's table, with a default on + * every column that is optional. + * + * GH-1245. Schema::createTable() emits `NOT NULL` with no DEFAULT for + * almost everything a caller does not spell out, which is the same defect + * schema step 286 repairs on an existing install -- except that install() + * calls uninstall() first, and uninstall() DROPS the table. So a plugin + * being installed, or reinstalled, put the bare columns straight back and + * the step could not help: 72 of the 99 columns across the plugin tables + * came back mandatory, and under the server's own sql_mode any INSERT + * omitting one fails with error 1364. + * + * WHICH COLUMNS KEEP THEIR TEETH. Not a judgement call, and not a list + * kept by hand -- FOG already states it, and this class already holds the + * statement: $databaseFieldsRequired, resolved up the model's inheritance + * chain by the constructor. Three kinds of column are left bare: + * + * - the primary key and the auto-increment column; + * - anything the model declares required; + * - anything whose name ends in ID, because an INSERT that forgets the + * row it hangs off should fail rather than make a silent orphan. + * Deliberately not gated on an integer type: taskLog.taskID is a + * mediumtext and is no less a foreign key for it. + * + * That is deliberately the SAME rule schema step 286 applies, so a table + * created by a plugin install and a table migrated by the step say the + * same thing. Two installs of the same FOG should not have two different + * schemas. + * + * A default the caller passed explicitly always wins; this only fills in + * where there was nothing. + * + * The signature mirrors Schema::createTable() exactly so a call site + * changes by one token. + * + * @param string $name What are we calling the table? + * @param bool $exists If not exists? + * @param array $fields The fields and names. + * @param array $types The types for the fields. + * @param array $nulls Which fields to have null or not. + * @param array $default Default values for field(s). + * @param array $unique The unique fields. + * @param string $engine The db engine for the table. + * @param string $charset The charset to use for the table. + * @param string $prime The primary field, if one. + * @param string $autoin The auto increment field. + * + * @return string + */ + public function createTableSql( + $name, + $exists, + $fields, + $types, + $nulls, + $default, + $unique, + $engine = 'InnoDB', + $charset = 'utf8', + $prime = '', + $autoin = '' + ) { + $keep = array(); + foreach ((array)$this->databaseFieldsRequired as $friendly) { + if (isset($this->databaseFields[$friendly])) { + $keep[strtolower($this->databaseFields[$friendly])] = true; + } + } + if ($prime) { + $keep[strtolower($prime)] = true; + } + if ($autoin) { + $keep[strtolower($autoin)] = true; + } + foreach ((array)$fields as $i => $field) { + $notNull = isset($nulls[$i]) && $nulls[$i] === false; + $hasDefault = isset($default[$i]) + && false !== $default[$i] + && null !== $default[$i] + && '' !== $default[$i]; + if (!$notNull + || $hasDefault + || isset($keep[strtolower($field)]) + || preg_match('/ID$/', $field) + ) { + continue; + } + $fill = Schema::emptyDefaultFor($types[$i]); + if (null === $fill) { + // A TEXT or BLOB column on a server too old to carry a + // default for one. Nothing to do and nothing broken by + // leaving it: save() writes the column explicitly and + // insertBatch() backfills it. + continue; + } + $default[$i] = $fill; + } + + return Schema::createTable( + $name, + $exists, + $fields, + $types, + $nulls, + $default, + $unique, + $engine, + $charset, + $prime, + $autoin + ); } /** * Uninstalls the table. @@ -1452,6 +2028,25 @@ function ( public function uninstall() { $sql = Schema::dropTable($this->tablename); - return self::$DB->query($sql); + self::$DB->query($sql); + // Declared @return bool and returned the PDODB object, which is + // truthy however the DROP went. A plugin uninstall that left its + // table in place reported success. + if (self::$DB->error) { + self::logFault( + sprintf( + '%s: %s: %s, %s: %s', + _('Table uninstall failed'), + _('Table'), + $this->tablename, + _('Error'), + self::$DB->error + ) + ); + + return false; + } + + return true; } } diff --git a/packages/web/lib/fog/fogpage.class.php b/packages/web/lib/fog/fogpage.class.php index 23dcf495b0..a25923cf7f 100644 --- a/packages/web/lib/fog/fogpage.class.php +++ b/packages/web/lib/fog/fogpage.class.php @@ -3,7 +3,7 @@ * Presents many defaults for the pages and is * the calling point by all other page items. * - * PHP version 5 + * PHP version 7.4+ * * @category FOGPage * @package FOGProject @@ -244,6 +244,112 @@ public static function webrootPath($webroot = null) * @var function */ protected static $returnData; + /** + * The "List All X" / "Create New X" pair for one node, written out. + * + * GH-435. These used to be built by sprintf()ing a translated noun into a + * translated format string -- `List All %s` and `Create New %s` -- and + * that cannot be translated correctly, in any language that inflects. + * + * French was the report: `Creer un nouveau %s` is masculine, so `machine` + * and `image` (both feminine) need `une nouvelle`, and `utilisateur` needs + * `nouvel` before its vowel. One format string cannot be all three. German + * inflects the adjective the same way -- `Neue %s erstellen` should be + * `Neuen Benutzer erstellen` for a masculine noun. + * + * The plural was broken independently of gender, and worse here than on + * working-1.6: the list label appended a literal `s` to $this->childClass, + * which was never translated at all, so every locale read a half-English + * label like "Alle Hosts auflisten". Japanese marks no plural, German + * `Rechner` is its own plural, and French nouns in -al take -aux. + * + * Whole phrases fix both at once and cost nothing anywhere else: each is + * an ordinary literal xgettext extracts, and a translator sees the entire + * sentence rather than a fragment with a hole in it. + * + * Keyed on childClass rather than on the node, because that is the value + * the composed form used and it already folds the storage special cases + * (node `storage` becomes StorageNode or StorageGroup above). + * + * Nodes NOT listed here fall back to the composed form, which is what + * plugins get: a plugin's class name is not knowable from here, and a + * plugin can ship its own catalog. The fallback is no worse for them than + * it was before this change. + * + * @param string $childClass the page's child class name + * + * @return array empty when the class has no written-out pair + */ + private static function _nodeMenuStrings($childClass) + { + switch ($childClass) { + case 'Group': + return array('list' => _('List All Groups'), + 'add' => _('Create New Group')); + case 'Host': + return array('list' => _('List All Hosts'), + 'add' => _('Create New Host')); + case 'Image': + return array('list' => _('List All Images'), + 'add' => _('Create New Image')); + case 'Printer': + return array('list' => _('List All Printers'), + 'add' => _('Create New Printer')); + case 'Snapin': + return array('list' => _('List All Snapins'), + 'add' => _('Create New Snapin')); + case 'StorageGroup': + return array('list' => _('List All Storage Groups'), + 'add' => _('Create New Storage Group')); + case 'StorageNode': + return array('list' => _('List All Storage Nodes'), + 'add' => _('Create New Storage Node')); + case 'User': + return array('list' => _('List All Users'), + 'add' => _('Create New User')); + } + return array(); + } + + /** + * sprintf() for a format string that came out of a translation catalog. + * + * The catalog is edited by translators, so this format string is not under + * the codebase's control -- and a bad one fails DIFFERENTLY depending on + * the PHP version, which is why both arms below are needed: + * + * PHP 8 sprintf('Lister 100% des %s', $n) ArgumentCountError + * sprintf('List %q of %s', $n) ValueError + * PHP 7 both of the above warning, returns false + * + * So on 8 an uncaught one takes the whole navigation menu out with a 500 + * on every page, and on 7 nothing throws at all and `false` flows on to be + * rendered as an empty menu label. A Throwable catch alone would silently + * leave the 7.x half broken. + * + * Falling back to the untranslated argument keeps the menu rendering. It + * is not a good label, but it is a legible one, and it degrades in the one + * language whose catalog is at fault rather than everywhere. + * + * @param string $format translated format string + * @param string $value already-translated noun to substitute + * + * @return string + */ + private static function _composeMenuLabel($format, $value) + { + try { + $out = sprintf((string)$format, $value); + } catch (\Throwable $e) { + return (string)$value; + } + // The cast is the PHP 7 arm, not decoration: there sprintf() returns + // FALSE rather than throwing, and (string)false is ''. Comparing to '' + // rather than to false covers both that and a catalog entry that is + // simply empty. + return '' === (string)$out ? (string)$value : (string)$out; + } + /** * Initializes the page class * @@ -435,20 +541,29 @@ public function __construct($name = '') ); $exportMenu = sprintf('Export%s', $this->childClass); $importMenu = sprintf('Import%s', $this->childClass); + $pair = self::_nodeMenuStrings($this->childClass); + if (!count($pair)) { + $pair = array( + /** + * No _() around the sprintf: a msgid built at runtime can + * never match the literal xgettext extracted, so the outer + * call was a guaranteed miss that returned its own argument. + * Dropping it changes nothing at runtime and stops the line + * claiming to be translatable. + */ + 'list' => self::_composeMenuLabel( + self::$foglang['ListAll'], + sprintf('%ss', $this->childClass) + ), + 'add' => self::_composeMenuLabel( + self::$foglang['CreateNew'], + _($this->childClass) + ), + ); + } $this->menu = array( - 'list' => sprintf( - self::$foglang['ListAll'], - _( - sprintf( - '%ss', - $this->childClass - ) - ) - ), - 'add' => sprintf( - self::$foglang['CreateNew'], - _($this->childClass) - ), + 'list' => $pair['list'], + 'add' => $pair['add'], 'export' => isset(self::$foglang[$exportMenu]) ? sprintf(self::$foglang[$exportMenu]) : '', 'import' => isset(self::$foglang[$importMenu]) ? sprintf(self::$foglang[$importMenu]) : '', ); @@ -516,9 +631,8 @@ public function index() $this->title = _('Search'); if (in_array($this->node, self::$searchPages)) { $this->title = sprintf( - '%s %s', - _('All'), - _("{$this->childClass}s") + _('All %s'), + $this->childClass . 's' ); global $node; global $sub; @@ -1564,8 +1678,29 @@ public function deployPost() $TaskType = new TaskType($type); /** * Account Setup. + * + * Cast, because filter_input() answers NULL for a POST key that + * is not there -- and only a password reset task's form carries + * `account`. That NULL used to be harmless: PDODB cleared + * sql_mode on every connection, so the server quietly coerced it + * to '' on the way into `tasks`.`taskPassreset`, which is + * varchar(250) NOT NULL. GH-1245 removed the clear, so the same + * NULL is now + * + * SQLSTATE[23000]: 1048 Column 'taskPassreset' cannot be null + * + * and the whole statement is refused. It bites a GROUP task and + * not a single-host one because Group::createImagePackage() + * batch-inserts a fixed column list that always names passreset, + * while Host::createImagePackage() only sets it when it holds + * something. Reported on forum topic 18232 against group + * multicast; group deploy takes the same path. + * + * trim() as well, so an account of nothing but spaces is caught + * by the emptiness check below rather than stored. Same + * expression working-1.6 already uses. */ - $passreset = filter_input(INPUT_POST, 'account'); + $passreset = trim((string)filter_input(INPUT_POST, 'account')); /** * Snapin Setup. */ @@ -1614,7 +1749,7 @@ public function deployPost() * Schedule Type Setup. */ $scheduleType = strtolower( - filter_input(INPUT_POST, 'scheduleType') + (string)filter_input(INPUT_POST, 'scheduleType') ); $scheduleTypes = array( 'cron', @@ -1642,9 +1777,23 @@ public function deployPost() /** * Schedule delayed/cron checks. */ - $scheduleDeployTime = self::niceDate( - filter_input(INPUT_POST, 'scheduleSingleTime') - ); + $scheduleSingleTime = filter_input(INPUT_POST, 'scheduleSingleTime'); + /* + * GH-1245: reject the missing time instead of scheduling now. + * + * niceDate() used to read an absent or empty value as the current + * time, so a single schedule with no time silently became "run + * immediately". It now reads empty as "no value", which would + * trip the past-time check below with a message that does not + * describe what happened. + */ + if ('single' === $scheduleType + && (null === $scheduleSingleTime + || '' === trim((string) $scheduleSingleTime)) + ) { + throw new Exception(_('A scheduled time is required')); + } + $scheduleDeployTime = self::niceDate($scheduleSingleTime); switch ($scheduleType) { case 'single': if ($scheduleDeployTime < self::niceDate()) { @@ -2767,7 +2916,7 @@ function ($output, $info) use (&$httpCode) { '%s%s_%s', $backuppath, $destfile, - self::formatTime('', 'Ymd_His') + self::formatTime('now', 'Ymd_His') ); list( $tftpPass, @@ -3072,7 +3221,7 @@ function ($output, $info) use (&$httpCode) { '%s%s_%s', $backuppath, $destfile, - self::formatTime('', 'Ymd_His') + self::formatTime('now', 'Ymd_His') ); list( $tftpPass, @@ -3718,7 +3867,8 @@ public function clearAES() // Reset must leave nothing behind that authorize() would // still accept, grace token included. 'prev_sec_tok' => '', - 'sec_time' => '0000-00-00 00:00:00' + // GH-1245: no expiry, not an expiry in the year zero. + 'sec_time' => null ) ); } @@ -3958,7 +4108,7 @@ public function membership() . $this->node . '1" class="toggle-checkbox1" id="toggler"/>' . '', - _(ucfirst($objType) . ' Name') + sprintf(_('%s Name'), ucfirst($objType)) ); $this->templates = array( ''; echo ''; echo ''; @@ -4060,7 +4210,7 @@ public function membership() echo ''; echo '
'; echo '