From 2ea9570e88a7a1a360c21f974ec1331b33edf678 Mon Sep 17 00:00:00 2001 From: michael stack Date: Thu, 17 Sep 2026 18:58:50 -0700 Subject: [PATCH 1/2] Force recovery when an old-generation backup worker fails A backup worker draining an older generation's range was excluded from the log system's failure set, so only a later recovery could replace one. When none followed, the range stayed unfinished forever: oldestBackupEpoch pinned, the generation retained in oldTLogData, and FULLY_RECOVERED unreachable in a cluster that was otherwise healthy. Monitor those workers, so losing one raises backup_worker_failed and the resulting recovery re-recruits the range through recruitBackupWorkers, which already handles every unfinished range. Only unfinished work forces a recovery: a worker that has drained its range sent BackupWorkerDone and was erased from the log set, so it is no longer monitored. Nothing in simulation failed such a worker, so add a buggify-gated io_error to reach the repair, and a code probe on detection. --- fdbserver/backupworker/BackupWorker.cpp | 12 ++++++++++ fdbserver/logsystem/LogSystem.cpp | 31 +++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/fdbserver/backupworker/BackupWorker.cpp b/fdbserver/backupworker/BackupWorker.cpp index 54fe2f1be61..df2fe794722 100644 --- a/fdbserver/backupworker/BackupWorker.cpp +++ b/fdbserver/backupworker/BackupWorker.cpp @@ -36,6 +36,7 @@ #include "fdbserver/core/WaitFailure.h" #include "fdbserver/backupworker/BackupWorker.h" #include "fdbserver/core/WorkerInterface.h" +#include "flow/Buggify.h" #include "flow/Error.h" #include "flow/IRandom.h" @@ -1048,6 +1049,13 @@ Future monitorBackupPause(Database cx, } } +// Nothing else in simulation fails a worker draining an older generation's range, so the recovery that repairs such a +// range would otherwise never be exercised. The delay is short because such a worker often finishes within a second. +static Future buggifyOldEpochWorkerFailure() { + co_await delay(deterministicRandom()->random01() * 0.5); + throw io_error(); +} + Future backupWorker(BackupInterface interf, InitializeBackupRequest req, Reference const> db) { @@ -1078,6 +1086,10 @@ Future backupWorker(BackupInterface interf, /*pausedEvent=*/"BackupWorkerPaused", /*resumedEvent=*/"BackupWorkerResumed")); + if (req.backupEpoch != req.recruitedEpoch && buggify()) { + addActor.send(buggifyOldEpochWorkerFailure()); + } + // If the worker is on an old epoch and all backups starts a version >= the endVersion bool exitEarly = co_await shouldBackupWorkerExitEarly(&self); TraceEvent("BackupWorkerExitEarly", self.myId).detail("ExitEarly", exitEarly); diff --git a/fdbserver/logsystem/LogSystem.cpp b/fdbserver/logsystem/LogSystem.cpp index 754e0f2ab29..c38e05db985 100644 --- a/fdbserver/logsystem/LogSystem.cpp +++ b/fdbserver/logsystem/LogSystem.cpp @@ -607,6 +607,15 @@ void LogSystem::coreStateWritten(DBCoreState const& newState) { } } +namespace { + +Future probeOldEpochBackupWorkerFailure(Future failure) { + co_await failure; + CODE_PROBE(true, "Old-generation backup worker failure forced a recovery"); +} + +} // namespace + Future LogSystem::onError() const { // Never returns normally, but throws an error if the subsystem stops working while (true) { @@ -654,6 +663,26 @@ Future LogSystem::onError() const { } } + // Monitored unconditionally, unlike the old log routers below: only a recovery re-recruits an unfinished + // range, so losing one of these workers otherwise pins oldestBackupEpoch and FULLY_RECOVERED is never reached. + for (auto& old : oldLogData) { + for (auto& logSet : old.tLogs) { + for (const auto& worker : logSet->backupWorkers) { + if (worker->get().present()) { + backupFailed.push_back(probeOldEpochBackupWorkerFailure( + waitFailureClient(worker->get().interf().waitFailure, + /* failureReactionTime */ SERVER_KNOBS->BACKUP_TIMEOUT, + /* failureReactionSlope */ -SERVER_KNOBS->BACKUP_TIMEOUT / + SERVER_KNOBS->SECONDS_BEFORE_NO_FAILURE_DELAY, + /* trace */ true, + /* traceMsg */ "OldEpochBackupWorkerFailed"_sr))); + } else { + changes.push_back(worker->onChange()); + } + } + } + } + if (!recoveryCompleteWrittenToCoreState.get()) { failed.insert(failed.end(), routerFailed.begin(), routerFailed.end()); routerFailed.clear(); @@ -672,8 +701,6 @@ Future LogSystem::onError() const { } } } - // Old-generation backup workers are stateless and persist their progress. A failure can retain - // this generation, but must not restart transaction-system recovery. } } From 6193ffe412d1403bf5e9193c3549039f86273b3a Mon Sep 17 00:00:00 2001 From: michael stack Date: Fri, 18 Sep 2026 21:14:14 -0700 Subject: [PATCH 2/2] Mark the buggified old-epoch backup worker failure as an injected fault Every other simulated io_error in the tree is raised as io_error().asInjectedFault(), and checkIOTimeout goes out of its way to preserve injectedness. Injected faults are what let Trace.cpp downgrade a SevError, so without the marking this injection only avoids failing the run because endRole special-cases Role::BACKUP to SevWarnAlways. It also makes the trace match the nightly signature this reproduces, ErrorIsInjectedFault="1". --- fdbserver/backupworker/BackupWorker.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdbserver/backupworker/BackupWorker.cpp b/fdbserver/backupworker/BackupWorker.cpp index df2fe794722..6c3c626ef9b 100644 --- a/fdbserver/backupworker/BackupWorker.cpp +++ b/fdbserver/backupworker/BackupWorker.cpp @@ -1053,7 +1053,7 @@ Future monitorBackupPause(Database cx, // range would otherwise never be exercised. The delay is short because such a worker often finishes within a second. static Future buggifyOldEpochWorkerFailure() { co_await delay(deterministicRandom()->random01() * 0.5); - throw io_error(); + throw io_error().asInjectedFault(); } Future backupWorker(BackupInterface interf,