diff --git a/fdbserver/backupworker/BackupWorker.cpp b/fdbserver/backupworker/BackupWorker.cpp index 54fe2f1be61..6c3c626ef9b 100644 --- a/fdbserver/backupworker/BackupWorker.cpp +++ b/fdbserver/backupworker/BackupWorker.cpp @@ -36,6 +36,7 @@ #include "fdbserver/core/WaitFailure.h" #include "fdbserver/backupworker/BackupWorker.h" #include "fdbserver/core/WorkerInterface.h" +#include "flow/Buggify.h" #include "flow/Error.h" #include "flow/IRandom.h" @@ -1048,6 +1049,13 @@ Future monitorBackupPause(Database cx, } } +// Nothing else in simulation fails a worker draining an older generation's range, so the recovery that repairs such a +// range would otherwise never be exercised. The delay is short because such a worker often finishes within a second. +static Future buggifyOldEpochWorkerFailure() { + co_await delay(deterministicRandom()->random01() * 0.5); + throw io_error().asInjectedFault(); +} + Future backupWorker(BackupInterface interf, InitializeBackupRequest req, Reference const> db) { @@ -1078,6 +1086,10 @@ Future backupWorker(BackupInterface interf, /*pausedEvent=*/"BackupWorkerPaused", /*resumedEvent=*/"BackupWorkerResumed")); + if (req.backupEpoch != req.recruitedEpoch && buggify()) { + addActor.send(buggifyOldEpochWorkerFailure()); + } + // If the worker is on an old epoch and all backups starts a version >= the endVersion bool exitEarly = co_await shouldBackupWorkerExitEarly(&self); TraceEvent("BackupWorkerExitEarly", self.myId).detail("ExitEarly", exitEarly); diff --git a/fdbserver/logsystem/LogSystem.cpp b/fdbserver/logsystem/LogSystem.cpp index 754e0f2ab29..c38e05db985 100644 --- a/fdbserver/logsystem/LogSystem.cpp +++ b/fdbserver/logsystem/LogSystem.cpp @@ -607,6 +607,15 @@ void LogSystem::coreStateWritten(DBCoreState const& newState) { } } +namespace { + +Future probeOldEpochBackupWorkerFailure(Future failure) { + co_await failure; + CODE_PROBE(true, "Old-generation backup worker failure forced a recovery"); +} + +} // namespace + Future LogSystem::onError() const { // Never returns normally, but throws an error if the subsystem stops working while (true) { @@ -654,6 +663,26 @@ Future LogSystem::onError() const { } } + // Monitored unconditionally, unlike the old log routers below: only a recovery re-recruits an unfinished + // range, so losing one of these workers otherwise pins oldestBackupEpoch and FULLY_RECOVERED is never reached. + for (auto& old : oldLogData) { + for (auto& logSet : old.tLogs) { + for (const auto& worker : logSet->backupWorkers) { + if (worker->get().present()) { + backupFailed.push_back(probeOldEpochBackupWorkerFailure( + waitFailureClient(worker->get().interf().waitFailure, + /* failureReactionTime */ SERVER_KNOBS->BACKUP_TIMEOUT, + /* failureReactionSlope */ -SERVER_KNOBS->BACKUP_TIMEOUT / + SERVER_KNOBS->SECONDS_BEFORE_NO_FAILURE_DELAY, + /* trace */ true, + /* traceMsg */ "OldEpochBackupWorkerFailed"_sr))); + } else { + changes.push_back(worker->onChange()); + } + } + } + } + if (!recoveryCompleteWrittenToCoreState.get()) { failed.insert(failed.end(), routerFailed.begin(), routerFailed.end()); routerFailed.clear(); @@ -672,8 +701,6 @@ Future LogSystem::onError() const { } } } - // Old-generation backup workers are stateless and persist their progress. A failure can retain - // this generation, but must not restart transaction-system recovery. } }