Force recovery when an old-generation backup worker fails (BackupNewAndOldRestore failure)

fdbserver/backupworker/BackupWorker.cpp

@@ -36,6 +36,7 @@
 #include "fdbserver/core/WaitFailure.h"
 #include "fdbserver/backupworker/BackupWorker.h"
 #include "fdbserver/core/WorkerInterface.h"
+#include "flow/Buggify.h"
 #include "flow/Error.h"
 
 #include "flow/IRandom.h"
@@ -1048,6 +1049,13 @@ Future<Void> monitorBackupPause(Database cx,
 	}
 }
 
+// Nothing else in simulation fails a worker draining an older generation's range, so the recovery that repairs such a
+// range would otherwise never be exercised. The delay is short because such a worker often finishes within a second.
+static Future<Void> buggifyOldEpochWorkerFailure() {
+	co_await delay(deterministicRandom()->random01() * 0.5);
+	throw io_error().asInjectedFault();
+}
+
 Future<Void> backupWorker(BackupInterface interf,
                           InitializeBackupRequest req,
                           Reference<AsyncVar<ServerDBInfo> const> db) {
@@ -1078,6 +1086,10 @@ Future<Void> backupWorker(BackupInterface interf,
 		                                 /*pausedEvent=*/"BackupWorkerPaused",
 		                                 /*resumedEvent=*/"BackupWorkerResumed"));
 
+		if (req.backupEpoch != req.recruitedEpoch && buggify()) {
+			addActor.send(buggifyOldEpochWorkerFailure());
+		}
+
 		// If the worker is on an old epoch and all backups starts a version >= the endVersion
 		bool exitEarly = co_await shouldBackupWorkerExitEarly(&self);
 		TraceEvent("BackupWorkerExitEarly", self.myId).detail("ExitEarly", exitEarly);

fdbserver/logsystem/LogSystem.cpp

@@ -607,6 +607,15 @@ void LogSystem::coreStateWritten(DBCoreState const& newState) {
 	}
 }
 
+namespace {
+
+Future<Void> probeOldEpochBackupWorkerFailure(Future<Void> failure) {
+	co_await failure;
+	CODE_PROBE(true, "Old-generation backup worker failure forced a recovery");
+}
+
+} // namespace
+
 Future<Void> LogSystem::onError() const {
 	// Never returns normally, but throws an error if the subsystem stops working
 	while (true) {
@@ -654,6 +663,26 @@ Future<Void> LogSystem::onError() const {
 			}
 		}
 
+		// Monitored unconditionally, unlike the old log routers below: only a recovery re-recruits an unfinished
+		// range, so losing one of these workers otherwise pins oldestBackupEpoch and FULLY_RECOVERED is never reached.
+		for (auto& old : oldLogData) {
+			for (auto& logSet : old.tLogs) {
+				for (const auto& worker : logSet->backupWorkers) {
+					if (worker->get().present()) {
+						backupFailed.push_back(probeOldEpochBackupWorkerFailure(
+						    waitFailureClient(worker->get().interf().waitFailure,
+						                      /* failureReactionTime */ SERVER_KNOBS->BACKUP_TIMEOUT,
+						                      /* failureReactionSlope */ -SERVER_KNOBS->BACKUP_TIMEOUT /
+						                          SERVER_KNOBS->SECONDS_BEFORE_NO_FAILURE_DELAY,
+						                      /* trace */ true,
+						                      /* traceMsg */ "OldEpochBackupWorkerFailed"_sr)));
+					} else {
+						changes.push_back(worker->onChange());
+					}
+				}
+			}
+		}
+
 		if (!recoveryCompleteWrittenToCoreState.get()) {
 			failed.insert(failed.end(), routerFailed.begin(), routerFailed.end());
 			routerFailed.clear();
@@ -672,8 +701,6 @@ Future<Void> LogSystem::onError() const {
 						}
 					}
 				}
-				// Old-generation backup workers are stateless and persist their progress. A failure can retain
-				// this generation, but must not restart transaction-system recovery.
 			}
 		}