Recover from repeated route execution failures

scripts/test-playerbot-gameplay.ps1

@@ -1015,6 +1015,31 @@ function Assert-NavigationEvents {
     }
 }
 
+function Assert-NavigationRecoveryEvents {
+	param([string]$Logs)
+
+	$events = @(ConvertFrom-PlayerbotLogs -Logs $Logs)
+	$mismatches = @($events | Where-Object {
+		$_.event -eq "action_result" -and $_.action -eq "navigate" -and
+		$_.result -eq "failed" -and $_.reason -eq "step_result_mismatch"
+	})
+	$recovery = @($events | Where-Object {
+		$_.event -eq "hunt_region_patrol" -and $_.result -eq "skipped" -and
+		$_.reason -eq "repeated_step_failure" -and $_.step_failures -eq 3 -and $_.region_id -eq $null
+	})
+	$waypoints = @($events | Where-Object {
+		$_.event -eq "action_result" -and $_.action -eq "hunt_waypoint" -and $_.result -eq "reached"
+	})
+	$terminal = @($events | Where-Object { $_.event -eq "terminal" })
+	$firstWaypoint = if ($waypoints.Count -gt 0) {
+		"$($waypoints[0].position.x),$($waypoints[0].position.y),$($waypoints[0].position.z)"
+	} else { "" }
+	if ($mismatches.Count -lt 1 -or $recovery.Count -ne 1 -or $firstWaypoint -ne "32103,32124,8" -or
+		$terminal.Count -ne 0) {
+		throw "Repeated route execution recovery failed. mismatches=$($mismatches.Count), recovery=$($recovery.Count), firstWaypoint=$firstWaypoint, terminal=$($terminal.Count)."
+	}
+}
+
 function Assert-TargetPursuitEvents {
 	param([string]$Logs)
 
@@ -1770,6 +1795,15 @@ try {
 			$navigationLogs = Wait-ForPlayerbotEventCount -Action "hunt_waypoint" -Count 5
 			Assert-NavigationEvents -Logs $navigationLogs
 		}
+		Invoke-Scenario -Name "navigation_recovery" -DefaultTimeoutSeconds 150 -Body {
+			Invoke-Compose down --volumes --remove-orphans
+			$env:PLAYERBOT_GAMEPLAY_MODE = "navigation_recovery"
+			$env:PLAYERBOT_HUNT_DURATION_SECONDS = "900"
+			Invoke-Compose up --detach
+			Wait-ForLog -Pattern 'PLAYERBOT_GAMEPLAY_TEST NAVIGATION_RECOVERY_START' | Out-Null
+			$recoveryLogs = Wait-ForPlayerbotEventCount -Action "hunt_waypoint" -Count 1
+			Assert-NavigationRecoveryEvents -Logs $recoveryLogs
+		}
 	}
 
 	if ($TargetPursuit) {

server/src/playerbotcombat.cpp

@@ -985,22 +985,36 @@ void PlayerBotController::processTraversal(Player* player, const Position& curre
 	const Position& target = patrolPoints && !patrolPoints->empty() ?
 	                         (*patrolPoints)[huntRouteIndex % patrolPoints->size()] : huntingLoop[huntRouteIndex];
 	if (!processNavigation(player, currentPosition, target)) {
-		if (activeHuntRegion && (navigationOscillationDetected || fixedTargetRouteFailureCount >= 3)) {
+		const bool repeatedStepFailure = blockedStepCount >= maximumRepeatedNavigationStepFailures;
+		if (navigationOscillationDetected || repeatedStepFailure ||
+		    (activeHuntRegion && fixedTargetRouteFailureCount >= 3)) {
+			const char* reason = navigationOscillationDetected ? "position_oscillation" :
+			                     repeatedStepFailure ? "repeated_step_failure" : "unreachable";
 			emit("hunt_region_patrol", currentPosition,
 			     "\"result\":\"skipped\",\"reason\":" +
-			         jsonString(navigationOscillationDetected ? "position_oscillation" : "unreachable") +
+			         jsonString(reason) +
+			         ",\"step_failures\":" + std::to_string(blockedStepCount) +
 			         ",\"region_id\":" +
-			         std::to_string(activeHuntRegion->id) + ",\"destination\":{\"x\":" +
+			         (activeHuntRegion ? std::to_string(activeHuntRegion->id) : "null") +
+			         ",\"destination\":{\"x\":" +
 			         std::to_string(target.x) + ",\"y\":" + std::to_string(target.y) +
 			         ",\"z\":" + std::to_string(target.z) + "}");
+			if (repeatedStepFailure) {
+				++counters.stuckEvents;
+			}
 			fixedTargetRouteFailureCount = 0;
+			blockedStepCount = 0;
 			clearNavigation();
-			activeHuntRegion->patrolPoints.erase(activeHuntRegion->patrolPoints.begin() + huntRouteIndex);
-			if (activeHuntRegion->patrolPoints.empty()) {
-				huntRegionCooldowns[activeHuntRegion->center] = std::chrono::steady_clock::now() + huntRegionCooldown;
-				beginService(player, currentPosition, "hunt_region_patrol_unreachable");
+			if (activeHuntRegion) {
+				activeHuntRegion->patrolPoints.erase(activeHuntRegion->patrolPoints.begin() + huntRouteIndex);
+				if (activeHuntRegion->patrolPoints.empty()) {
+					huntRegionCooldowns[activeHuntRegion->center] = std::chrono::steady_clock::now() + huntRegionCooldown;
+					beginService(player, currentPosition, "hunt_region_patrol_unreachable");
+				} else {
+					huntRouteIndex %= activeHuntRegion->patrolPoints.size();
+				}
 			} else {
-				huntRouteIndex %= activeHuntRegion->patrolPoints.size();
+				huntRouteIndex = (huntRouteIndex + 1) % huntingLoop.size();
 			}
 		}
 		return;

server/src/playerbotcontroller.cpp

@@ -35,7 +35,8 @@ const PlayerBotTestPolicy& playerbot::testPolicyFromEnvironment()
 				 std::strcmp(gameplayMode, "equipment_shadow_unaffordable") == 0 ||
 				 std::strcmp(gameplayMode, "equipment_shadow_no_upgrade") == 0);
 		const bool startInHunt = gameplayMode &&
-			(std::strcmp(gameplayMode, "navigation") == 0 || std::strcmp(gameplayMode, "corpse") == 0 ||
+			(std::strcmp(gameplayMode, "navigation") == 0 || std::strcmp(gameplayMode, "navigation_recovery") == 0 ||
+			 std::strcmp(gameplayMode, "corpse") == 0 ||
 			 (std::strcmp(gameplayMode, "target_pursuit") == 0 || std::strcmp(gameplayMode, "target_pursuit_abandon") == 0) ||
 			 std::strcmp(gameplayMode, "healing") == 0 || std::strcmp(gameplayMode, "healing_resupply") == 0 ||
 			 std::strcmp(gameplayMode, "value") == 0 || std::strcmp(gameplayMode, "departure_interrupt") == 0 ||
@@ -75,6 +76,7 @@ const PlayerBotTestPolicy& playerbot::testPolicyFromEnvironment()
 			gameplayMode && std::strcmp(gameplayMode, "hunt_planning") == 0,
 			gameplayMode && std::strcmp(gameplayMode, "hunt_planning") == 0,
 			gameplayMode && std::strcmp(gameplayMode, "hunt_planning") == 0,
+			gameplayMode && std::strcmp(gameplayMode, "navigation_recovery") == 0,
 		};
 	}();
 	return policy;
@@ -85,7 +87,11 @@ PlayerBotController::PlayerBotController(const Player& player,
                             const PlayerBotTestPolicy& testPolicy) :
 	playerId(player.getID()), playerGuid(player.getGUID()), playerName(player.getName()), testPolicy(testPolicy),
 	huntRegionCooldowns(sharedHuntRegionCooldowns)
-{}
+{
+	if (testPolicy.forceRepeatedNavigationStepFailures) {
+		forcedNavigationStepFailuresRemaining = maximumRepeatedNavigationStepFailures;
+	}
+}
 
 void PlayerBotController::start(const Position& position, bool recovered, uint32_t recoveryCount)
 {
@@ -366,6 +372,7 @@ void PlayerBotController::clearNavigation()
 	navigationPending = false;
 	worldChangePending = false;
 	navigationTarget = Position();
+	blockedStepCount = 0;
 }
 
 void PlayerBotController::onDeath(const Player& player, const Creature* killer, const Creature* mostDamageKiller)
@@ -417,6 +424,10 @@ bool PlayerBotController::executeNavigationStep(Player* player, const PlayerBotN
 {
 	++counters.actionsAttempted;
 	if (step.action == PlayerBotNavigationAction::Move) {
+		if (forcedNavigationStepFailuresRemaining > 0) {
+			--forcedNavigationStepFailuresRemaining;
+			return true;
+		}
 		g_game.playerMove(playerId, step.direction);
 		return true;
 	}
@@ -561,6 +572,10 @@ bool PlayerBotController::processNavigation(Player* player, const Position& curr
 				std::chrono::steady_clock::now() + navigationBlockSuppression;
 			logActionFailure("navigate", "step_result_mismatch", currentPosition);
 			++blockedStepCount;
+			if (blockedStepCount >= maximumRepeatedNavigationStepFailures) {
+				schedule(blockedRouteRetryInterval);
+				return false;
+			}
 		}
 	}
 	if (worldChangePending) {
@@ -576,6 +591,7 @@ bool PlayerBotController::processNavigation(Player* player, const Position& curr
 	if (navigationTarget != destination) {
 		navigationSteps.clear();
 		navigationTarget = destination;
+		blockedStepCount = 0;
 	}
 	if (navigationSteps.empty()) {
 		const auto now = std::chrono::steady_clock::now();

server/src/playerbotcontroller.h

@@ -71,6 +71,7 @@ namespace playerbot {
 	inline constexpr std::chrono::seconds navigationBlockSuppression(10);
 	inline constexpr std::chrono::minutes navigationOscillationSuppression(2);
 	inline constexpr std::chrono::seconds navigationStepTimeout(2);
+	inline constexpr uint32_t maximumRepeatedNavigationStepFailures = 3;
 	inline constexpr std::chrono::seconds healingRetryInterval(2);
 	inline constexpr std::chrono::minutes stableLifetimeReset(5);
 	inline constexpr std::chrono::minutes huntRegionCooldown(10);
@@ -145,6 +146,7 @@ namespace playerbot {
 		bool forceFirstHuntCandidateUnreachable;
 		bool forceSecondHuntCandidateNodeLimit;
 		bool cancelHuntPlanningAtScoreBarrier;
+		bool forceRepeatedNavigationStepFailures;
 	};
 
 	std::string jsonString(const std::string& value);
@@ -796,6 +798,7 @@ class PlayerBotController : public std::enable_shared_from_this<PlayerBotControl
 		ScenarioStage scenarioStage = ScenarioStage::Traverse;
 		uint32_t fixedTargetRouteFailureCount = 0;
 		uint32_t blockedStepCount = 0;
+		uint32_t forcedNavigationStepFailuresRemaining = 0;
 		uint32_t corpseSearchAttempts = 0;
 		uint32_t corpseOpenAttempts = 0;
 		uint16_t pendingLootItemId = 0;

server/tests/playerbot-gameplay/playerbot_gameplay.lua

@@ -569,7 +569,7 @@ function login.onLogin(player)
     end
 
     local mode = os.getenv("PLAYERBOT_GAMEPLAY_MODE") or "cycle"
-	assert(mode == "mainland" or mode == "cycle" or mode == "depot" or mode == "navigation" or mode == "target_pursuit" or mode == "target_pursuit_abandon" or mode == "corpse" or mode == "death" or mode == "healing" or mode == "spell_use" or
+	assert(mode == "mainland" or mode == "cycle" or mode == "depot" or mode == "navigation" or mode == "navigation_recovery" or mode == "target_pursuit" or mode == "target_pursuit_abandon" or mode == "corpse" or mode == "death" or mode == "healing" or mode == "spell_use" or
 		mode == "healing_resupply" or mode == "value" or mode == "progression" or mode == "progression_bundle" or
 		mode == "progression_nested" or
         mode == "progression_resume" or mode == "progression_nested_resume" or mode == "progression_space" or
@@ -918,6 +918,11 @@ function login.onLogin(player)
         print("PLAYERBOT_GAMEPLAY_TEST NAVIGATION_START")
         return true
     end
+	if mode == "navigation_recovery" then
+		suppressNearbyMonsters(player:getId())
+		print("PLAYERBOT_GAMEPLAY_TEST NAVIGATION_RECOVERY_START")
+		return true
+	end
     if mode == "healing" then
         suppressNearbyMonsters(player:getId())
 		local potions = player:getItemCount(potionItemId)