From a13d9a2c6613e343bcf8ef09bed92214ae6a1686 Mon Sep 17 00:00:00 2001
From: Claude <claude@firefrostgaming.com>
Date: Wed, 8 Apr 2026 08:39:34 +0000
Subject: [PATCH] Add 10-minute retry for failed server syncs

When hourly sync encounters servers that fail (e.g., mid-restart):
- Logs the failure count
- Schedules automatic retry in 10 minutes
- Retry only targets previously failed servers
- Clears error state on successful retry

Fixes issue where servers in daily restart would stay in error state
until manual intervention.

Chronicler #69
---
 services/arbiter-3.0/src/sync/cron.js      | 26 ++++++++++++++++-
 services/arbiter-3.0/src/sync/immediate.js | 34 +++++++++++++++++-----
 2 files changed, 52 insertions(+), 8 deletions(-)

diff --git a/services/arbiter-3.0/src/sync/cron.js b/services/arbiter-3.0/src/sync/cron.js
index 9c68553..d0f70b6 100644
--- a/services/arbiter-3.0/src/sync/cron.js
+++ b/services/arbiter-3.0/src/sync/cron.js
@@ -2,6 +2,8 @@ const cron = require('node-cron');
 const { triggerImmediateSync } = require('./immediate');
 const { processExpiredGracePeriods } = require('./graceExpiration');
 
+let retryTimeout = null;
+
 function initCron() {
   // Hourly whitelist reconciliation
   cron.schedule('0 * * * *', async () => {
@@ -12,7 +14,29 @@ function initCron() {
     
     // 2. Whitelist reconciliation
     console.log("Starting whitelist reconciliation...");
-    await triggerImmediateSync();
+    const { failCount } = await triggerImmediateSync();
+    
+    // 3. Schedule retry if there were failures
+    if (failCount > 0) {
+      console.log(`⏳ ${failCount} servers failed. Scheduling retry in 10 minutes...`);
+      
+      // Clear any existing retry timeout
+      if (retryTimeout) {
+        clearTimeout(retryTimeout);
+      }
+      
+      // Retry failed servers after 10 minutes
+      retryTimeout = setTimeout(async () => {
+        console.log("🔄 Running retry sync for failed servers...");
+        const { failCount: retryFailCount } = await triggerImmediateSync(true);
+        
+        if (retryFailCount > 0) {
+          console.log(`⚠️ ${retryFailCount} servers still failing after retry.`);
+        } else {
+          console.log("✅ All previously failed servers now synced successfully.");
+        }
+      }, 10 * 60 * 1000); // 10 minutes
+    }
     
     console.log("✅ Hourly sync jobs complete");
   });
diff --git a/services/arbiter-3.0/src/sync/immediate.js b/services/arbiter-3.0/src/sync/immediate.js
index 1781e6b..03e07c0 100644
--- a/services/arbiter-3.0/src/sync/immediate.js
+++ b/services/arbiter-3.0/src/sync/immediate.js
@@ -3,8 +3,8 @@ const { getMinecraftServers } = require('../panel/discovery');
 const { writeWhitelistFile } = require('../panel/files');
 const { reloadWhitelistCommand } = require('../panel/commands');
 
-async function triggerImmediateSync() {
-  console.log("--- Starting Whitelist Sync ---");
+async function triggerImmediateSync(retryOnly = false) {
+  console.log(retryOnly ? "--- Starting Retry Sync (failed servers only) ---" : "--- Starting Whitelist Sync ---");
   try {
     // 1. Fetch Players (Now includes 'lifetime' for the Trinity)
     const { rows: players, rowCount: playerCount } = await db.query(
@@ -16,12 +16,27 @@ async function triggerImmediateSync() {
     console.log(`[Sync] Retrieved ${playerCount} active players from database.`);
 
     // 2. Fetch Servers
-    const servers = await getMinecraftServers();
-    console.log(`[Sync] Discovered ${servers.length} target servers.`);
+    let servers = await getMinecraftServers();
+    
+    // If retry mode, only sync servers that previously failed
+    if (retryOnly) {
+      const { rows: failedServers } = await db.query(
+        `SELECT server_identifier FROM server_sync_log WHERE is_online = false`
+      );
+      const failedIds = failedServers.map(s => s.server_identifier);
+      servers = servers.filter(s => failedIds.includes(s.identifier));
+      console.log(`[Sync] Retrying ${servers.length} previously failed servers.`);
+    } else {
+      console.log(`[Sync] Discovered ${servers.length} target servers.`);
+    }
 
     if (servers.length === 0) {
-      console.warn("[Sync] WARN: 0 servers discovered. Check MINECRAFT_NEST_IDS in .env.");
-      return;
+      if (retryOnly) {
+        console.log("[Sync] No failed servers to retry.");
+      } else {
+        console.warn("[Sync] WARN: 0 servers discovered. Check MINECRAFT_NEST_IDS in .env.");
+      }
+      return { successCount: 0, failCount: 0 };
     }
 
     // 3. Process Servers Sequentially
@@ -34,10 +49,13 @@ async function triggerImmediateSync() {
         await reloadWhitelistCommand(server.identifier);
         
         await db.query(
-          "INSERT INTO server_sync_log (server_identifier, last_successful_sync, is_online) VALUES ($1, NOW(), true) ON CONFLICT (server_identifier) DO UPDATE SET last_successful_sync = NOW(), is_online = true",
+          "INSERT INTO server_sync_log (server_identifier, last_successful_sync, is_online, last_error) VALUES ($1, NOW(), true, NULL) ON CONFLICT (server_identifier) DO UPDATE SET last_successful_sync = NOW(), is_online = true, last_error = NULL",
           [server.identifier]
         );
         successCount++;
+        if (retryOnly) {
+          console.log(`[Sync] ✅ Retry succeeded for ${server.name}`);
+        }
       } catch (err) {
         console.error(`[Sync] ❌ Failed for server ${server.name} (${server.identifier}):`, err.message);
         await db.query(
@@ -50,8 +68,10 @@ async function triggerImmediateSync() {
     
     console.log(`[Sync] Complete. Success: ${successCount}, Failed: ${failCount}`);
     console.log("-------------------------------");
+    return { successCount, failCount };
   } catch (error) {
     console.error("[Sync] Critical failure during execution:", error);
+    return { successCount: 0, failCount: 0 };
   }
 }