firefrost-gaming/firefrost-services
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add 10-minute retry for failed server syncs

Browse Source 
When hourly sync encounters servers that fail (e.g., mid-restart):
- Logs the failure count
- Schedules automatic retry in 10 minutes
- Retry only targets previously failed servers
- Clears error state on successful retry

Fixes issue where servers in daily restart would stay in error state
until manual intervention.

Chronicler #69
This commit is contained in:
Claude
2026-04-08 08:39:34 +00:00
parent c2b6610e6d
commit a13d9a2c66
2 changed files with 52 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
services/arbiter-3.0/src/sync/cron.js
View File

@@ -2,6 +2,8 @@ const cron = require('node-cron');
const { triggerImmediateSync } = require('./immediate');
const { processExpiredGracePeriods } = require('./graceExpiration');
let retryTimeout = null;
function initCron() {
// Hourly whitelist reconciliation
cron.schedule('0 * * * *', async () => {
@@ -12,7 +14,29 @@ function initCron() {
// 2. Whitelist reconciliation
console.log("Starting whitelist reconciliation...");
await triggerImmediateSync();
const { failCount } = await triggerImmediateSync();
// 3. Schedule retry if there were failures
if (failCount > 0) {
console.log(`⏳ ${failCount} servers failed. Scheduling retry in 10 minutes...`);
// Clear any existing retry timeout
if (retryTimeout) {
clearTimeout(retryTimeout);
}
// Retry failed servers after 10 minutes
retryTimeout = setTimeout(async () => {
console.log("🔄 Running retry sync for failed servers...");
const { failCount: retryFailCount } = await triggerImmediateSync(true);
if (retryFailCount > 0) {
console.log(`⚠️ ${retryFailCount} servers still failing after retry.`);
} else {
console.log("✅ All previously failed servers now synced successfully.");
}
}, 10 * 60 * 1000); // 10 minutes
}
console.log("✅ Hourly sync jobs complete");
});

34
services/arbiter-3.0/src/sync/immediate.js
View File

@@ -3,8 +3,8 @@ const { getMinecraftServers } = require('../panel/discovery');
const { writeWhitelistFile } = require('../panel/files');
const { reloadWhitelistCommand } = require('../panel/commands');
async function triggerImmediateSync() {
console.log("--- Starting Whitelist Sync ---");
async function triggerImmediateSync(retryOnly = false) {
console.log(retryOnly ? "--- Starting Retry Sync (failed servers only) ---" : "--- Starting Whitelist Sync ---");
try {
// 1. Fetch Players (Now includes 'lifetime' for the Trinity)
const { rows: players, rowCount: playerCount } = await db.query(
@@ -16,12 +16,27 @@ async function triggerImmediateSync() {
console.log(`[Sync] Retrieved ${playerCount} active players from database.`);
// 2. Fetch Servers
const servers = await getMinecraftServers();
console.log(`[Sync] Discovered ${servers.length} target servers.`);
let servers = await getMinecraftServers();
// If retry mode, only sync servers that previously failed
if (retryOnly) {
const { rows: failedServers } = await db.query(
`SELECT server_identifier FROM server_sync_log WHERE is_online = false`
);
const failedIds = failedServers.map(s => s.server_identifier);
servers = servers.filter(s => failedIds.includes(s.identifier));
console.log(`[Sync] Retrying ${servers.length} previously failed servers.`);
} else {
console.log(`[Sync] Discovered ${servers.length} target servers.`);
}
if (servers.length === 0) {
console.warn("[Sync] WARN: 0 servers discovered. Check MINECRAFT_NEST_IDS in .env.");
return;
if (retryOnly) {
console.log("[Sync] No failed servers to retry.");
} else {
console.warn("[Sync] WARN: 0 servers discovered. Check MINECRAFT_NEST_IDS in .env.");
}
return { successCount: 0, failCount: 0 };
}
// 3. Process Servers Sequentially
@@ -34,10 +49,13 @@ async function triggerImmediateSync() {
await reloadWhitelistCommand(server.identifier);
await db.query(
"INSERT INTO server_sync_log (server_identifier, last_successful_sync, is_online) VALUES ($1, NOW(), true) ON CONFLICT (server_identifier) DO UPDATE SET last_successful_sync = NOW(), is_online = true",
"INSERT INTO server_sync_log (server_identifier, last_successful_sync, is_online, last_error) VALUES ($1, NOW(), true, NULL) ON CONFLICT (server_identifier) DO UPDATE SET last_successful_sync = NOW(), is_online = true, last_error = NULL",
[server.identifier]
);
successCount++;
if (retryOnly) {
console.log(`[Sync] ✅ Retry succeeded for ${server.name}`);
}
} catch (err) {
console.error(`[Sync] ❌ Failed for server ${server.name} (${server.identifier}):`, err.message);
await db.query(
@@ -50,8 +68,10 @@ async function triggerImmediateSync() {
console.log(`[Sync] Complete. Success: ${successCount}, Failed: ${failCount}`);
console.log("-------------------------------");
return { successCount, failCount };
} catch (error) {
console.error("[Sync] Critical failure during execution:", error);
return { successCount: 0, failCount: 0 };
}
}