From d0ee584f55277f6bfc5cc0a9f28100de5aa6b8dd Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Apr 2026 09:35:42 +0000 Subject: [PATCH] Task #94: Global Restart Scheduler - Full Architecture WHAT THIS IS: Trinity Console feature to manage staggered restart schedules for all 21 Minecraft servers across TX1 and NC1 nodes. FEATURES: - Visual timeline showing restart sequence - Configurable base time + interval per node - Drag-and-drop sort order for boot priority - One-click sync to Pterodactyl API - Audit system to detect/remove conflicting schedules - Rate-limited API calls (200ms delay) - Full audit trail logging DATABASE TABLES: - global_restart_config (node settings) - server_restart_schedules (per-server state) - sync_logs (audit trail) DEFAULT PATTERN: - TX1: 04:00 UTC, 5-min intervals - NC1: 04:30 UTC, 5-min intervals CONSULTATION: Full architecture session with Gemini AI (April 5, 2026) IMPLEMENTATION: Complete code provided - ready for next Chronicler Signed-off-by: Claude (Chronicler #60) --- .../README.md | 680 ++++++++++++++++++ 1 file changed, 680 insertions(+) create mode 100644 docs/tasks/task-094-global-restart-scheduler/README.md diff --git a/docs/tasks/task-094-global-restart-scheduler/README.md b/docs/tasks/task-094-global-restart-scheduler/README.md new file mode 100644 index 0000000..b68af99 --- /dev/null +++ b/docs/tasks/task-094-global-restart-scheduler/README.md @@ -0,0 +1,680 @@ +# Task #94: Global Restart Scheduler + +**Status:** Ready for Implementation +**Priority:** Medium +**Estimated Effort:** 4-6 hours +**Dependencies:** Arbiter 3.5.0, Pterodactyl Client API +**Consultation:** Gemini (April 5, 2026) + +--- + +## Overview + +Add a **Global Restart Scheduler** to the Trinity Console that manages staggered restart times for all 21 Minecraft servers across TX1 and NC1 nodes. This eliminates the need to manually configure individual schedules in Pterodactyl. + +### The Problem +- 21 servers need restart schedules +- Manual configuration in Pterodactyl is tedious +- No central visibility of restart timing +- Risk of overlapping restarts hammering nodes + +### The Solution +- Trinity Console page with visual timeline +- Staggered restarts (configurable base time + interval per node) +- One-click sync to Pterodactyl API +- Audit system to detect/remove conflicting schedules +- Drag-and-drop sort order for boot priority + +--- + +## Architecture Decisions + +### Source of Truth +**Store schedule intent in `arbiter_db`, sync to Pterodactyl as execution.** + +- `global_restart_config` table holds base time + interval per node +- `server_restart_schedules` table holds per-server state +- Trinity calculates `effective_time` and pushes to Pterodactyl +- If Pterodactyl is wiped, Trinity can re-provision instantly + +### Schedule Naming +All Trinity-managed schedules use prefix: `[Trinity] Daily Restart` + +This prevents confusion with manual schedules and allows Trinity to identify its own schedules. + +### Sort Order +**Manual sort via drag-and-drop.** Boot order matters: +- Proxy servers should restart first +- Hub/Lobby servers should restart last +- Heavy modpacks need spacing + +### Conflict Resolution +Before first sync, run **Audit** to detect existing restart schedules. Trinity can "nuke" rogue schedules before taking control. + +--- + +## Database Schema + +### Migration File: `migrations/094_global_restart_scheduler.sql` + +```sql +-- 1. Configuration for Node-wide Stagger Logic +CREATE TABLE IF NOT EXISTS global_restart_config ( + id SERIAL PRIMARY KEY, + node VARCHAR(10) UNIQUE NOT NULL, -- 'TX1', 'NC1' + base_time TIME NOT NULL, -- e.g., '04:00:00' (UTC) + interval_minutes INT DEFAULT 5, -- Stagger gap + is_enabled BOOLEAN DEFAULT true, -- Global master switch per node + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_by VARCHAR(50) -- Discord Username +); + +-- 2. Individual Server Execution State +CREATE TABLE IF NOT EXISTS server_restart_schedules ( + id SERIAL PRIMARY KEY, + server_id VARCHAR(50) UNIQUE NOT NULL, -- Pterodactyl 8-char short ID + server_name VARCHAR(100) NOT NULL, + node VARCHAR(10) NOT NULL, + sort_order INT NOT NULL DEFAULT 0, -- Manual boot order + effective_time TIME, -- Calculated: base + (sort * interval) + ptero_schedule_id INT DEFAULT NULL, -- ID of schedule on Pterodactyl + skip_restart BOOLEAN DEFAULT false, -- Individual "Maintenance Mode" + sync_status VARCHAR(20) DEFAULT 'PENDING', -- 'SUCCESS', 'PENDING', 'FAILED' + last_error TEXT DEFAULT NULL, -- API error capture + last_synced_at TIMESTAMP NULL, + + CONSTRAINT fk_node_config + FOREIGN KEY (node) + REFERENCES global_restart_config(node) + ON UPDATE CASCADE +); + +-- 3. Audit Trail for Sync Operations +CREATE TABLE IF NOT EXISTS sync_logs ( + id SERIAL PRIMARY KEY, + server_id VARCHAR(50) NOT NULL, + action VARCHAR(255) NOT NULL, -- e.g., 'Deleted Rogue Schedule', 'Created Schedule' + status VARCHAR(20) NOT NULL, -- 'SUCCESS', 'FAILED' + error_message TEXT DEFAULT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +-- 4. Performance Indexes +CREATE INDEX idx_server_node_order ON server_restart_schedules (node, sort_order); +CREATE INDEX idx_sync_status ON server_restart_schedules (sync_status); +CREATE INDEX idx_sync_logs_server ON sync_logs (server_id); +CREATE INDEX idx_sync_logs_created ON sync_logs (created_at); + +-- 5. Initial Seed Data +INSERT INTO global_restart_config (node, base_time, interval_minutes, updated_by) +VALUES + ('TX1', '04:00:00', 5, 'The Wizard'), + ('NC1', '04:30:00', 5, 'The Wizard') +ON CONFLICT (node) DO NOTHING; +``` + +--- + +## Implementation Files + +### 1. Utility: Stagger Calculation + +**File:** `src/utils/scheduler.js` + +```javascript +const { addMinutes, format, parse } = require('date-fns'); + +function calculateStagger(baseTime, interval, servers) { + const start = parse(baseTime, 'HH:mm', new Date()); + + return servers.map((server, index) => ({ + ...server, + effective_time: format(addMinutes(start, index * interval), 'HH:mm:ss') + })); +} + +module.exports = { calculateStagger }; +``` + +### 2. Pterodactyl API Functions + +**File:** `src/lib/ptero-sync.js` + +```javascript +const axios = require('axios'); +const db = require('../db'); + +const PTERO_URL = 'https://panel.firefrostgaming.com/api/client/servers'; +const headers = { + 'Authorization': `Bearer ${process.env.PTERO_CLIENT_KEY}`, + 'Content-Type': 'application/json', + 'Accept': 'application/json' +}; + +/** + * Sync a single server's schedule to Pterodactyl + */ +async function syncToPterodactyl(serverId) { + const result = await db.query('SELECT * FROM server_restart_schedules WHERE server_id = $1', [serverId]); + const server = result.rows[0]; + const [hour, minute] = server.effective_time.split(':'); + + const pteroUrl = `${PTERO_URL}/${server.server_id}/schedules`; + + const payload = { + name: "[Trinity] Daily Restart", + minute, + hour, + day_of_week: "*", + day_of_month: "*", + month: "*", + is_active: !server.skip_restart + }; + + try { + let scheduleId = server.ptero_schedule_id; + + if (!scheduleId) { + // Create new schedule + const res = await axios.post(pteroUrl, payload, { headers }); + scheduleId = res.data.attributes.id; + + // Attach the restart task + await axios.post(`${pteroUrl}/${scheduleId}/tasks`, { + action: "power", + payload: "restart", + time_offset: 0 + }, { headers }); + } else { + // Update existing schedule (Pterodactyl uses POST for updates too) + await axios.post(`${pteroUrl}/${scheduleId}`, payload, { headers }); + } + + await db.query( + `UPDATE server_restart_schedules + SET ptero_schedule_id = $1, sync_status = $2, last_error = NULL, last_synced_at = NOW() + WHERE server_id = $3`, + [scheduleId, 'SUCCESS', server.server_id] + ); + + return { success: true }; + } catch (err) { + const errorMsg = err.response?.data?.errors?.[0]?.detail || err.message; + await db.query( + `UPDATE server_restart_schedules + SET sync_status = $1, last_error = $2 + WHERE server_id = $3`, + ['FAILED', errorMsg, server.server_id] + ); + + return { success: false, error: errorMsg }; + } +} + +/** + * Find existing restart schedules NOT owned by Trinity + */ +async function findRogueRestarts(serverId) { + const url = `${PTERO_URL}/${serverId}/schedules?include=tasks`; + + const res = await axios.get(url, { headers }); + const rogueSchedules = []; + + for (const sched of res.data.data) { + // Ignore Trinity's own schedules + if (sched.attributes.name === "[Trinity] Daily Restart") continue; + + // Check if any task is a restart command + const hasRestart = sched.attributes.relationships.tasks.data.some( + task => task.attributes.action === 'power' && task.attributes.payload === 'restart' + ); + + if (hasRestart) { + rogueSchedules.push({ + id: sched.attributes.id, + name: sched.attributes.name, + cron: `${sched.attributes.minute} ${sched.attributes.hour} * * *` + }); + } + } + return rogueSchedules; +} + +/** + * Delete a rogue schedule and log the action + */ +async function deleteRogueSchedule(serverId, scheduleId, scheduleName, adminUser) { + const url = `${PTERO_URL}/${serverId}/schedules/${scheduleId}`; + + try { + await axios.delete(url, { headers }); + + await db.query( + `INSERT INTO sync_logs (server_id, action, status, error_message) + VALUES ($1, $2, $3, $4)`, + [serverId, `Deleted Rogue Schedule: ${scheduleName}`, 'SUCCESS', `Initiated by ${adminUser}`] + ); + return true; + } catch (err) { + console.error(`Failed to delete schedule ${scheduleId} on ${serverId}`, err.message); + return false; + } +} + +module.exports = { syncToPterodactyl, findRogueRestarts, deleteRogueSchedule }; +``` + +### 3. Express Routes + +**File:** `src/routes/admin/scheduler.js` + +```javascript +const express = require('express'); +const router = express.Router(); +const db = require('../../db'); +const { calculateStagger } = require('../../utils/scheduler'); +const { syncToPterodactyl, findRogueRestarts, deleteRogueSchedule } = require('../../lib/ptero-sync'); + +// GET: Main Scheduler Page +router.get('/', async (req, res) => { + const configs = await db.query('SELECT * FROM global_restart_config ORDER BY node'); + const servers = await db.query('SELECT * FROM server_restart_schedules ORDER BY node, sort_order'); + res.render('admin/scheduler', { configs: configs.rows, servers: servers.rows }); +}); + +// POST: Update Config (HTMX real-time preview) +router.post('/update-config', async (req, res) => { + const { node, base_time, interval_minutes } = req.body; + const admin = req.user?.username || 'The Wizard'; + + await db.query( + 'UPDATE global_restart_config SET base_time = $1, interval_minutes = $2, updated_by = $3, updated_at = NOW() WHERE node = $4', + [base_time, interval_minutes, admin, node] + ); + + const servers = await db.query('SELECT * FROM server_restart_schedules WHERE node = $1 ORDER BY sort_order', [node]); + const updated = calculateStagger(base_time, parseInt(interval_minutes), servers.rows); + + for (const s of updated) { + await db.query('UPDATE server_restart_schedules SET effective_time = $1 WHERE server_id = $2', [s.effective_time, s.server_id]); + } + + const allServers = await db.query('SELECT * FROM server_restart_schedules ORDER BY node, sort_order'); + res.render('admin/partials/scheduler-table', { servers: allServers.rows }); +}); + +// POST: Sync All Schedules to Pterodactyl +router.post('/sync-all', async (req, res) => { + const servers = await db.query('SELECT server_id FROM server_restart_schedules WHERE skip_restart = false'); + + for (const s of servers.rows) { + await syncToPterodactyl(s.server_id); + await new Promise(resolve => setTimeout(resolve, 200)); // Rate limit buffer + } + + const allServers = await db.query('SELECT * FROM server_restart_schedules ORDER BY node, sort_order'); + res.render('admin/partials/scheduler-table', { servers: allServers.rows }); +}); + +// POST: Reorder Servers (drag-and-drop) +router.post('/reorder-servers', async (req, res) => { + const { orderedIds } = req.body; + + try { + for (let i = 0; i < orderedIds.length; i++) { + await db.query('UPDATE server_restart_schedules SET sort_order = $1 WHERE server_id = $2', [i, orderedIds[i]]); + } + + // Recalculate effective times for all nodes + const nodes = await db.query('SELECT node, base_time, interval_minutes FROM global_restart_config'); + + for (const config of nodes.rows) { + const servers = await db.query('SELECT * FROM server_restart_schedules WHERE node = $1 ORDER BY sort_order', [config.node]); + const updated = calculateStagger(config.base_time, config.interval_minutes, servers.rows); + + for (const s of updated) { + await db.query('UPDATE server_restart_schedules SET effective_time = $1 WHERE server_id = $2', [s.effective_time, s.server_id]); + } + } + res.sendStatus(200); + } catch (error) { + console.error("Reorder failed:", error); + res.status(500).send("Failed to reorder servers."); + } +}); + +// GET: Table fragment for HTMX refresh +router.get('/table-only', async (req, res) => { + const allServers = await db.query('SELECT * FROM server_restart_schedules ORDER BY node, sort_order'); + res.render('admin/partials/scheduler-table', { servers: allServers.rows }); +}); + +// GET: Audit Modal for a specific node +router.get('/audit-modal/:node', async (req, res) => { + const { node } = req.params; + const servers = await db.query('SELECT server_id, server_name FROM server_restart_schedules WHERE node = $1', [node]); + + const auditResults = []; + let totalRogue = 0; + + for (const s of servers.rows) { + try { + const rogue = await findRogueRestarts(s.server_id); + if (rogue.length > 0) { + auditResults.push({ serverName: s.server_name, serverId: s.server_id, rogueSchedules: rogue }); + totalRogue += rogue.length; + } + } catch (err) { + console.error(`Audit failed for ${s.server_name}`, err.message); + } + await new Promise(resolve => setTimeout(resolve, 200)); + } + + res.render('admin/partials/audit-modal', { + results: auditResults, + node, + totalRogue, + serverCount: auditResults.length + }); +}); + +// POST: Nuke Rogue Schedules and Trigger Sync +router.post('/audit/nuke/:node', async (req, res) => { + const { node } = req.params; + const adminUser = req.user?.username || 'The Wizard'; + const schedulesToNuke = JSON.parse(req.body.nukeData); + + for (const item of schedulesToNuke) { + await deleteRogueSchedule(item.serverId, item.scheduleId, item.scheduleName, adminUser); + await new Promise(resolve => setTimeout(resolve, 200)); + } + + // Close modal and trigger sync via HTMX + res.set('HX-Trigger', 'start-global-sync'); + res.send(''); +}); + +module.exports = router; +``` + +### 4. Main View + +**File:** `src/views/admin/scheduler.ejs` + +```html +
+

Trinity Global Scheduler

+ + +
+ <% configs.forEach(config => { %> +
+ +

+ Node: <%= config.node %> +

+
+
+ + +
+
+ + +
+
+
+ <% }) %> +
+ + +
+ + + + +
+ + +
+ + +
+ <%- include('partials/scheduler-table', { servers: servers }) %> +
+ + + +
+ + + + +``` + +### 5. Table Partial + +**File:** `src/views/admin/partials/scheduler-table.ejs` + +```html + + + + + + + + + + + + + <% servers.forEach((server, i) => { %> + + + + + + + + + <% }) %> + +
ServerNodeRestart TimeSync StatusTimeline
+ ☰ + <%= server.server_name %> + + <%= server.node %> + + <%= server.effective_time %> + <% if (server.sync_status === 'SUCCESS') { %> + ● Synced + <% } else if (server.sync_status === 'FAILED') { %> + × Error + <% } else { %> + ○ Pending + <% } %> + +
+
+
+
+``` + +### 6. Audit Modal Partial + +**File:** `src/views/admin/partials/audit-modal.ejs` + +```html +
+
+ +

⚠ Action Required: Conflicts Detected

+ + <% if (totalRogue > 0) { %> +

+ Found <%= totalRogue %> rogue restart schedule(s) across + <%= serverCount %> server(s) on Node <%= node %>. + These must be removed before Trinity can take control. +

+ +
+
    + <% results.forEach(result => { %> +
  • + <%= result.serverName %> +
      + <% result.rogueSchedules.forEach(sched => { %> +
    • - "<%= sched.name %>" (Cron: <%= sched.cron %>)
      • + <% }) %> +
    +
    • + <% }) %> +
+
+ + <% + const nukePayload = []; + results.forEach(r => r.rogueSchedules.forEach(s => nukePayload.push({ + serverId: r.serverId, scheduleId: s.id, scheduleName: s.name + }))); + %> + +
+ + +
+ + +
+
+ + <% } else { %> +

No conflicts found on Node <%= node %>. The Frostwall is clear.

+
+ +
+ <% } %> +
+
+``` + +--- + +## Implementation Checklist + +### Phase 1: Database Setup +- [ ] Run SQL migration on arbiter_db +- [ ] Populate `server_restart_schedules` with all 21 servers from Pterodactyl + +### Phase 2: Backend +- [ ] Install dependency: `npm install date-fns` +- [ ] Create `src/utils/scheduler.js` +- [ ] Create `src/lib/ptero-sync.js` +- [ ] Create `src/routes/admin/scheduler.js` +- [ ] Add `PTERO_CLIENT_KEY` to environment (webuser_api key) +- [ ] Register route in main app: `app.use('/admin/scheduler', require('./routes/admin/scheduler'))` + +### Phase 3: Frontend +- [ ] Create `src/views/admin/scheduler.ejs` +- [ ] Create `src/views/admin/partials/scheduler-table.ejs` +- [ ] Create `src/views/admin/partials/audit-modal.ejs` +- [ ] Add link to Trinity Console sidebar + +### Phase 4: First Run +- [ ] Run Audit on TX1 — nuke any rogue restarts +- [ ] Run Audit on NC1 — nuke any rogue restarts +- [ ] Deploy schedules to Frostwall +- [ ] Verify in Pterodactyl that `[Trinity] Daily Restart` schedules exist + +### Phase 5: Verification +- [ ] Check a server's schedule in Pterodactyl panel +- [ ] Confirm cron time matches Trinity's effective_time +- [ ] Wait for first scheduled restart to confirm it works + +--- + +## API Notes + +### Pterodactyl Client API Quirks +1. **Uses POST for updates** — not PATCH +2. **8-character short ID required** — not full UUID +3. **Two-step schedule creation** — create schedule, then add task +4. **Rate limiting** — add 200ms delay between calls + +### Default Stagger Pattern +| Node | Base Time | Interval | Window | +|------|-----------|----------|--------| +| TX1 | 04:00 UTC | 5 min | ~50 min for 10 servers | +| NC1 | 04:30 UTC | 5 min | ~55 min for 11 servers | + +--- + +## Consultation Record + +This task was architected with Gemini AI on April 5, 2026. Full consultation covered: +- Source of truth architecture (DB vs API) +- UI/UX patterns for staggered visualization +- Conflict detection and resolution +- Drag-and-drop sorting +- Rate limiting and API quirks +- Audit trail logging + +--- + +**Fire + Frost + Foundation = Where Love Builds Legacy** 🔥❄️