{ "available": true, "mode": "observe", "status": "active", "note": "Bindings from a classified fault to a declared action in the inventory's automation catalogue. The console evaluates these and shows what would happen; scripts/jarvisos-remediate.mjs is the only thing that executes them.", "version": 1, "generatedAt": "2026-08-17", "at": "2026-08-18T07:01:46.058Z", "localNode": "vin", "policyStatement": { "autonomy": "A machine may run an action on its own only when the inventory declares that action \"safe\" — read-only, restartable, or reversible by doing nothing. Everything disruptive or destructive is proposed to a human with the exact command, its precondition and its undo. The ceiling is enforced by the validator, not by convention: a policy that binds a disruptive action in auto mode fails the build.", "verification": "An action is not a fix until the fault clears and holds. Every automatic attempt carries a verification window and a stability hold; if the entity is not ok by the end of the window the attempt is a failure, is rolled back where an undo exists, and escalates with the attempt record attached. Evidence-gathering steps declare verify.kind \"action-succeeded\": collecting replication state does not make anything better and must not claim to have.", "budgets": "Four independent bounds, because a remediation loop with one bound has none: attempts per incident, a cooldown between them, a per-entity and estate-wide budget inside a rolling hour, and one action in flight at a time. Above all of them sits a circuit breaker — three failures inside an hour suspends every automatic action for thirty minutes and escalates instead. Faults younger than the settle window are left alone entirely, so a single missed heartbeat can never start anything.", "escalation": "Escalation is not a notification. It carries the fault, the incident, the last known good state, every attempt and its outcome, what the loss costs in capability, and the exact next action with its precondition and its undo. A fault escalates when no policy covers it, when the action needs a credential, spending, an irreversible change or hands on hardware, when the node that would run it is unreachable, when attempts are exhausted, when an attempt ran and did not verify, or when the breaker is open.", "degradation": "Every gate is evaluated from state VIN already holds. Losing Home1 removes the actions that would have run there — they are refused as unreachable and escalated, naming what has to come back first — and removes the Home Assistant notification channel, which is declared best-effort and degrades to the local record. It does not remove the engine, the ledger, the policies, or any VIN-local remediation. Home Assistant is an input and a notifier here, never a gate.", "rollback": "Delete config/remediation.json. The console reports the engine as unloaded, the /remediation page says so, and nothing else changes: incidents, recovery tasks, the dependency graph and the action catalogue do not depend on it. To disable automatic execution without losing the analysis, stop the timer that runs scripts/jarvisos-remediate.mjs — the console has never executed anything and cannot start." }, "defaults": { "maxAttempts": 2, "cooldownSec": 300, "verifyWithinSec": 300, "verifyStableSec": 60, "settleSec": 120, "flapMax": 3 }, "safety": { "autoRiskCeiling": "safe", "requireRootCause": true, "requireHealthyPrereqs": true, "localNode": "vin" }, "budgets": { "windowSec": 3600, "used": 0, "limit": 6, "perEntityLimit": 2, "inFlight": 0, "maxConcurrent": 1, "breaker": { "open": false, "openedAt": null, "until": null, "failures": 0, "trips": 0, "resetAt": null } }, "classes": [ { "id": "credentials", "label": "Needs a credential a machine must not hold", "detail": "The action requires signing in to something — a vendor dashboard, an OAuth consent, an API token with write scope. Those credentials are operator-held by design; automating them would mean storing them where an automated remediation can reach them." }, { "id": "spend", "label": "Costs money", "detail": "Raising a quota, buying a drive, changing a plan. A machine must not decide to spend, however small the amount and however obvious the need." }, { "id": "irreversible", "label": "Cannot be undone", "detail": "Promotion, fencing, deletion. The estate cannot be returned to the state it was in before the action, so a person owns the decision." }, { "id": "physical", "label": "Needs hands on the hardware", "detail": "Power, cabling, a drive swap, a look at the UPS. Nothing in software gets past this one." }, { "id": "vendor", "label": "Only the vendor can fix it", "detail": "The fault is outside operator control. The action here is confirming it, applying the declared workaround and knowing when to stop trying." } ], "channels": [ { "id": "console", "label": "This console", "target": "/remediation and /incidents on VIN, loopback only", "required": true, "dependsOn": [], "degradesTo": null, "available": true, "blockers": [], "note": "The one channel that is up whenever anything is. It needs no network beyond the loopback interface and no other node." }, { "id": "ledger", "label": "Audit ledger", "target": "remediation.jsonl in JARVISOS_STATE_DIR on VIN, append-only, 0600", "required": true, "dependsOn": [], "degradesTo": null, "available": true, "blockers": [], "note": "The durable half. Every attempt, outcome and escalation lands here before anything tries to notify anybody." }, { "id": "home-assistant", "label": "Home Assistant notification", "target": "HA notify service, reached through the control plane on Home1", "required": false, "dependsOn": [ "home1" ], "degradesTo": "console", "available": true, "blockers": [], "note": "Best effort, and deliberately so: HA lives on the half of the estate most likely to be in the incident. When it is unavailable the escalation still exists on VIN and says that nobody was pushed a notification." } ], "policies": [ { "id": "worker-restart", "entityId": "jarvisos-worker", "entityName": "JarvisOS Worker (VIN)", "faults": [ "heartbeat-stale", "self-reported" ], "mode": "auto", "reason": "The flagship automatic case: stateless, VIN-local, declared safe, and the fault it fixes — a wedged worker — is the most common recoverable one in this estate. It also keeps working with the home rack dark, which is the point.", "action": { "id": "restart-worker", "label": "Restart the worker", "safety": "safe", "runsOn": "vin", "command": "systemctl restart jarvisos-worker", "requires": "Workers are stateless; in-flight tasks are at-least-once and re-run." }, "graphAction": "restart-worker", "verify": { "kind": "entity-ok", "withinSec": 300, "stableSec": 60 }, "limits": { "maxAttempts": 2, "cooldownSec": 300 }, "rollback": { "kind": "none", "reason": "A worker holds no state and is restarted, not changed. There is nothing to undo; if the restart does not clear the fault the escalation carries the attempt and its output." }, "human": { "classes": [ "irreversible" ], "action": "Read the worker's journal on VIN (`journalctl -u jarvisos-worker -n 200`). Two restarts have not fixed it, so this is a dispatch or control-plane problem rather than a wedged process." }, "requires": [], "available": { "runsOn": "vin", "local": true, "reachable": true, "blockedBy": [] } }, { "id": "offsite-sync-retry", "entityId": "gwscli-s3-offsite", "entityName": "Offsite backup replica", "faults": [ "self-reported", "heartbeat-stale" ], "mode": "auto", "reason": "Backups fail silently and the estate stops being recoverable without anything looking broken. Retrying costs bandwidth and nothing else, and the freshness of the offsite copy is the whole blast radius of losing the house.", "action": { "id": "retry-offsite-sync", "label": "Re-run the offsite sync job", "safety": "safe", "runsOn": "nas", "command": "jarvisos-agent remediate offsite-sync", "requires": "Additive upload only; it never deletes remote objects." }, "graphAction": "rerun-offsite-sync", "verify": { "kind": "entity-ok", "withinSec": 2700, "stableSec": 120 }, "limits": { "maxAttempts": 2, "cooldownSec": 3600 }, "rollback": { "kind": "none", "reason": "A sync is idempotent and additive: a failed run leaves the previous offsite copy exactly as it was. Undoing it would mean deleting a copy, which is the opposite of the point." }, "human": { "classes": [ "vendor", "credentials" ], "action": "Check the object-store credential (GWSCLI_S3 credentials in /etc/jarvisos/config.env) and the provider status page. Two retries failing usually means auth or the far end, not the transfer." }, "requires": [ "nas-backups", "isp-uplink" ], "available": { "runsOn": "nas", "local": false, "reachable": true, "blockedBy": [] } }, { "id": "home1-wake", "entityId": "home1", "entityName": "Home1", "faults": [ "heartbeat-stale", "no-signal" ], "mode": "auto", "reason": "A node that went to sleep and a node that lost power look identical from VIN, and one of them is free to fix. Sending the packet is safe, costs nothing, and either answers the question or narrows it to something physical.", "action": { "id": "wake-on-lan", "label": "Send Wake-on-LAN to the Home1 NIC", "safety": "safe", "runsOn": "nas", "command": "jarvisos-agent remediate wake-on-lan --target home1", "requires": "Something on the home LAN is still up to send it from." }, "graphAction": "restore-home1", "verify": { "kind": "entity-ok", "withinSec": 600, "stableSec": 120 }, "limits": { "maxAttempts": 2, "cooldownSec": 600 }, "rollback": { "kind": "none", "reason": "A magic packet either wakes a machine or is ignored. Nothing changes on any host, so there is nothing to undo." }, "human": { "classes": [ "physical" ], "action": "Go to the home rack. Check mains and the UPS event log before anything else, then the switch port LED, then `tailscale ping uk-james-home1` from VIN. Do not power-cycle until you have confirmed the host is not mid-write on the Postgres primary." }, "requires": [ "nas" ], "available": { "runsOn": "nas", "local": false, "reachable": true, "blockedBy": [] } }, { "id": "postgres-primary-evidence", "entityId": "postgres-primary", "entityName": "Postgres (primary)", "faults": [ "heartbeat-stale", "no-signal", "self-reported" ], "mode": "auto", "reason": "The one thing automation should do about a database outage is make the human decision a better-informed one. Promotion needs the lag figure, and reading it costs nothing and can be done while somebody is still walking to a keyboard.", "action": { "id": "collect-replication-state", "label": "Collect replication lag and WAL positions from both ends", "safety": "safe", "runsOn": "postgres-replica", "command": "jarvisos-agent remediate pg-replication-state", "requires": "Read-only. Run this before any promotion decision." }, "graphAction": null, "verify": { "kind": "action-succeeded", "withinSec": 120, "stableSec": 0 }, "limits": { "maxAttempts": 1, "cooldownSec": 900 }, "rollback": { "kind": "none", "reason": "Read-only. It collects lag and WAL positions and changes nothing on either end." }, "human": { "classes": [ "irreversible" ], "action": "Read the collected replication state before deciding anything. It is the number that decides whether promotion is a recovery or a data-loss event." }, "requires": [ "postgres-replica" ], "available": { "runsOn": "postgres-replica", "local": false, "reachable": true, "blockedBy": [] } }, { "id": "postgres-primary-fence", "entityId": "postgres-primary", "entityName": "Postgres (primary)", "faults": [ "heartbeat-stale", "no-signal" ], "mode": "escalate", "reason": "Destructive and ordering-sensitive. A machine that fences a database because it stopped answering has turned a maybe-network-fault into a certain outage.", "action": { "id": "fence-primary", "label": "Fence the old primary so it cannot accept writes", "safety": "destructive", "runsOn": "home1", "command": "jarvisos-agent remediate pg-fence --confirm", "requires": "A promotion has been decided on. Fencing before promotion is what prevents split-brain; fencing without one is a self-inflicted outage." }, "graphAction": null, "verify": { "kind": "entity-ok", "withinSec": 300, "stableSec": 60 }, "limits": { "maxAttempts": 2, "cooldownSec": 300 }, "rollback": { "kind": "automation", "automation": "unfence-primary", "reason": "Fencing is undone by unfencing, but only while the replica has NOT been promoted. After a promotion the undo becomes the split-brain the fence prevented, which is why the undo carries its own precondition." }, "human": { "classes": [ "irreversible", "credentials" ], "action": "Only if you have decided to promote: fence the old primary first with `jarvisos-agent remediate pg-fence --confirm` on home1. Fencing without a promotion decision is a self-inflicted outage; promoting without fencing is unrecoverable." }, "requires": [], "available": { "runsOn": "home1", "local": false, "reachable": true, "blockedBy": [] } }, { "id": "postgres-primary-promotion", "entityId": "postgres-primary", "entityName": "Postgres (primary)", "faults": [ "heartbeat-stale", "no-signal" ], "mode": "escalate", "reason": "The single most consequential action in this estate. Every transaction in the replication gap is lost by it, so the decision belongs to a person who has read the gap.", "action": { "id": "promote-replica", "label": "Promote the streaming replica to primary", "safety": "destructive", "runsOn": "postgres-replica", "command": "jarvisos-agent remediate pg-promote --confirm", "requires": "The old primary is fenced AND replication lag is acceptable. Promotion is one-way until a fresh base backup is taken." }, "graphAction": "promote-postgres-replica", "verify": { "kind": "entity-ok", "withinSec": 300, "stableSec": 60 }, "limits": { "maxAttempts": 2, "cooldownSec": 300 }, "rollback": { "kind": "none", "reason": "Promotion is one-way. The replica stops being a replica and the old primary must be rebuilt from a fresh base backup; there is no undo, only a rebuild." }, "human": { "classes": [ "irreversible" ], "action": "Confirm replication lag is at or near zero from the collected state, confirm the old primary is fenced, then run `jarvisos-agent remediate pg-promote --confirm` on postgres-replica and repoint jarvisos-control-plane at the new endpoint." }, "requires": [], "available": { "runsOn": "postgres-replica", "local": false, "reachable": true, "blockedBy": [] } }, { "id": "control-plane-restart", "entityId": "jarvisos-control-plane", "entityName": "JarvisOS Control Plane", "faults": [ "heartbeat-stale", "self-reported" ], "mode": "propose", "reason": "Disruptive, and the failure it usually indicates is Postgres rather than the process. Restarting it automatically would race the standby.", "action": { "id": "restart-control-plane", "label": "Restart the control plane on its current host", "safety": "disruptive", "runsOn": "home1", "command": "systemctl restart jarvisos-control-plane", "requires": "No standby is already accepting work. Two control planes against one database is the failure mode this guards against." }, "graphAction": null, "verify": { "kind": "entity-ok", "withinSec": 300, "stableSec": 60 }, "limits": { "maxAttempts": 1, "cooldownSec": 600 }, "rollback": { "kind": "none", "reason": "A restart has no undo. In-flight dispatches are lost and re-queued by the caller; that is the cost, and it is paid the moment the command runs." }, "human": { "classes": [ "irreversible" ], "action": "Run `systemctl restart jarvisos-control-plane` on home1 — but check first that the standby on VIN has not already taken over. Two control planes against one database is the failure this proposal exists to avoid." }, "requires": [], "available": { "runsOn": "home1", "local": false, "reachable": true, "blockedBy": [] } }, { "id": "auth-restart", "entityId": "jarvisos-auth", "entityName": "JarvisOS Auth (SSO)", "faults": [ "heartbeat-stale", "self-reported" ], "mode": "propose", "reason": "An auth outage looks like a total outage, which makes it exactly the fault where an automatic restart is most tempting and most likely to be aimed at the wrong thing.", "action": { "id": "restart-auth", "label": "Restart the auth service", "safety": "disruptive", "runsOn": "vin", "command": "systemctl restart jarvisos-auth", "requires": "Redis is reachable, otherwise the restart fails the same way it is failing now." }, "graphAction": "restart-auth", "verify": { "kind": "entity-ok", "withinSec": 300, "stableSec": 60 }, "limits": { "maxAttempts": 1, "cooldownSec": 600 }, "rollback": { "kind": "none", "reason": "Existing sessions survive a restart; anything mid-handshake is asked to sign in again. Neither is undoable and neither is worth undoing." }, "human": { "classes": [ "irreversible" ], "action": "Check redis first — auth failures here are usually the session store. If redis is healthy, run `systemctl restart jarvisos-auth` on VIN." }, "requires": [], "available": { "runsOn": "vin", "local": true, "reachable": true, "blockedBy": [] } }, { "id": "redis-restart", "entityId": "redis", "entityName": "Redis (cache / queue)", "faults": [ "heartbeat-stale", "self-reported" ], "mode": "propose", "reason": "Cheap to restart and disruptive to everything holding a session. Proposed rather than automatic because the fault is usually upstream of Redis.", "action": { "id": "restart-redis", "label": "Restart Redis", "safety": "disruptive", "runsOn": "home1", "command": "systemctl restart redis", "requires": "Signs every session out — jarvisos-auth stores sessions here. Durable data is unaffected." }, "graphAction": "restart-redis", "verify": { "kind": "entity-ok", "withinSec": 300, "stableSec": 60 }, "limits": { "maxAttempts": 1, "cooldownSec": 600 }, "rollback": { "kind": "none", "reason": "Contents are rebuildable, so a restart is cheap — but it is still a restart, and there is no undo for the queue that was in flight." }, "human": { "classes": [ "irreversible" ], "action": "Run `systemctl restart redis` on home1. If the queue is wedged rather than the process, flush-queue is the declared next step and it is destructive — read what is in it first." }, "requires": [], "available": { "runsOn": "home1", "local": false, "reachable": true, "blockedBy": [] } }, { "id": "wireguard-restart", "entityId": "wireguard-tunnel", "entityName": "WireGuard (Home1 <-> VIN)", "faults": [ "heartbeat-stale", "self-reported" ], "mode": "propose", "reason": "Restarting the private link while it is the only administrative path into the estate is a thing to do deliberately, with a second path confirmed.", "action": { "id": "restart-wireguard", "label": "Bounce the WireGuard interface on VIN", "safety": "disruptive", "runsOn": "vin", "command": "systemctl restart wg-quick@wg0", "requires": "Replication and heartbeats cut for the duration. Never combine this with a firewall change." }, "graphAction": "restart-wireguard", "verify": { "kind": "entity-ok", "withinSec": 300, "stableSec": 60 }, "limits": { "maxAttempts": 1, "cooldownSec": 600 }, "rollback": { "kind": "none", "reason": "The link comes back or it does not; there is no previous state to restore. Replication and worker dispatch both reconnect on their own." }, "human": { "classes": [ "irreversible" ], "action": "Confirm the far end is actually up before restarting the link — a tunnel to a dead node is a symptom, not the fault. Then restart it on VIN." }, "requires": [], "available": { "runsOn": "vin", "local": true, "reachable": true, "blockedBy": [] } }, { "id": "home1-agent-restart", "entityId": "home1", "entityName": "Home1", "faults": [ "self-reported" ], "mode": "propose", "reason": "The action most likely to be reached for and least likely to help. Proposing it puts that sentence in front of whoever is about to run it.", "action": { "id": "restart-agent", "label": "Restart the JarvisOS agent on Home1", "safety": "disruptive", "runsOn": "home1", "command": "systemctl restart jarvisos-agent", "requires": "The host answers SSH. This fixes a reporting fault, never a host fault." }, "graphAction": null, "verify": { "kind": "entity-ok", "withinSec": 300, "stableSec": 60 }, "limits": { "maxAttempts": 1, "cooldownSec": 600 }, "rollback": { "kind": "none", "reason": "Restarting the reporting agent has no undo and no effect on anything it reports about." }, "human": { "classes": [ "irreversible" ], "action": "Run `systemctl restart jarvisos-agent` on home1 over SSH. This fixes a reporting fault and never a host fault — if the host itself is unwell, this will make it look healthier without making it healthier." }, "requires": [], "available": { "runsOn": "home1", "local": false, "reachable": true, "blockedBy": [] } }, { "id": "caddy-config-reload", "entityId": "vin", "entityName": "VIN / VPS", "faults": [ "self-reported" ], "mode": "propose", "reason": "Every published hostname terminates here. The validation step is what makes it survivable, and a machine that skips it takes the whole public edge down.", "action": { "id": "reload-caddy", "label": "Validate and reload the Caddy configuration", "safety": "disruptive", "runsOn": "vin", "command": "caddy validate --config /etc/caddy/Caddyfile && systemctl reload caddy", "requires": "Validation passes first. A reload with a bad config drops every published route at once, and the Caddyfile is managed configuration that is never edited during an incident." }, "graphAction": "restart-caddy", "verify": { "kind": "entity-ok", "withinSec": 300, "stableSec": 60 }, "limits": { "maxAttempts": 1, "cooldownSec": 600 }, "rollback": { "kind": "none", "reason": "There is no undo declared for a reload, and deliberately so: the Caddyfile is managed configuration that is never edited during incident response, so the state before the reload is the state after it." }, "human": { "classes": [ "irreversible" ], "action": "Validate before reloading: `caddy validate --config /etc/caddy/Caddyfile && systemctl reload caddy` on VIN. A reload with a bad config drops every published route at once." }, "requires": [], "available": { "runsOn": "vin", "local": true, "reachable": true, "blockedBy": [] } }, { "id": "nas-pool-evidence", "entityId": "nas", "entityName": "NAS", "faults": [ "capacity/pool_health", "self-reported" ], "mode": "auto", "reason": "A degraded pool is not an outage and reacting to it as one is how a resilver becomes a data loss. The safe action is to find out which of the two it is.", "action": { "id": "pool-scrub-status", "label": "Collect pool status and SMART data", "safety": "safe", "runsOn": "nas", "command": "jarvisos-agent remediate pool-status", "requires": "Read-only. Safe at any time, including mid-resilver." }, "graphAction": null, "verify": { "kind": "action-succeeded", "withinSec": 300, "stableSec": 0 }, "limits": { "maxAttempts": 1, "cooldownSec": 3600 }, "rollback": { "kind": "none", "reason": "Read-only: it reports scrub and resilver state and touches nothing." }, "human": { "classes": [ "physical", "spend" ], "action": "Read the scrub state before touching the array. If a resilver is in progress the pool is working, not failing — do not power-cycle it. A failed drive needs a replacement drive, which is a purchase and a trip to the rack." }, "requires": [], "available": { "runsOn": "nas", "local": false, "reachable": true, "blockedBy": [] } }, { "id": "nas-pool-space", "entityId": "nas", "entityName": "NAS", "faults": [ "capacity/disk_pool" ], "mode": "escalate", "reason": "The one capacity fault where the obvious automatic fix — delete something — is irreversible and aimed at the dataset that makes the estate recoverable.", "action": null, "graphAction": null, "verify": { "kind": "entity-ok", "withinSec": 300, "stableSec": 60 }, "limits": { "maxAttempts": 2, "cooldownSec": 300 }, "rollback": { "kind": "none", "reason": "Freeing space by deleting data cannot be undone, which is precisely why no policy here does it." }, "human": { "classes": [ "irreversible", "spend" ], "action": "Confirm the reading on the host first — a wedged exporter reports a stuck value. Then free space from nas-media only, which policy accepts as re-acquirable, or add capacity. Never delete from nas-backups to make room." }, "requires": [], "available": null }, { "id": "cloudflare-dns-escalation", "entityId": "cloudflare-dns", "entityName": "Public DNS", "faults": [ "*" ], "mode": "escalate", "reason": "Public name resolution looks exactly like a total outage from outside and clients cache the failure. It is also entirely outside this estate: no action here is a machine's to take.", "action": null, "graphAction": null, "verify": { "kind": "entity-ok", "withinSec": 300, "stableSec": 60 }, "limits": { "maxAttempts": 2, "cooldownSec": 300 }, "rollback": { "kind": "none", "reason": "Nothing here changes anything, so there is nothing to undo. Re-pointing records is a human action in a dashboard and carries its own rollback: the previous record values, which are in the incident." }, "human": { "classes": [ "credentials", "vendor" ], "action": "Check the Cloudflare status page first. If the zone is the problem, sign in to the dashboard (the operator holds that credential; the token named in /etc/jarvisos/config.env is scoped for reads and cannot re-point a record) and compare the records against docs/RECOVERY.md#rb-cloudflare-dns. Propagation is bounded by the record TTL, not by how fast you fix it." }, "requires": [], "available": null }, { "id": "google-oauth-escalation", "entityId": "google-oauth", "entityName": "Google OAuth", "faults": [ "*" ], "mode": "escalate", "reason": "Sign-in failures are the loudest possible fault and this one has no local fix. Saying so immediately is worth more than any automation.", "action": null, "graphAction": null, "verify": { "kind": "entity-ok", "withinSec": 300, "stableSec": 60 }, "limits": { "maxAttempts": 2, "cooldownSec": 300 }, "rollback": { "kind": "none", "reason": "No action is taken, so there is nothing to undo." }, "human": { "classes": [ "vendor", "credentials" ], "action": "Confirm from a second network that it is the provider and not the estate. Existing sessions keep working; new sign-ins do not. If it is a consent or credential expiry rather than an outage, GOOGLE_CLIENT_SECRET in /etc/jarvisos/config.env is the operator-held credential that has to be rotated in the provider console." }, "requires": [], "available": null }, { "id": "isp-uplink-escalation", "entityId": "isp-uplink", "entityName": "ISP Uplink", "faults": [ "*" ], "mode": "escalate", "reason": "The uplink is outside operator control and the failure is usually confirmable in ten seconds by looking at a light. Automation has nothing to add.", "action": null, "graphAction": null, "verify": { "kind": "entity-ok", "withinSec": 300, "stableSec": 60 }, "limits": { "maxAttempts": 2, "cooldownSec": 300 }, "rollback": { "kind": "none", "reason": "No action is taken, so there is nothing to undo." }, "human": { "classes": [ "vendor", "physical" ], "action": "Check the router and the ONT lights at the house before calling anyone. VIN is on the other side of this link, so the console you are reading is unaffected and the tailnet may still be up — confirm which of the two paths is actually gone before reporting a fault." }, "requires": [], "available": null }, { "id": "gwscli-s3-escalation", "entityId": "gwscli-s3", "entityName": "GWSCLI / S3", "faults": [ "*" ], "mode": "escalate", "reason": "The one fault class where the fix costs money. A machine must never make that call, and the retry policy above must never be allowed to look like it is trying.", "action": null, "graphAction": null, "verify": { "kind": "entity-ok", "withinSec": 300, "stableSec": 60 }, "limits": { "maxAttempts": 2, "cooldownSec": 300 }, "rollback": { "kind": "none", "reason": "No action is taken, so there is nothing to undo." }, "human": { "classes": [ "spend", "vendor", "credentials" ], "action": "Check whether this is the provider or a quota. A quota is a purchase, which is a decision for the operator, not for a retry loop — and until it is made, the offsite copy stops advancing and the blast radius of losing the house grows by exactly that staleness." }, "requires": [], "available": null } ], "decisions": [], "attempts": [], "escalations": [], "resolvedEscalations": [], "notices": [], "coverage": [ { "entityId": "home1", "name": "Home1", "tier": "critical", "steps": [ { "id": "wake-on-lan", "label": "Send Wake-on-LAN to the Home1 NIC", "safety": "safe", "runsOn": "nas", "boundBy": [ "home1-wake" ] }, { "id": "restart-agent", "label": "Restart the JarvisOS agent on Home1", "safety": "disruptive", "runsOn": "home1", "boundBy": [ "home1-agent-restart" ] } ], "policies": [ { "id": "home1-wake", "mode": "auto", "faults": [ "heartbeat-stale", "no-signal" ], "automation": "wake-on-lan" }, { "id": "home1-agent-restart", "mode": "propose", "faults": [ "self-reported" ], "automation": "restart-agent" } ], "automated": true }, { "entityId": "vin", "name": "VIN / VPS", "tier": "critical", "steps": [ { "id": "reload-caddy", "label": "Validate and reload the Caddy configuration", "safety": "disruptive", "runsOn": "vin", "boundBy": [ "caddy-config-reload" ] } ], "policies": [ { "id": "caddy-config-reload", "mode": "propose", "faults": [ "self-reported" ], "automation": "reload-caddy" } ], "automated": false }, { "entityId": "nas", "name": "NAS", "tier": "critical", "steps": [ { "id": "pool-scrub-status", "label": "Collect pool status and SMART data", "safety": "safe", "runsOn": "nas", "boundBy": [ "nas-pool-evidence" ] } ], "policies": [ { "id": "nas-pool-evidence", "mode": "auto", "faults": [ "capacity/pool_health", "self-reported" ], "automation": "pool-scrub-status" }, { "id": "nas-pool-space", "mode": "escalate", "faults": [ "capacity/disk_pool" ], "automation": null } ], "automated": true }, { "entityId": "gwscli-s3", "name": "GWSCLI / S3", "tier": "important", "steps": [], "policies": [ { "id": "gwscli-s3-escalation", "mode": "escalate", "faults": [ "*" ], "automation": null } ], "automated": false }, { "entityId": "jarvisos-control-plane", "name": "JarvisOS Control Plane", "tier": "critical", "steps": [ { "id": "restart-control-plane", "label": "Restart the control plane on its current host", "safety": "disruptive", "runsOn": "home1", "boundBy": [ "control-plane-restart" ] } ], "policies": [ { "id": "control-plane-restart", "mode": "propose", "faults": [ "heartbeat-stale", "self-reported" ], "automation": "restart-control-plane" } ], "automated": false }, { "entityId": "jarvisos-worker", "name": "JarvisOS Worker (VIN)", "tier": "important", "steps": [ { "id": "restart-worker", "label": "Restart the worker", "safety": "safe", "runsOn": "vin", "boundBy": [ "worker-restart" ] } ], "policies": [ { "id": "worker-restart", "mode": "auto", "faults": [ "heartbeat-stale", "self-reported" ], "automation": "restart-worker" } ], "automated": true }, { "entityId": "postgres-primary", "name": "Postgres (primary)", "tier": "critical", "steps": [ { "id": "collect-replication-state", "label": "Collect replication lag and WAL positions from both ends", "safety": "safe", "runsOn": "postgres-replica", "boundBy": [ "postgres-primary-evidence" ] }, { "id": "fence-primary", "label": "Fence the old primary so it cannot accept writes", "safety": "destructive", "runsOn": "home1", "boundBy": [ "postgres-primary-fence" ] }, { "id": "unfence-primary", "label": "Remove the fence and let the old primary accept writes again", "safety": "disruptive", "runsOn": "home1", "boundBy": [ "postgres-primary-fence" ] }, { "id": "promote-replica", "label": "Promote the streaming replica to primary", "safety": "destructive", "runsOn": "postgres-replica", "boundBy": [ "postgres-primary-promotion" ] } ], "policies": [ { "id": "postgres-primary-evidence", "mode": "auto", "faults": [ "heartbeat-stale", "no-signal", "self-reported" ], "automation": "collect-replication-state" }, { "id": "postgres-primary-fence", "mode": "escalate", "faults": [ "heartbeat-stale", "no-signal" ], "automation": "fence-primary" }, { "id": "postgres-primary-promotion", "mode": "escalate", "faults": [ "heartbeat-stale", "no-signal" ], "automation": "promote-replica" } ], "automated": true }, { "entityId": "redis", "name": "Redis (cache / queue)", "tier": "important", "steps": [ { "id": "restart-redis", "label": "Restart Redis", "safety": "disruptive", "runsOn": "home1", "boundBy": [ "redis-restart" ] }, { "id": "flush-queue", "label": "Flush the task queue to clear a wedged job", "safety": "destructive", "runsOn": "home1", "boundBy": [] } ], "policies": [ { "id": "redis-restart", "mode": "propose", "faults": [ "heartbeat-stale", "self-reported" ], "automation": "restart-redis" } ], "automated": false }, { "entityId": "jarvisos-auth", "name": "JarvisOS Auth (SSO)", "tier": "critical", "steps": [ { "id": "restart-auth", "label": "Restart the auth service", "safety": "disruptive", "runsOn": "vin", "boundBy": [ "auth-restart" ] } ], "policies": [ { "id": "auth-restart", "mode": "propose", "faults": [ "heartbeat-stale", "self-reported" ], "automation": "restart-auth" } ], "automated": false }, { "entityId": "google-oauth", "name": "Google OAuth", "tier": "critical", "steps": [], "policies": [ { "id": "google-oauth-escalation", "mode": "escalate", "faults": [ "*" ], "automation": null } ], "automated": false }, { "entityId": "wireguard-tunnel", "name": "WireGuard (Home1 <-> VIN)", "tier": "critical", "steps": [ { "id": "restart-wireguard", "label": "Bounce the WireGuard interface on VIN", "safety": "disruptive", "runsOn": "vin", "boundBy": [ "wireguard-restart" ] } ], "policies": [ { "id": "wireguard-restart", "mode": "propose", "faults": [ "heartbeat-stale", "self-reported" ], "automation": "restart-wireguard" } ], "automated": false }, { "entityId": "cloudflare-dns", "name": "Public DNS", "tier": "critical", "steps": [], "policies": [ { "id": "cloudflare-dns-escalation", "mode": "escalate", "faults": [ "*" ], "automation": null } ], "automated": false }, { "entityId": "isp-uplink", "name": "ISP Uplink", "tier": "critical", "steps": [], "policies": [ { "id": "isp-uplink-escalation", "mode": "escalate", "faults": [ "*" ], "automation": null } ], "automated": false }, { "entityId": "gwscli-s3-offsite", "name": "Offsite backup replica", "tier": "critical", "steps": [ { "id": "retry-offsite-sync", "label": "Re-run the offsite sync job", "safety": "safe", "runsOn": "nas", "boundBy": [ "offsite-sync-retry" ] } ], "policies": [ { "id": "offsite-sync-retry", "mode": "auto", "faults": [ "self-reported", "heartbeat-stale" ], "automation": "retry-offsite-sync" } ], "automated": true } ], "findings": [], "summary": { "policies": 18, "autoPolicies": 5, "proposePolicies": 6, "escalatePolicies": 7, "attempts": 0, "verified": 0, "failed": 0, "rolledBack": 0, "inFlight": 0, "openEscalations": 0, "escalationsByClass": {}, "breakerOpen": false, "channelsAvailable": 3, "channels": 3, "decisions": 0, "cycles": 2 }, "source": "/app/config/remediation.json" }