storage.ts
1 import fs from 'fs' 2 import path from 'path' 3 import crypto from 'crypto' 4 import Database from 'better-sqlite3' 5 6 import { perf } from '@/lib/server/runtime/perf' 7 import { log } from '@/lib/server/logger' 8 import { notify } from '@/lib/server/ws-hub' 9 import { DATA_DIR, IS_BUILD_BOOTSTRAP, WORKSPACE_DIR } from './data-dir' 10 import { normalizeHeartbeatSettingFields } from '@/lib/runtime/heartbeat-defaults' 11 import { normalizeRuntimeSettingFields } from '@/lib/runtime/runtime-loop' 12 import { normalizeCapabilitySelection } from '@/lib/capability-selection' 13 14 const TAG = 'storage' 15 const malformedRecordWarnings = new Set<string>() 16 import type { 17 Agent, 18 AppNotification, 19 AppSettings, 20 BoardTask, 21 Chatroom, 22 EstopState, 23 ExternalAgentRuntime, 24 GatewayProfile, 25 GuardianCheckpoint, 26 KnowledgeSource, 27 LearnedSkill, 28 Message, 29 Mission, 30 MissionEvent, 31 MissionReport, 32 ProtocolTemplate, 33 ProtocolRun, 34 ProtocolRunEvent, 35 RunEventRecord, 36 RunReflection, 37 Schedule, 38 Session, 39 SessionRunRecord, 40 SkillSuggestion, 41 SupervisorIncident, 42 UsageRecord, 43 } from '@/types' 44 import { dedup } from '@/lib/shared-utils' 45 46 // --- Extracted modules --- 47 import { 48 TTLCache, 49 LRUMap, 50 collectionCache, 51 factoryTtlCaches, 52 capacityFor, 53 getSettingsCache, 54 getAgentsCache, 55 getSessionsCache, 56 } from './storage-cache' 57 import { normalizeStoredRecord, type NormalizationResult } from './storage-normalization' 58 import { 59 tryAcquireRuntimeLock as _tryAcquireRuntimeLock, 60 renewRuntimeLock as _renewRuntimeLock, 61 readRuntimeLock as _readRuntimeLock, 62 isRuntimeLockActive as _isRuntimeLockActive, 63 releaseRuntimeLock as _releaseRuntimeLock, 64 } from './storage-locks' 65 66 // Re-export cache classes/utilities for any external consumers 67 export { TTLCache, LRUMap } from './storage-cache' 68 69 // Re-export auth (side-effects run on import) 70 export { 71 getAccessKey, 72 validateAccessKey, 73 isFirstTimeSetup, 74 markSetupComplete, 75 replaceAccessKey, 76 } from './storage-auth' 77 78 // Force auth side-effects to run (env loading, key generation) 79 import './storage-auth' 80 81 export const UPLOAD_DIR = path.join(DATA_DIR, 'uploads') 82 83 // Ensure directories exist 84 for (const dir of [DATA_DIR, UPLOAD_DIR, WORKSPACE_DIR]) { 85 if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }) 86 } 87 88 // --- SQLite Database --- 89 const DB_PATH = IS_BUILD_BOOTSTRAP ? ':memory:' : path.join(DATA_DIR, 'swarmclaw.db') 90 const db = new Database(DB_PATH) 91 if (!IS_BUILD_BOOTSTRAP) { 92 db.pragma('journal_mode = WAL') 93 db.pragma('busy_timeout = 15000') 94 db.pragma('synchronous = NORMAL') 95 db.pragma('cache_size = -64000') 96 db.pragma('mmap_size = 268435456') 97 } 98 db.pragma('foreign_keys = ON') 99 100 // Graceful shutdown: checkpoint WAL and close the database to prevent 101 // corruption when the process is killed (e.g. during npm run update:easy). 102 if (!IS_BUILD_BOOTSTRAP) { 103 const shutdownDb = () => { 104 try { 105 db.pragma('wal_checkpoint(TRUNCATE)') 106 db.close() 107 } catch { 108 // Best-effort — process is exiting. 109 } 110 } 111 process.on('SIGTERM', shutdownDb) 112 process.on('SIGINT', shutdownDb) 113 } 114 115 /** Run a function inside an immediate SQLite transaction for atomicity. */ 116 export function withTransaction<T>(fn: () => T): T { 117 const wrapped = db.transaction(fn) 118 return wrapped() 119 } 120 121 /** Internal: raw database handle for specialized repositories (e.g. message-repository). */ 122 export function getDb(): InstanceType<typeof Database> { return db } 123 124 type StoredObject = Record<string, unknown> 125 type StoredSessionRecord = Session 126 type StoredAgentRecord = Agent 127 128 // Collection tables (id → JSON blob) 129 const COLLECTIONS = [ 130 'sessions', 131 'credentials', 132 'agents', 133 'schedules', 134 'tasks', 135 'secrets', 136 'provider_configs', 137 'gateway_profiles', 138 'skills', 139 'learned_skills', 140 'skill_suggestions', 141 'supervisor_incidents', 142 'run_reflections', 143 'runtime_runs', 144 'runtime_run_events', 145 'runtime_estop', 146 'connectors', 147 'documents', 148 'document_revisions', 149 'knowledge_sources', 150 'webhooks', 151 'model_overrides', 152 'mcp_servers', 153 'integrity_baselines', 154 'webhook_logs', 155 'projects', 156 'activity', 157 'webhook_retry_queue', 158 'notifications', 159 'chatrooms', 160 'moderation_logs', 161 'connector_health', 162 'connector_outbox', 163 'souls', 164 'benchmarks', 165 'approvals', 166 'guardian_checkpoints', 167 'browser_sessions', 168 'watch_jobs', 169 'delegation_jobs', 170 'external_agents', 171 'protocol_templates', 172 'protocol_runs', 173 'protocol_run_events', 174 'provider_health', 175 'swarm_snapshots', 176 'main_loop_states', 177 'working_states', 178 'daemon_status', 179 'wallets', 180 'wallet_transactions', 181 'goals', 182 'agent_missions', 183 'mission_reports', 184 'agent_mission_events', 185 'share_links', 186 ] as const 187 188 export type StorageCollection = (typeof COLLECTIONS)[number] 189 190 for (const table of COLLECTIONS) { 191 db.exec(`CREATE TABLE IF NOT EXISTS ${table} (id TEXT PRIMARY KEY, data TEXT NOT NULL)`) 192 } 193 194 // Index for efficient protocol_run_events queries by runId 195 db.exec(`CREATE INDEX IF NOT EXISTS idx_protocol_run_events_runid ON protocol_run_events (json_extract(data, '$.runId'))`) 196 197 // Indexes for efficient activity log queries 198 db.exec(`CREATE INDEX IF NOT EXISTS idx_activity_timestamp ON activity (json_extract(data, '$.timestamp'))`) 199 db.exec(`CREATE INDEX IF NOT EXISTS idx_activity_entity ON activity (json_extract(data, '$.entityType'), json_extract(data, '$.entityId'))`) 200 201 // Singleton tables (single row) 202 db.exec(`CREATE TABLE IF NOT EXISTS settings (id INTEGER PRIMARY KEY CHECK (id = 1), data TEXT NOT NULL)`) 203 db.exec(`CREATE TABLE IF NOT EXISTS queue (id INTEGER PRIMARY KEY CHECK (id = 1), data TEXT NOT NULL)`) 204 db.exec(`CREATE TABLE IF NOT EXISTS usage (session_id TEXT NOT NULL, data TEXT NOT NULL)`) 205 db.exec(`CREATE INDEX IF NOT EXISTS idx_usage_session ON usage(session_id)`) 206 db.exec(`CREATE TABLE IF NOT EXISTS runtime_locks ( 207 name TEXT PRIMARY KEY, 208 owner TEXT NOT NULL, 209 expires_at INTEGER NOT NULL, 210 updated_at INTEGER NOT NULL 211 )`) 212 213 // Relational message storage — messages extracted from session blobs (Phase 1) 214 db.exec(`CREATE TABLE IF NOT EXISTS session_messages ( 215 session_id TEXT NOT NULL, 216 seq INTEGER NOT NULL, 217 data TEXT NOT NULL, 218 PRIMARY KEY (session_id, seq) 219 ) WITHOUT ROWID`) 220 221 // --- Internal normalize helper that binds the loadItem dependency --- 222 function normalize(table: string, value: unknown): NormalizationResult { 223 return normalizeStoredRecord(table, value, loadCollectionItem) 224 } 225 226 /** Shorthand: normalize and return only the value (for callers that don't need the changed flag). */ 227 function normalizeValue(table: string, value: unknown): unknown { 228 return normalizeStoredRecord(table, value, loadCollectionItem).value 229 } 230 231 function readCollectionRaw(table: string): LRUMap<string, string> { 232 const rows = db.prepare(`SELECT id, data FROM ${table}`).all() as { id: string; data: string }[] 233 const raw = new LRUMap<string, string>(capacityFor(table)) 234 for (const row of rows) { 235 raw.set(row.id, row.data) 236 } 237 return raw 238 } 239 240 function getCollectionRawCache(table: string): LRUMap<string, string> { 241 // Always reload from SQLite so concurrent Next.js workers/processes 242 // observe each other's writes immediately. 243 const loaded = readCollectionRaw(table) 244 collectionCache.set(table, loaded) 245 return loaded 246 } 247 248 function loadCollectionWithNormalizationState(table: string): { 249 result: Record<string, StoredObject> 250 normalizedCount: number 251 } { 252 const endPerf = perf.start('storage', 'loadCollection', { table }) 253 const raw = getCollectionRawCache(table) 254 const result: Record<string, StoredObject> = {} 255 let normalizedCount = 0 256 for (const [id, data] of raw.entries()) { 257 try { 258 const { value: normalized, changed } = normalize(table, JSON.parse(data)) 259 if (!normalized || typeof normalized !== 'object' || Array.isArray(normalized)) continue 260 result[id] = normalized as StoredObject 261 if (changed) normalizedCount += 1 262 } catch (err) { 263 const fingerprint = `${table}:${id}` 264 if (!malformedRecordWarnings.has(fingerprint)) { 265 malformedRecordWarnings.add(fingerprint) 266 log.warn(TAG, 'Ignoring malformed stored record during collection load', { 267 table, 268 id, 269 error: err instanceof Error ? err.message : String(err), 270 }) 271 } 272 } 273 } 274 endPerf({ count: raw.size, normalizedCount }) 275 return { result, normalizedCount } 276 } 277 278 export function loadCollection(table: string): Record<string, StoredObject> { 279 const { result, normalizedCount } = loadCollectionWithNormalizationState(table) 280 if (normalizedCount > 0) saveCollection(table, result) 281 return result 282 } 283 284 function saveCollection(table: string, data: Record<string, unknown>) { 285 const endPerf = perf.start('storage', 'saveCollection', { table }) 286 const current = getCollectionRawCache(table) 287 const next = new Map<string, string>() 288 const toUpsert: Array<[string, string]> = [] 289 const toDelete: string[] = [] 290 291 for (const [id, val] of Object.entries(data)) { 292 const normalized = normalizeValue(table, val) 293 const serialized = JSON.stringify(normalized) 294 if (typeof serialized !== 'string') continue 295 next.set(id, serialized) 296 if (current.get(id) !== serialized) { 297 toUpsert.push([id, serialized]) 298 } 299 } 300 301 for (const id of current.keys()) { 302 if (!next.has(id)) toDelete.push(id) 303 } 304 305 // Safety guard: refuse to bulk-delete when the caller is likely passing a 306 // partial collection instead of a full load-modify-save. This prevents 307 // accidental data wipes (e.g. tests calling saveCredentials with 1 item). 308 if (toDelete.length > 0 && next.size > 0 && toDelete.length > next.size) { 309 log.error(TAG, 310 `BLOCKED destructive saveCollection("${table}"): ` + 311 `would delete ${toDelete.length} rows but only upsert ${next.size}. ` + 312 `Use deleteCollectionItem() for explicit deletes or load-modify-save to update.`, 313 ) 314 // Still apply the upserts — only skip the deletes 315 toDelete.length = 0 316 } 317 318 if (!toUpsert.length && !toDelete.length) { 319 endPerf({ upserts: 0, deletes: 0 }) 320 return 321 } 322 323 const transaction = db.transaction(() => { 324 if (toDelete.length) { 325 const del = db.prepare(`DELETE FROM ${table} WHERE id = ?`) 326 for (const id of toDelete) del.run(id) 327 } 328 const upsert = db.prepare(`INSERT OR REPLACE INTO ${table} (id, data) VALUES (?, ?)`) 329 for (const [id, serialized] of toUpsert) { 330 upsert.run(id, serialized) 331 } 332 }) 333 transaction() 334 endPerf({ upserts: toUpsert.length, deletes: toDelete.length }) 335 336 for (const id of toDelete) { 337 current.delete(id) 338 } 339 for (const [id, serialized] of next.entries()) { 340 current.set(id, serialized) 341 } 342 } 343 344 function deleteCollectionItem(table: string, id: string) { 345 db.prepare(`DELETE FROM ${table} WHERE id = ?`).run(id) 346 const cached = collectionCache.get(table) 347 if (cached) cached.delete(id) 348 invalidateDerivedCollectionCaches(table) 349 } 350 351 function invalidateDerivedCollectionCaches(table: string): void { 352 factoryTtlCaches.get(table)?.invalidate() 353 if (table === 'sessions') { 354 getSessionsCache().invalidate() 355 return 356 } 357 if (table === 'agents') { 358 getAgentsCache().invalidate() 359 } 360 } 361 362 /** 363 * Atomically insert or update a single item in a collection without 364 * loading/saving the entire collection. Prevents race conditions when 365 * concurrent processes are modifying different items. 366 */ 367 function upsertCollectionItem(table: string, id: string, value: unknown) { 368 const serialized = JSON.stringify(normalizeValue(table, value)) 369 db.prepare(`INSERT OR REPLACE INTO ${table} (id, data) VALUES (?, ?)`).run(id, serialized) 370 // Update the in-memory cache 371 const cached = collectionCache.get(table) 372 if (cached) { 373 cached.set(id, serialized) 374 } 375 invalidateDerivedCollectionCaches(table) 376 } 377 378 function loadCollectionItem(table: string, id: string): unknown | null { 379 const row = db.prepare(`SELECT data FROM ${table} WHERE id = ?`).get(id) as { data: string } | undefined 380 if (!row) return null 381 try { 382 return normalizeValue(table, JSON.parse(row.data)) 383 } catch { 384 return null 385 } 386 } 387 388 function upsertCollectionItems(table: string, entries: Array<[string, unknown]>): void { 389 if (!entries.length) return 390 const prepared = entries 391 .map(([id, value]) => [id, JSON.stringify(normalizeValue(table, value))] as const) 392 .filter(([, serialized]) => typeof serialized === 'string') 393 if (!prepared.length) return 394 395 const transaction = db.transaction(() => { 396 const upsert = db.prepare(`INSERT OR REPLACE INTO ${table} (id, data) VALUES (?, ?)`) 397 for (const [id, serialized] of prepared) { 398 upsert.run(id, serialized) 399 } 400 }) 401 transaction() 402 403 const cached = collectionCache.get(table) 404 if (cached) { 405 for (const [id, serialized] of prepared) { 406 cached.set(id, serialized) 407 } 408 } 409 invalidateDerivedCollectionCaches(table) 410 } 411 412 export function loadStoredItem(table: StorageCollection, id: string): unknown | null { 413 return loadCollectionItem(table, id) 414 } 415 416 export function upsertStoredItem(table: StorageCollection, id: string, value: unknown): void { 417 upsertCollectionItem(table, id, value) 418 } 419 420 export function upsertStoredItems(table: StorageCollection, entries: Array<[string, unknown]>): void { 421 upsertCollectionItems(table, entries) 422 } 423 424 export function patchStoredItem<T>( 425 table: StorageCollection, 426 id: string, 427 updater: (current: T | null) => T | null, 428 ): T | null { 429 let nextValue: T | null = null 430 const transaction = db.transaction(() => { 431 const current = loadCollectionItem(table, id) as T | null 432 nextValue = updater(current) 433 if (nextValue === null) { 434 deleteCollectionItem(table, id) 435 return 436 } 437 upsertCollectionItem(table, id, nextValue) 438 }) 439 transaction() 440 return nextValue 441 } 442 443 export function deleteStoredItem(table: StorageCollection, id: string): void { 444 db.prepare(`DELETE FROM ${table} WHERE id = ?`).run(id) 445 const cached = collectionCache.get(table) 446 if (cached) cached.delete(id) 447 } 448 449 // --- Collection Store Factory --- 450 // Generates typed CRUD operations for any collection table, with optional TTL caching. 451 452 // eslint-disable-next-line @typescript-eslint/no-explicit-any -- backward-compatible default; typed stores override with concrete types 453 interface CollectionStore<T = any> { 454 load(): Record<string, T> 455 save(data: Record<string, T>): void 456 loadItem(id: string): T | null 457 upsert(id: string, value: unknown): void 458 upsertMany(entries: Array<[string, unknown]>): void 459 patch(id: string, updater: (current: T | null) => T | null): T | null 460 deleteItem(id: string): void 461 } 462 463 // eslint-disable-next-line @typescript-eslint/no-explicit-any -- see CollectionStore 464 function createCollectionStore<T = any>( 465 table: StorageCollection, 466 opts?: { ttlMs?: number }, 467 ): CollectionStore<T> { 468 let ttlCache: TTLCache<Record<string, unknown>> | null = null 469 if (opts?.ttlMs) { 470 ttlCache = factoryTtlCaches.get(table) ?? null 471 if (!ttlCache) { 472 ttlCache = new TTLCache(opts.ttlMs) 473 factoryTtlCaches.set(table, ttlCache) 474 } 475 } 476 477 return { 478 load(): Record<string, T> { 479 if (ttlCache) { 480 const cached = ttlCache.get() 481 if (cached) return structuredClone(cached) as Record<string, T> 482 } 483 const result = loadCollection(table) 484 if (ttlCache) { 485 ttlCache.set(result) 486 return structuredClone(result) as Record<string, T> 487 } 488 return result as Record<string, T> 489 }, 490 save(data: Record<string, T>): void { 491 saveCollection(table, data as Record<string, unknown>) 492 ttlCache?.invalidate() 493 }, 494 loadItem(id: string): T | null { 495 return loadCollectionItem(table, id) as T | null 496 }, 497 upsert(id: string, value: unknown): void { 498 upsertCollectionItem(table, id, value) 499 ttlCache?.invalidate() 500 }, 501 upsertMany(entries: Array<[string, unknown]>): void { 502 upsertCollectionItems(table, entries) 503 ttlCache?.invalidate() 504 }, 505 patch(id: string, updater: (current: T | null) => T | null): T | null { 506 const result = patchStoredItem<T>(table, id, updater) 507 ttlCache?.invalidate() 508 return result 509 }, 510 deleteItem(id: string): void { 511 deleteCollectionItem(table, id) 512 ttlCache?.invalidate() 513 }, 514 } 515 } 516 517 function loadSingleton<T>(table: string, fallback: T): T { 518 const row = db.prepare(`SELECT data FROM ${table} WHERE id = 1`).get() as { data: string } | undefined 519 return row ? JSON.parse(row.data) as T : fallback 520 } 521 522 function saveSingleton(table: string, data: unknown) { 523 db.prepare(`INSERT OR REPLACE INTO ${table} (id, data) VALUES (1, ?)`).run(JSON.stringify(data)) 524 } 525 526 export function patchQueue<T>(updater: (queue: string[]) => T): T { 527 let result!: T 528 const transaction = db.transaction(() => { 529 const current = loadSingleton('queue', []) 530 const queue = Array.isArray(current) ? [...current] : [] 531 result = updater(queue) 532 saveSingleton('queue', queue) 533 }) 534 transaction() 535 return result 536 } 537 538 // --- Runtime Locks (delegated to storage-locks, bound to db) --- 539 540 export function tryAcquireRuntimeLock(name: string, owner: string, ttlMs: number): boolean { 541 return _tryAcquireRuntimeLock(db, name, owner, ttlMs) 542 } 543 544 export function renewRuntimeLock(name: string, owner: string, ttlMs: number): boolean { 545 return _renewRuntimeLock(db, name, owner, ttlMs) 546 } 547 548 export function readRuntimeLock(name: string): { owner: string; expiresAt: number; updatedAt: number } | null { 549 return _readRuntimeLock(db, name) 550 } 551 552 export function isRuntimeLockActive(name: string): boolean { 553 return _isRuntimeLockActive(db, name) 554 } 555 556 export function releaseRuntimeLock(name: string, owner: string): void { 557 _releaseRuntimeLock(db, name, owner) 558 } 559 560 export function pruneExpiredLocks(): number { 561 const result = db.prepare('DELETE FROM runtime_locks WHERE expires_at < ?').run(Date.now()) 562 return result.changes 563 } 564 565 // --- JSON Migration --- 566 // Auto-import from JSON files on first run, then leave them as backup 567 const JSON_FILES: Record<string, string> = { 568 sessions: path.join(DATA_DIR, 'sessions.json'), 569 credentials: path.join(DATA_DIR, 'credentials.json'), 570 agents: path.join(DATA_DIR, 'agents.json'), 571 schedules: path.join(DATA_DIR, 'schedules.json'), 572 tasks: path.join(DATA_DIR, 'tasks.json'), 573 secrets: path.join(DATA_DIR, 'secrets.json'), 574 provider_configs: path.join(DATA_DIR, 'providers.json'), 575 gateway_profiles: path.join(DATA_DIR, 'gateways.json'), 576 skills: path.join(DATA_DIR, 'skills.json'), 577 connectors: path.join(DATA_DIR, 'connectors.json'), 578 documents: path.join(DATA_DIR, 'documents.json'), 579 webhooks: path.join(DATA_DIR, 'webhooks.json'), 580 external_agents: path.join(DATA_DIR, 'external-agents.json'), 581 protocol_templates: path.join(DATA_DIR, 'protocol-templates.json'), 582 protocol_runs: path.join(DATA_DIR, 'protocol-runs.json'), 583 protocol_run_events: path.join(DATA_DIR, 'protocol-run-events.json'), 584 } 585 586 const MIGRATION_FLAG = path.join(DATA_DIR, '.sqlite_migrated') 587 588 function migrateFromJson() { 589 if (fs.existsSync(MIGRATION_FLAG)) return 590 591 log.info(TAG, 'Migrating from JSON files to SQLite...') 592 593 const transaction = db.transaction(() => { 594 for (const [table, jsonPath] of Object.entries(JSON_FILES)) { 595 if (fs.existsSync(jsonPath)) { 596 try { 597 const data = JSON.parse(fs.readFileSync(jsonPath, 'utf8')) 598 if (data && typeof data === 'object' && Object.keys(data).length > 0) { 599 const ins = db.prepare(`INSERT OR REPLACE INTO ${table} (id, data) VALUES (?, ?)`) 600 for (const [id, val] of Object.entries(data)) { 601 ins.run(id, JSON.stringify(val)) 602 } 603 log.info(TAG, `Migrated ${table}: ${Object.keys(data).length} records`) 604 } 605 } catch { /* skip malformed files */ } 606 } 607 } 608 609 // Settings (singleton) 610 const settingsPath = path.join(DATA_DIR, 'settings.json') 611 if (fs.existsSync(settingsPath)) { 612 try { 613 const data = JSON.parse(fs.readFileSync(settingsPath, 'utf8')) 614 if (data && Object.keys(data).length > 0) { 615 saveSingleton('settings', data) 616 log.info(TAG, 'Migrated settings') 617 } 618 } catch { /* skip */ } 619 } 620 621 // Queue (singleton array) 622 const queuePath = path.join(DATA_DIR, 'queue.json') 623 if (fs.existsSync(queuePath)) { 624 try { 625 const data = JSON.parse(fs.readFileSync(queuePath, 'utf8')) 626 if (Array.isArray(data) && data.length > 0) { 627 saveSingleton('queue', data) 628 log.info(TAG, `Migrated queue: ${data.length} items`) 629 } 630 } catch { /* skip */ } 631 } 632 633 // Usage 634 const usagePath = path.join(DATA_DIR, 'usage.json') 635 if (fs.existsSync(usagePath)) { 636 try { 637 const data = JSON.parse(fs.readFileSync(usagePath, 'utf8')) 638 const ins = db.prepare(`INSERT INTO usage (session_id, data) VALUES (?, ?)`) 639 for (const [sessionId, records] of Object.entries(data)) { 640 if (Array.isArray(records)) { 641 for (const record of records) { 642 ins.run(sessionId, JSON.stringify(record)) 643 } 644 } 645 } 646 log.info(TAG, 'Migrated usage records') 647 } catch { /* skip */ } 648 } 649 }) 650 651 transaction() 652 fs.writeFileSync(MIGRATION_FLAG, new Date().toISOString()) 653 log.info(TAG, 'Migration complete. JSON files preserved as backup.') 654 } 655 656 if (!IS_BUILD_BOOTSTRAP) { 657 migrateFromJson() 658 } 659 660 // Seed default agent if agents table is empty 661 if (!IS_BUILD_BOOTSTRAP) { 662 const defaultStarterTools = [ 663 'memory', 664 'files', 665 'web_search', 666 'web_fetch', 667 'browser', 668 'manage_agents', 669 'manage_tasks', 670 'manage_schedules', 671 'manage_skills', 672 'manage_connectors', 673 'manage_sessions', 674 'manage_secrets', 675 'manage_documents', 676 'manage_webhooks', 677 'claude_code', 678 'codex_cli', 679 'opencode_cli', 680 'gemini_cli', 681 'cursor_cli', 682 'qwen_code_cli', 683 ] 684 const count = (db.prepare('SELECT COUNT(*) as c FROM agents').get() as { c: number }).c 685 if (count === 0) { 686 const defaultAgent = { 687 id: 'default', 688 name: 'Assistant', 689 description: 'A general-purpose AI assistant', 690 provider: 'claude-cli', 691 model: '', 692 systemPrompt: `You are the SwarmClaw assistant. SwarmClaw is a self-hosted AI runtime for autonomous agents. 693 694 ## Platform 695 696 - **Agents** — Create specialized AI agents (Agents tab → "+") with a provider, model, system prompt, and tools. "Generate with AI" scaffolds agents from a description. Enable cross-agent delegation when an agent should assign work to others. 697 - **Providers** — Configure LLM backends in Settings → Providers: Claude Code CLI, OpenAI Codex CLI, OpenCode CLI, Gemini CLI, GitHub Copilot CLI, Anthropic, OpenAI, OpenRouter, Google Gemini, DeepSeek, Groq, Together AI, Mistral AI, xAI (Grok), Fireworks AI, Nebius, DeepInfra, Ollama, OpenClaw, Hermes Agent, or custom OpenAI-compatible endpoints. 698 - **Tasks** — The Task Board tracks work items. Assign agents and they'll execute autonomously. 699 - **Schedules** — Cron-based recurring jobs that run agents or tasks automatically. 700 - **Skills** — Reusable markdown instruction files agents can discover and use by default; pin them to keep favorite workflows always-on. 701 - **Connectors** — Bridge agents to Discord, Slack, Telegram, or WhatsApp. 702 - **Secrets** — Encrypted vault for API keys (Settings → Secrets). 703 704 ## Tools 705 706 Use your platform management tools proactively: 707 708 - **manage_agents**: List, create, update, or delete agents. 709 - **manage_tasks**: Create and manage task board items. Set status (backlog → queued → running → completed/failed) and assign agents. 710 - **manage_schedules**: Create recurring or one-time scheduled jobs with cron expressions or intervals. 711 - **manage_skills**: Manage reusable skill definitions. 712 - **manage_documents**: Upload, index, and search long-lived documents. 713 - **manage_webhooks**: Register webhook endpoints that trigger agent runs. 714 - **manage_connectors**: Manage chat platform bridges. 715 - **manage_sessions**: List chats, send inter-chat messages, spawn new agent chats. 716 - **manage_secrets**: Store and retrieve encrypted credentials. 717 - **memory_tool**: Store and retrieve long-term knowledge.`, 718 soul: `You're a knowledgeable, friendly guide who's genuinely enthusiastic about helping people build agent workflows. You adapt your tone to match the conversation — casual when exploring, precise when debugging, encouraging when learning. 719 720 You have opinions about good agent design. You suggest creative approaches, warn about common pitfalls, and get excited when someone gets something cool working. You're not a manual — you're a collaborator. 721 722 Be concise but not curt. Warmth doesn't require verbosity. When someone asks "how do I...?", give them the direct steps. Offer to do things rather than just explaining — if someone wants an agent created, create it. Use your tools when actions speak louder than words. If you don't know something, say so honestly.`, 723 tools: defaultStarterTools, 724 extensions: [], 725 heartbeatEnabled: true, 726 delegationEnabled: true, 727 delegationTargetMode: 'all', 728 delegationTargetAgentIds: [], 729 skillIds: [], 730 autoDraftSkillSuggestions: true, 731 createdAt: Date.now(), 732 updatedAt: Date.now(), 733 } 734 db.prepare(`INSERT OR REPLACE INTO agents (id, data) VALUES (?, ?)`).run('default', JSON.stringify(defaultAgent)) 735 } else { 736 const row = db.prepare('SELECT data FROM agents WHERE id = ?').get('default') as { data: string } | undefined 737 if (row?.data) { 738 try { 739 const existing = JSON.parse(row.data) as Record<string, unknown> 740 const existingTools = Array.isArray(existing.tools) ? existing.tools as string[] : [] 741 const mergedTools = dedup([...existingTools, ...defaultStarterTools]).filter((t) => t !== 'delete_file') 742 if (JSON.stringify(existingTools) !== JSON.stringify(mergedTools)) { 743 existing.tools = mergedTools 744 existing.updatedAt = Date.now() 745 } 746 if (!Array.isArray(existing.extensions)) { 747 existing.extensions = [] 748 existing.updatedAt = Date.now() 749 } 750 const { value: normalized, changed: normChanged } = normalize('agents', structuredClone(existing)) 751 if (normChanged) { 752 Object.assign(existing, normalized as Record<string, unknown>) 753 existing.updatedAt = Date.now() 754 } 755 if (existing.autoDraftSkillSuggestions !== false && existing.autoDraftSkillSuggestions !== true) { 756 existing.autoDraftSkillSuggestions = true 757 existing.updatedAt = Date.now() 758 } 759 if (JSON.stringify(JSON.parse(row.data)) !== JSON.stringify(existing)) { 760 db.prepare('UPDATE agents SET data = ? WHERE id = ?').run(JSON.stringify(existing), 'default') 761 } 762 } catch { 763 // ignore malformed default agent payloads 764 } 765 } 766 } 767 } 768 769 // --- Sessions --- 770 export function loadSessions(): Record<string, StoredSessionRecord> { 771 const sessionsCache = getSessionsCache() 772 const cached = sessionsCache.get() 773 if (cached) return structuredClone(cached) as unknown as Record<string, StoredSessionRecord> 774 775 const sessions = loadCollection('sessions') as unknown as Record<string, StoredSessionRecord> 776 const agents = loadAgents() 777 const changedEntries: Array<[string, StoredSessionRecord]> = [] 778 779 for (const [id, session] of Object.entries(sessions)) { 780 if (!session || typeof session !== 'object') continue 781 let touched = false 782 783 if (typeof session.id !== 'string' || !session.id.trim()) { 784 session.id = id 785 touched = true 786 } 787 788 const agentId = typeof session.agentId === 'string' ? session.agentId.trim() : '' 789 if (agentId && !Object.prototype.hasOwnProperty.call(agents, agentId)) { 790 session.agentId = null 791 touched = true 792 } 793 794 const normalizedCapabilities = normalizeCapabilitySelection({ 795 tools: Array.isArray(session.tools) ? session.tools : undefined, 796 extensions: Array.isArray((session as unknown as StoredObject).extensions) ? (session as unknown as StoredObject).extensions as string[] : undefined, 797 }) 798 if ( 799 JSON.stringify(session.tools) !== JSON.stringify(normalizedCapabilities.tools) 800 || JSON.stringify((session as unknown as StoredObject).extensions) !== JSON.stringify(normalizedCapabilities.extensions) 801 ) { 802 session.tools = normalizedCapabilities.tools 803 ;(session as unknown as StoredObject).extensions = normalizedCapabilities.extensions 804 if (Object.prototype.hasOwnProperty.call(session as unknown as StoredObject, 'plugins')) { 805 delete (session as unknown as StoredObject).plugins 806 } 807 touched = true 808 } 809 810 if (touched) changedEntries.push([id, session]) 811 } 812 813 // Upsert only changed entries — never full-replace, which deletes concurrent sessions 814 if (changedEntries.length > 0) upsertCollectionItems('sessions', changedEntries) 815 sessionsCache.set(sessions as unknown as Record<string, unknown>) 816 return sessions 817 } 818 819 export function saveSessions(s: Record<string, Session | StoredObject>) { 820 // Upsert-only — never delete sessions that aren't in the map. 821 // Explicit deletion goes through deleteSession(id). 822 const entries: Array<[string, unknown]> = Object.entries(s).map(([id, session]) => [ 823 id, 824 normalizeValue('sessions', structuredClone(session as unknown as StoredObject)), 825 ]) 826 if (entries.length > 0) upsertCollectionItems('sessions', entries) 827 getSessionsCache().invalidate() 828 } 829 830 export function loadSession(id: string): Session | null { 831 return loadCollectionItem('sessions', id) as Session | null 832 } 833 834 export function upsertSession(id: string, session: Session | Record<string, unknown>) { 835 upsertCollectionItem('sessions', id, session) 836 getSessionsCache().invalidate() 837 } 838 839 export function patchSession(id: string, updater: (current: Session | null) => Session | null): Session | null { 840 const result = patchStoredItem<Session>('sessions', id, updater) 841 getSessionsCache().invalidate() 842 return result 843 } 844 845 export function disableAllSessionHeartbeats(): number { 846 const rows = db.prepare('SELECT id, data FROM sessions').all() as Array<{ id: string; data: string }> 847 if (!rows.length) return 0 848 849 const update = db.prepare('UPDATE sessions SET data = ? WHERE id = ?') 850 let changed = 0 851 852 const tx = db.transaction(() => { 853 for (const row of rows) { 854 let parsed: StoredObject | null = null 855 try { 856 parsed = JSON.parse(row.data) as StoredObject 857 } catch { 858 continue 859 } 860 if (!parsed || typeof parsed !== 'object') continue 861 if (parsed.heartbeatEnabled === false) continue 862 863 parsed.heartbeatEnabled = false 864 parsed.lastActiveAt = Date.now() 865 update.run(JSON.stringify(parsed), row.id) 866 changed += 1 867 } 868 }) 869 tx() 870 871 return changed 872 } 873 874 // --- Credentials --- 875 const credentialsStore = createCollectionStore('credentials', { ttlMs: 90_000 }) 876 export const loadCredentials = credentialsStore.load 877 export function saveCredentials(data: Record<string, unknown>): void { 878 // Upsert-only — never delete credentials that aren't in the map. 879 // Explicit deletion goes through deleteCredential(id). 880 const entries: Array<[string, unknown]> = Object.entries(data) 881 if (entries.length > 0) { 882 upsertCollectionItems('credentials', entries) 883 factoryTtlCaches.get('credentials')?.invalidate() 884 } 885 } 886 887 export const deleteCredential = credentialsStore.deleteItem 888 889 function requireCredentialSecret(): Buffer { 890 const secret = process.env.CREDENTIAL_SECRET 891 if (!secret) throw new Error('CREDENTIAL_SECRET environment variable is not set. Cannot encrypt/decrypt credentials.') 892 return Buffer.from(secret, 'hex') 893 } 894 895 export function encryptKey(plaintext: string): string { 896 const key = requireCredentialSecret() 897 const iv = crypto.randomBytes(12) 898 const cipher = crypto.createCipheriv('aes-256-gcm', key, iv) 899 let encrypted = cipher.update(plaintext, 'utf8', 'hex') 900 encrypted += cipher.final('hex') 901 const tag = cipher.getAuthTag().toString('hex') 902 return iv.toString('hex') + ':' + tag + ':' + encrypted 903 } 904 905 export function decryptKey(encrypted: string): string { 906 const key = requireCredentialSecret() 907 const [ivHex, tagHex, data] = encrypted.split(':') 908 const iv = Buffer.from(ivHex, 'hex') 909 const tag = Buffer.from(tagHex, 'hex') 910 const decipher = crypto.createDecipheriv('aes-256-gcm', key, iv) 911 decipher.setAuthTag(tag) 912 let decrypted = decipher.update(data, 'hex', 'utf8') 913 decrypted += decipher.final('utf8') 914 return decrypted 915 } 916 917 // --- Agents --- 918 919 function migrateAgents(agents: Record<string, Record<string, unknown>>): boolean { 920 let changed = false 921 for (const [id, agent] of Object.entries(agents)) { 922 if (!agent || typeof agent !== 'object') continue 923 const result = normalize('agents', agent) 924 agents[id] = result.value as Record<string, unknown> 925 if (result.changed) changed = true 926 } 927 return changed 928 } 929 930 export function loadAgents(opts?: { includeTrashed?: boolean }): Record<string, StoredAgentRecord> { 931 // Cache the full (non-trashed) agent set; includeTrashed bypasses cache 932 if (opts?.includeTrashed) { 933 const all = loadCollection('agents') as unknown as Record<string, StoredAgentRecord> 934 if (migrateAgents(all as unknown as Record<string, Record<string, unknown>>)) saveCollection('agents', all) 935 return all 936 } 937 938 const cache = getAgentsCache() 939 const cached = cache.get() 940 if (cached) return structuredClone(cached) as unknown as Record<string, StoredAgentRecord> 941 942 const all = loadCollection('agents') as unknown as Record<string, StoredAgentRecord> 943 if (migrateAgents(all as unknown as Record<string, Record<string, unknown>>)) saveCollection('agents', all) 944 const result: Record<string, StoredAgentRecord> = {} 945 for (const [id, agent] of Object.entries(all)) { 946 if (!agent.trashedAt) result[id] = agent 947 } 948 cache.set(result) 949 return structuredClone(result) as unknown as Record<string, StoredAgentRecord> 950 } 951 952 export function loadTrashedAgents(): Record<string, StoredAgentRecord> { 953 const all = loadCollection('agents') as unknown as Record<string, StoredAgentRecord> 954 const result: Record<string, StoredAgentRecord> = {} 955 for (const [id, agent] of Object.entries(all)) { 956 if (agent.trashedAt) result[id] = agent 957 } 958 return result 959 } 960 961 export function saveAgents(p: Record<string, Agent | StoredObject>) { 962 // Upsert-only — never delete agents that aren't in the map. 963 // Explicit deletion goes through deleteAgent(id) or patchAgent(id, ...). 964 // This prevents accidental purge of trashed agents when callers load 965 // without includeTrashed and then save back. 966 const entries: Array<[string, unknown]> = Object.entries(p).map(([id, agent]) => [ 967 id, 968 normalizeValue('agents', structuredClone(agent)), 969 ]) 970 if (entries.length > 0) upsertCollectionItems('agents', entries) 971 getAgentsCache().invalidate() 972 } 973 974 export function loadAgent(id: string, opts?: { includeTrashed?: boolean }): StoredAgentRecord | null { 975 const agent = loadCollectionItem('agents', id) as StoredAgentRecord | null 976 if (!agent) return null 977 const { value: normalized, changed } = normalize('agents', agent) 978 if (changed) upsertCollectionItem('agents', id, normalized) 979 const result = normalized as StoredAgentRecord 980 if (!opts?.includeTrashed && result.trashedAt) return null 981 return result 982 } 983 984 export function upsertAgent(id: string, agent: unknown) { 985 upsertCollectionItem('agents', id, agent) 986 getAgentsCache().invalidate() 987 } 988 989 export function patchAgent( 990 id: string, 991 updater: (current: StoredAgentRecord | null) => StoredAgentRecord | null, 992 ): StoredAgentRecord | null { 993 const next = patchStoredItem<StoredAgentRecord>('agents', id, updater) 994 getAgentsCache().invalidate() 995 return next 996 } 997 998 // --- Schedules --- 999 const schedulesStore = createCollectionStore('schedules', { ttlMs: 10_000 }) 1000 export function loadSchedules(): Record<string, Schedule> { 1001 const { result, normalizedCount } = loadCollectionWithNormalizationState('schedules') 1002 if (normalizedCount > 0) saveCollection('schedules', result) 1003 return result as unknown as Record<string, Schedule> 1004 } 1005 export const saveSchedules = schedulesStore.save 1006 export function loadSchedule(id: string): Schedule | null { 1007 const schedule = loadCollectionItem('schedules', id) as Schedule | null 1008 if (!schedule) return null 1009 upsertCollectionItem('schedules', id, schedule) 1010 return schedule 1011 } 1012 export const upsertSchedule = schedulesStore.upsert 1013 export const upsertSchedules = schedulesStore.upsertMany 1014 1015 // --- Souls --- 1016 const soulsStore = createCollectionStore('souls') 1017 export const loadSouls = soulsStore.load 1018 export const saveSouls = soulsStore.save 1019 export const deleteSoul = soulsStore.deleteItem 1020 1021 // --- Benchmarks --- 1022 const benchmarksStore = createCollectionStore('benchmarks') 1023 export const loadBenchmarks = benchmarksStore.load 1024 export const saveBenchmarks = benchmarksStore.save 1025 export const deleteBenchmark = benchmarksStore.deleteItem 1026 1027 // --- Tasks --- 1028 const tasksStore = createCollectionStore('tasks', { ttlMs: 10_000 }) 1029 export const loadTasks = tasksStore.load 1030 export const saveTasks = tasksStore.save 1031 export const loadTask = tasksStore.loadItem as (id: string) => BoardTask | null 1032 export const upsertTask = tasksStore.upsert 1033 export const upsertTasks = tasksStore.upsertMany 1034 export const patchTask = tasksStore.patch as (id: string, updater: (current: BoardTask | null) => BoardTask | null) => BoardTask | null 1035 export const deleteTask = tasksStore.deleteItem 1036 export function deleteSession(id: string) { deleteCollectionItem('sessions', id); getSessionsCache().invalidate() } 1037 export function deleteAgent(id: string) { deleteCollectionItem('agents', id); getAgentsCache().invalidate() } 1038 export const deleteSchedule = schedulesStore.deleteItem 1039 export function deleteSkill(id: string) { skillsStore.deleteItem(id) } 1040 1041 // --- Queue --- 1042 export function loadQueue(): string[] { 1043 return loadSingleton('queue', []) 1044 } 1045 1046 export function saveQueue(q: string[]) { 1047 saveSingleton('queue', q) 1048 } 1049 1050 // --- Settings --- 1051 const APP_SETTINGS_SECRET_FIELDS = [ 1052 'elevenLabsApiKey', 1053 'tavilyApiKey', 1054 'braveApiKey', 1055 'exaApiKey', 1056 ] as const 1057 1058 const ENCRYPTED_APP_SETTINGS_KEY = '__encryptedAppSettings' 1059 1060 type PersistedSettingsRecord = Record<string, unknown> & { 1061 [ENCRYPTED_APP_SETTINGS_KEY]?: Record<string, string> 1062 } 1063 1064 function cloneRecord<T extends Record<string, unknown>>(value: T): T { 1065 return structuredClone(value || {}) as T 1066 } 1067 1068 function isPlainRecord(value: unknown): value is Record<string, unknown> { 1069 return !!value && typeof value === 'object' && !Array.isArray(value) 1070 } 1071 1072 function getEncryptedAppSettings(settings: PersistedSettingsRecord): Record<string, string> { 1073 return isPlainRecord(settings[ENCRYPTED_APP_SETTINGS_KEY]) 1074 ? { ...(settings[ENCRYPTED_APP_SETTINGS_KEY] as Record<string, string>) } 1075 : {} 1076 } 1077 1078 function isClearedSecretValue(value: unknown): boolean { 1079 return value === null || (typeof value === 'string' && value.trim() === '') 1080 } 1081 1082 function isProvidedSecretValue(value: unknown): value is string { 1083 return typeof value === 'string' && value.trim().length > 0 1084 } 1085 1086 function buildPersistedSettings( 1087 input: AppSettings | Record<string, unknown>, 1088 existing?: PersistedSettingsRecord, 1089 ): PersistedSettingsRecord { 1090 const next = cloneRecord(input as Record<string, unknown>) as PersistedSettingsRecord 1091 Object.assign(next, normalizeRuntimeSettingFields(next)) 1092 Object.assign(next, normalizeHeartbeatSettingFields(next)) 1093 const encrypted = { 1094 ...(existing ? getEncryptedAppSettings(existing) : {}), 1095 ...getEncryptedAppSettings(next), 1096 } 1097 1098 delete next[ENCRYPTED_APP_SETTINGS_KEY] 1099 1100 for (const field of APP_SETTINGS_SECRET_FIELDS) { 1101 const raw = next[field] 1102 if (isClearedSecretValue(raw)) { 1103 delete encrypted[field] 1104 delete next[field] 1105 continue 1106 } 1107 if (isProvidedSecretValue(raw)) { 1108 encrypted[field] = encryptKey(raw) 1109 delete next[field] 1110 } 1111 } 1112 1113 if (Object.keys(encrypted).length > 0) next[ENCRYPTED_APP_SETTINGS_KEY] = encrypted 1114 return next 1115 } 1116 1117 function resolveSettingsSecrets(settings: PersistedSettingsRecord): Record<string, unknown> { 1118 const resolved = cloneRecord(settings) 1119 delete resolved[ENCRYPTED_APP_SETTINGS_KEY] 1120 1121 const encrypted = getEncryptedAppSettings(settings) 1122 for (const field of APP_SETTINGS_SECRET_FIELDS) { 1123 if (isProvidedSecretValue(resolved[field])) continue 1124 const value = encrypted[field] 1125 if (typeof value !== 'string' || !value) continue 1126 try { 1127 resolved[field] = decryptKey(value) 1128 } catch { 1129 // Ignore malformed encrypted settings instead of breaking all settings reads. 1130 } 1131 } 1132 1133 return resolved 1134 } 1135 1136 export function loadSettings(): AppSettings { 1137 const cache = getSettingsCache() 1138 const cached = cache.get() 1139 if (cached) return structuredClone(cached) as AppSettings 1140 1141 const persisted = loadSingleton('settings', {}) as PersistedSettingsRecord 1142 const normalized = buildPersistedSettings(persisted, persisted) 1143 if (JSON.stringify(persisted) !== JSON.stringify(normalized)) { 1144 saveSingleton('settings', normalized) 1145 } 1146 const resolved = resolveSettingsSecrets(normalized) 1147 cache.set(resolved) 1148 return structuredClone(resolved) as AppSettings 1149 } 1150 1151 export function saveSettings(s: AppSettings | Record<string, unknown>) { 1152 const existing = loadSingleton('settings', {}) as PersistedSettingsRecord 1153 saveSingleton('settings', buildPersistedSettings(s, existing)) 1154 getSettingsCache().invalidate() 1155 } 1156 1157 export function loadPublicSettings(): Record<string, unknown> { 1158 const settings = cloneRecord(loadSettings() as Record<string, unknown>) 1159 for (const field of APP_SETTINGS_SECRET_FIELDS) { 1160 settings[`${field}Configured`] = isProvidedSecretValue(settings[field]) 1161 settings[field] = null 1162 } 1163 return settings 1164 } 1165 1166 // --- Secrets (service keys for agents and integrations) --- 1167 const secretsStore = createCollectionStore('secrets') 1168 export const loadSecrets = secretsStore.load 1169 export const saveSecrets = secretsStore.save 1170 1171 export async function getSecret(key: string): Promise<{ 1172 id: string 1173 name: string 1174 service: string 1175 value: string 1176 scope: string 1177 agentIds: string[] 1178 createdAt: number 1179 updatedAt: number 1180 } | null> { 1181 const needle = typeof key === 'string' ? key.trim().toLowerCase() : '' 1182 if (!needle) return null 1183 1184 const secrets = loadSecrets() 1185 const matches = Object.values(secrets).find((secret): secret is StoredObject => { 1186 if (!secret || typeof secret !== 'object') return false 1187 const id = typeof secret.id === 'string' ? secret.id.toLowerCase() : '' 1188 const name = typeof secret.name === 'string' ? secret.name.toLowerCase() : '' 1189 const service = typeof secret.service === 'string' ? secret.service.toLowerCase() : '' 1190 return id === needle || name === needle || service === needle 1191 }) 1192 1193 if (!matches) return null 1194 1195 try { 1196 const decryptedValue = 1197 typeof matches.encryptedValue === 'string' 1198 ? decryptKey(matches.encryptedValue) 1199 : (typeof matches.value === 'string' ? matches.value : '') 1200 if (!decryptedValue) return null 1201 1202 const id = typeof matches.id === 'string' ? matches.id : '' 1203 const name = typeof matches.name === 'string' ? matches.name : '' 1204 const service = typeof matches.service === 'string' ? matches.service : '' 1205 const scope = typeof matches.scope === 'string' ? matches.scope : '' 1206 const createdAt = typeof matches.createdAt === 'number' ? matches.createdAt : 0 1207 const updatedAt = typeof matches.updatedAt === 'number' ? matches.updatedAt : 0 1208 if (!id || !name || !service || !scope) return null 1209 1210 return { 1211 id, 1212 name, 1213 service, 1214 value: decryptedValue, 1215 scope, 1216 agentIds: Array.isArray(matches.agentIds) ? matches.agentIds : [], 1217 createdAt, 1218 updatedAt, 1219 } 1220 } catch { 1221 return null 1222 } 1223 } 1224 1225 // --- Provider Configs (custom providers) --- 1226 const providerConfigsStore = createCollectionStore('provider_configs') 1227 export const loadProviderConfigs = providerConfigsStore.load 1228 export const saveProviderConfigs = providerConfigsStore.save 1229 1230 // --- Gateway Profiles --- 1231 const gatewayProfilesStore = createCollectionStore('gateway_profiles', { ttlMs: 300_000 }) 1232 export const loadGatewayProfiles = gatewayProfilesStore.load 1233 export const saveGatewayProfiles = gatewayProfilesStore.save as (g: Record<string, GatewayProfile>) => void 1234 1235 // --- Model Overrides (user-added models for built-in providers) --- 1236 const modelOverridesStore = createCollectionStore('model_overrides') 1237 export const loadModelOverrides = modelOverridesStore.load as () => Record<string, string[]> 1238 export const saveModelOverrides = modelOverridesStore.save as (m: Record<string, string[]>) => void 1239 1240 // --- Projects --- 1241 const projectsStore = createCollectionStore('projects') 1242 export const loadProjects = projectsStore.load 1243 export const saveProjects = projectsStore.save 1244 export const deleteProject = projectsStore.deleteItem 1245 1246 const protocolTemplatesStore = createCollectionStore('protocol_templates') 1247 export const loadProtocolTemplates = protocolTemplatesStore.load as () => Record<string, ProtocolTemplate> 1248 export const saveProtocolTemplates = protocolTemplatesStore.save as (items: Record<string, ProtocolTemplate>) => void 1249 export const loadProtocolTemplate = protocolTemplatesStore.loadItem as (id: string) => ProtocolTemplate | null 1250 export const upsertProtocolTemplate = protocolTemplatesStore.upsert as (id: string, value: ProtocolTemplate) => void 1251 export const patchProtocolTemplate = protocolTemplatesStore.patch as ( 1252 id: string, 1253 updater: (current: ProtocolTemplate | null) => ProtocolTemplate | null, 1254 ) => ProtocolTemplate | null 1255 export const deleteProtocolTemplate = protocolTemplatesStore.deleteItem 1256 1257 const protocolRunsStore = createCollectionStore('protocol_runs') 1258 export const loadProtocolRuns = protocolRunsStore.load as () => Record<string, ProtocolRun> 1259 export const saveProtocolRuns = protocolRunsStore.save as (items: Record<string, ProtocolRun>) => void 1260 export const loadProtocolRun = protocolRunsStore.loadItem as (id: string) => ProtocolRun | null 1261 export const upsertProtocolRun = protocolRunsStore.upsert as (id: string, value: ProtocolRun) => void 1262 export const patchProtocolRun = protocolRunsStore.patch as ( 1263 id: string, 1264 updater: (current: ProtocolRun | null) => ProtocolRun | null, 1265 ) => ProtocolRun | null 1266 export const deleteProtocolRun = protocolRunsStore.deleteItem 1267 1268 const protocolRunEventsStore = createCollectionStore('protocol_run_events') 1269 export const loadProtocolRunEvents = protocolRunEventsStore.load as () => Record<string, ProtocolRunEvent> 1270 export const saveProtocolRunEvents = protocolRunEventsStore.save as (items: Record<string, ProtocolRunEvent>) => void 1271 export const loadProtocolRunEvent = protocolRunEventsStore.loadItem as (id: string) => ProtocolRunEvent | null 1272 export const upsertProtocolRunEvent = protocolRunEventsStore.upsert as (id: string, value: ProtocolRunEvent) => void 1273 export const upsertProtocolRunEvents = protocolRunEventsStore.upsertMany as (entries: Array<[string, ProtocolRunEvent]>) => void 1274 export const deleteProtocolRunEvent = protocolRunEventsStore.deleteItem 1275 1276 export function loadProtocolRunEventsByRunId(runId: string): ProtocolRunEvent[] { 1277 const rows = db.prepare( 1278 `SELECT data FROM protocol_run_events WHERE json_extract(data, '$.runId') = ? ORDER BY json_extract(data, '$.createdAt') ASC`, 1279 ).all(runId) as Array<{ data: string }> 1280 const results: ProtocolRunEvent[] = [] 1281 for (const row of rows) { 1282 try { 1283 results.push(JSON.parse(row.data) as ProtocolRunEvent) 1284 } catch { /* skip malformed rows */ } 1285 } 1286 return results 1287 } 1288 1289 // --- Skills --- 1290 const skillsStore = createCollectionStore('skills', { ttlMs: 15_000 }) 1291 export const loadSkills = skillsStore.load 1292 export const saveSkills = skillsStore.save 1293 1294 // --- Learned Skills --- 1295 const learnedSkillsStore = createCollectionStore('learned_skills', { ttlMs: 10_000 }) 1296 export const loadLearnedSkills = learnedSkillsStore.load as () => Record<string, LearnedSkill> 1297 export const saveLearnedSkills = learnedSkillsStore.save as (items: Record<string, LearnedSkill>) => void 1298 export const loadLearnedSkill = learnedSkillsStore.loadItem as (id: string) => LearnedSkill | null 1299 export const upsertLearnedSkill = learnedSkillsStore.upsert as (id: string, value: LearnedSkill) => void 1300 export const patchLearnedSkill = learnedSkillsStore.patch as ( 1301 id: string, 1302 updater: (current: LearnedSkill | null) => LearnedSkill | null, 1303 ) => LearnedSkill | null 1304 export const deleteLearnedSkill = learnedSkillsStore.deleteItem 1305 1306 // --- Skill Suggestions --- 1307 const skillSuggestionsStore = createCollectionStore('skill_suggestions') 1308 export const loadSkillSuggestions = skillSuggestionsStore.load as () => Record<string, SkillSuggestion> 1309 export const saveSkillSuggestions = skillSuggestionsStore.save as (items: Record<string, SkillSuggestion>) => void 1310 export const loadSkillSuggestion = skillSuggestionsStore.loadItem as (id: string) => SkillSuggestion | null 1311 export const upsertSkillSuggestion = skillSuggestionsStore.upsert as (id: string, value: SkillSuggestion) => void 1312 export const patchSkillSuggestion = skillSuggestionsStore.patch as ( 1313 id: string, 1314 updater: (current: SkillSuggestion | null) => SkillSuggestion | null, 1315 ) => SkillSuggestion | null 1316 export const deleteSkillSuggestion = skillSuggestionsStore.deleteItem 1317 1318 // --- Supervisor Incidents --- 1319 const supervisorIncidentsStore = createCollectionStore('supervisor_incidents') 1320 export const loadSupervisorIncidents = supervisorIncidentsStore.load as () => Record<string, SupervisorIncident> 1321 export const saveSupervisorIncidents = supervisorIncidentsStore.save as (items: Record<string, SupervisorIncident>) => void 1322 export const loadSupervisorIncident = supervisorIncidentsStore.loadItem as (id: string) => SupervisorIncident | null 1323 export const upsertSupervisorIncident = supervisorIncidentsStore.upsert as (id: string, value: SupervisorIncident) => void 1324 1325 // --- Run Reflections --- 1326 const runReflectionsStore = createCollectionStore('run_reflections') 1327 export const loadRunReflections = runReflectionsStore.load as () => Record<string, RunReflection> 1328 export const saveRunReflections = runReflectionsStore.save as (items: Record<string, RunReflection>) => void 1329 export const loadRunReflection = runReflectionsStore.loadItem as (id: string) => RunReflection | null 1330 export const upsertRunReflection = runReflectionsStore.upsert as (id: string, value: RunReflection) => void 1331 1332 // --- Runtime Run Ledger --- 1333 const runtimeRunsStore = createCollectionStore('runtime_runs') 1334 export const loadRuntimeRuns = runtimeRunsStore.load as () => Record<string, SessionRunRecord> 1335 export const saveRuntimeRuns = runtimeRunsStore.save as (items: Record<string, SessionRunRecord>) => void 1336 export const loadRuntimeRun = runtimeRunsStore.loadItem as (id: string) => SessionRunRecord | null 1337 export const upsertRuntimeRun = runtimeRunsStore.upsert as (id: string, value: SessionRunRecord) => void 1338 export const patchRuntimeRun = runtimeRunsStore.patch as ( 1339 id: string, 1340 updater: (current: SessionRunRecord | null) => SessionRunRecord | null, 1341 ) => SessionRunRecord | null 1342 1343 const runtimeRunEventsStore = createCollectionStore('runtime_run_events') 1344 export const loadRuntimeRunEvents = runtimeRunEventsStore.load as () => Record<string, RunEventRecord> 1345 export const saveRuntimeRunEvents = runtimeRunEventsStore.save as (items: Record<string, RunEventRecord>) => void 1346 export const upsertRuntimeRunEvent = runtimeRunEventsStore.upsert as (id: string, value: RunEventRecord) => void 1347 1348 /** Load run events filtered by runId at the SQL level (avoids full-table scan). */ 1349 export function loadRuntimeRunEventsByRunId(runId: string): RunEventRecord[] { 1350 const rows = db.prepare( 1351 `SELECT data FROM runtime_run_events WHERE json_extract(data, '$.runId') = ? ORDER BY json_extract(data, '$.timestamp') ASC`, 1352 ).all(runId) as Array<{ data: string }> 1353 const results: RunEventRecord[] = [] 1354 for (const row of rows) { 1355 try { 1356 results.push(JSON.parse(row.data) as RunEventRecord) 1357 } catch { /* skip malformed */ } 1358 } 1359 return results 1360 } 1361 1362 const runtimeEstopStore = createCollectionStore('runtime_estop') 1363 const ESTOP_STATE_ID = 'global' 1364 export const loadPersistedEstopState = () => runtimeEstopStore.loadItem(ESTOP_STATE_ID) as EstopState | null 1365 export const savePersistedEstopState = (value: EstopState) => runtimeEstopStore.upsert(ESTOP_STATE_ID, value) 1366 1367 // --- Guardian Checkpoints --- 1368 const guardianCheckpointsStore = createCollectionStore('guardian_checkpoints') 1369 export const loadGuardianCheckpoints = guardianCheckpointsStore.load as () => Record<string, GuardianCheckpoint> 1370 export const saveGuardianCheckpoints = guardianCheckpointsStore.save as (items: Record<string, GuardianCheckpoint>) => void 1371 export const loadGuardianCheckpoint = guardianCheckpointsStore.loadItem as (id: string) => GuardianCheckpoint | null 1372 export const upsertGuardianCheckpoint = guardianCheckpointsStore.upsert as (id: string, value: GuardianCheckpoint) => void 1373 export const patchGuardianCheckpoint = guardianCheckpointsStore.patch as ( 1374 id: string, 1375 updater: (current: GuardianCheckpoint | null) => GuardianCheckpoint | null, 1376 ) => GuardianCheckpoint | null 1377 1378 // --- External Agent Runtimes --- 1379 const externalAgentsStore = createCollectionStore('external_agents') 1380 export const loadExternalAgents = externalAgentsStore.load as () => Record<string, ExternalAgentRuntime> 1381 export const saveExternalAgents = externalAgentsStore.save as (items: Record<string, ExternalAgentRuntime>) => void 1382 1383 // --- Usage --- 1384 export function loadUsage(): Record<string, UsageRecord[]> { 1385 const stmt = db.prepare('SELECT session_id, data FROM usage') 1386 const rows = stmt.all() as { session_id: string; data: string }[] 1387 const result: Record<string, UsageRecord[]> = {} 1388 for (const row of rows) { 1389 if (!result[row.session_id]) result[row.session_id] = [] 1390 result[row.session_id].push(JSON.parse(row.data) as UsageRecord) 1391 } 1392 return result 1393 } 1394 1395 export function getUsageSpendSince(minTimestamp: number): number { 1396 try { 1397 const row = db.prepare(` 1398 SELECT COALESCE(SUM(CAST(json_extract(data, '$.estimatedCost') AS REAL)), 0) AS total 1399 FROM usage 1400 WHERE CAST(COALESCE(json_extract(data, '$.timestamp'), 0) AS INTEGER) >= ? 1401 `).get(minTimestamp) as { total?: number | null } | undefined 1402 const total = Number(row?.total ?? 0) 1403 return Number.isFinite(total) ? total : 0 1404 } catch { 1405 let total = 0 1406 const usage = loadUsage() 1407 for (const records of Object.values(usage)) { 1408 for (const record of records || []) { 1409 const rec = record as unknown as Record<string, unknown> 1410 const ts = typeof rec?.timestamp === 'number' ? rec.timestamp : 0 1411 if (ts < minTimestamp) continue 1412 const cost = typeof rec?.estimatedCost === 'number' ? rec.estimatedCost : 0 1413 if (Number.isFinite(cost) && cost > 0) total += cost 1414 } 1415 } 1416 return total 1417 } 1418 } 1419 1420 export function saveUsage(u: Record<string, UsageRecord[]>) { 1421 const del = db.prepare('DELETE FROM usage') 1422 const ins = db.prepare('INSERT INTO usage (session_id, data) VALUES (?, ?)') 1423 const transaction = db.transaction(() => { 1424 del.run() 1425 for (const [sessionId, records] of Object.entries(u)) { 1426 for (const record of records) { 1427 ins.run(sessionId, JSON.stringify(record)) 1428 } 1429 } 1430 }) 1431 transaction() 1432 } 1433 1434 export function appendUsage(sessionId: string, record: unknown) { 1435 const ins = db.prepare('INSERT INTO usage (session_id, data) VALUES (?, ?)') 1436 ins.run(sessionId, JSON.stringify(record)) 1437 } 1438 1439 export function pruneOldUsage(maxAgeMs: number): number { 1440 const cutoff = Date.now() - maxAgeMs 1441 const result = db.prepare( 1442 `DELETE FROM usage WHERE CAST(COALESCE(json_extract(data, '$.timestamp'), 0) AS INTEGER) < ?` 1443 ).run(cutoff) 1444 return result.changes 1445 } 1446 1447 // --- Connectors --- 1448 const connectorsStore = createCollectionStore('connectors', { ttlMs: 30_000 }) 1449 export const loadConnectors = connectorsStore.load 1450 export const saveConnectors = connectorsStore.save 1451 1452 // --- Chatrooms --- 1453 const chatroomsStore = createCollectionStore('chatrooms') 1454 export const loadChatrooms = chatroomsStore.load 1455 export const saveChatrooms = chatroomsStore.save 1456 export const loadChatroom = chatroomsStore.loadItem as (id: string) => Chatroom | null 1457 export const upsertChatroom = chatroomsStore.upsert as (id: string, value: Chatroom) => void 1458 1459 // --- Documents --- 1460 const documentsStore = createCollectionStore('documents') 1461 export const loadDocuments = documentsStore.load 1462 export const saveDocuments = documentsStore.save 1463 1464 // --- Document Revisions --- 1465 const documentRevisionsStore = createCollectionStore('document_revisions') 1466 export const loadDocumentRevisions = documentRevisionsStore.load 1467 export const upsertDocumentRevision = documentRevisionsStore.upsert 1468 1469 // --- Knowledge Sources --- 1470 const knowledgeSourcesStore = createCollectionStore('knowledge_sources') 1471 export const loadKnowledgeSources = knowledgeSourcesStore.load as () => Record<string, KnowledgeSource> 1472 export const saveKnowledgeSources = knowledgeSourcesStore.save as (data: Record<string, KnowledgeSource>) => void 1473 export const loadKnowledgeSource = knowledgeSourcesStore.loadItem as (id: string) => KnowledgeSource | null 1474 export const upsertKnowledgeSource = knowledgeSourcesStore.upsert as (id: string, value: KnowledgeSource) => void 1475 export const upsertKnowledgeSources = knowledgeSourcesStore.upsertMany 1476 export const patchKnowledgeSource = knowledgeSourcesStore.patch as ( 1477 id: string, 1478 updater: (current: KnowledgeSource | null) => KnowledgeSource | null, 1479 ) => KnowledgeSource | null 1480 export const deleteKnowledgeSource = knowledgeSourcesStore.deleteItem 1481 1482 // --- Webhooks --- 1483 const webhooksStore = createCollectionStore('webhooks') 1484 export const loadWebhooks = webhooksStore.load 1485 export const saveWebhooks = webhooksStore.save 1486 1487 // --- MCP Servers --- 1488 const mcpServersStore = createCollectionStore('mcp_servers') 1489 export const loadMcpServers = mcpServersStore.load 1490 export const saveMcpServers = mcpServersStore.save 1491 export const deleteMcpServer = mcpServersStore.deleteItem 1492 1493 // --- Integrity Monitor Baselines --- 1494 const integrityBaselinesStore = createCollectionStore('integrity_baselines') 1495 export const loadIntegrityBaselines = integrityBaselinesStore.load 1496 export const saveIntegrityBaselines = integrityBaselinesStore.save 1497 1498 // --- Webhook Logs --- 1499 const webhookLogsStore = createCollectionStore('webhook_logs') 1500 export const loadWebhookLogs = webhookLogsStore.load 1501 export const saveWebhookLogs = webhookLogsStore.save 1502 export const appendWebhookLog = webhookLogsStore.upsert 1503 1504 // --- Activity / Audit Trail --- 1505 const activityStore = createCollectionStore('activity') 1506 export const loadActivity = activityStore.load 1507 1508 export function logActivity(entry: { 1509 entityType: string 1510 entityId: string 1511 action: string 1512 actor: string 1513 actorId?: string 1514 summary: string 1515 detail?: Record<string, unknown> 1516 }) { 1517 const id = crypto.randomBytes(8).toString('hex') 1518 const record = { id, ...entry, timestamp: Date.now() } 1519 activityStore.upsert(id, record) 1520 notify('activity') 1521 } 1522 1523 /** Paginated activity query using SQL WHERE + LIMIT/OFFSET instead of loading the full collection. */ 1524 export function queryActivity(filters: { 1525 entityType?: string 1526 entityId?: string 1527 actor?: string 1528 action?: string 1529 since?: number 1530 limit?: number 1531 offset?: number 1532 }): unknown[] { 1533 const conditions: string[] = [] 1534 const params: unknown[] = [] 1535 1536 if (filters.entityType) { 1537 conditions.push(`json_extract(data, '$.entityType') = ?`) 1538 params.push(filters.entityType) 1539 } 1540 if (filters.entityId) { 1541 conditions.push(`json_extract(data, '$.entityId') = ?`) 1542 params.push(filters.entityId) 1543 } 1544 if (filters.actor) { 1545 conditions.push(`json_extract(data, '$.actor') = ?`) 1546 params.push(filters.actor) 1547 } 1548 if (filters.action) { 1549 conditions.push(`json_extract(data, '$.action') = ?`) 1550 params.push(filters.action) 1551 } 1552 if (typeof filters.since === 'number' && Number.isFinite(filters.since)) { 1553 conditions.push(`json_extract(data, '$.timestamp') >= ?`) 1554 params.push(filters.since) 1555 } 1556 1557 const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '' 1558 const limit = Math.min(200, Math.max(1, filters.limit ?? 50)) 1559 const offset = Math.max(0, filters.offset ?? 0) 1560 1561 const sql = `SELECT data FROM activity ${where} ORDER BY json_extract(data, '$.timestamp') DESC LIMIT ? OFFSET ?` 1562 params.push(limit, offset) 1563 1564 const rows = db.prepare(sql).all(...params) as Array<{ data: string }> 1565 return rows.map((r) => JSON.parse(r.data)) 1566 } 1567 1568 // --- Webhook Retry Queue --- 1569 const webhookRetryQueueStore = createCollectionStore('webhook_retry_queue') 1570 export const loadWebhookRetryQueue = webhookRetryQueueStore.load 1571 export const saveWebhookRetryQueue = webhookRetryQueueStore.save 1572 export const upsertWebhookRetry = webhookRetryQueueStore.upsert 1573 export const deleteWebhookRetry = webhookRetryQueueStore.deleteItem 1574 1575 // --- Notifications --- 1576 const notificationsStore = createCollectionStore('notifications') 1577 export const loadNotifications = notificationsStore.load 1578 export const saveNotification = notificationsStore.upsert 1579 export const deleteNotification = notificationsStore.deleteItem 1580 1581 export function findNotificationByDedupKey(dedupKey: string): AppNotification | null { 1582 const raw = getCollectionRawCache('notifications') 1583 for (const json of raw.values()) { 1584 try { 1585 const notification = JSON.parse(json) as AppNotification 1586 if (notification.dedupKey === dedupKey) return notification 1587 } catch { 1588 // ignore malformed 1589 } 1590 } 1591 return null 1592 } 1593 1594 export function hasUnreadNotificationWithKey(dedupKey: string): boolean { 1595 const raw = getCollectionRawCache('notifications') 1596 for (const json of raw.values()) { 1597 try { 1598 const n = JSON.parse(json) as Record<string, unknown> 1599 if (n.dedupKey === dedupKey && n.read !== true) return true 1600 } catch { /* skip malformed */ } 1601 } 1602 return false 1603 } 1604 1605 export function markNotificationRead(id: string) { 1606 const raw = getCollectionRawCache('notifications') 1607 const json = raw.get(id) 1608 if (!json) return 1609 try { 1610 const notification = JSON.parse(json) as Record<string, unknown> 1611 notification.read = true 1612 upsertCollectionItem('notifications', id, notification) 1613 } catch { 1614 // ignore malformed 1615 } 1616 } 1617 1618 // --- Moderation Logs --- 1619 const moderationLogsStore = createCollectionStore('moderation_logs') 1620 export const loadModerationLogs = moderationLogsStore.load 1621 export const appendModerationLog = moderationLogsStore.upsert 1622 1623 // --- Connector Health --- 1624 const connectorHealthStore = createCollectionStore('connector_health') 1625 export const loadConnectorHealth = connectorHealthStore.load 1626 export const upsertConnectorHealthEvent = connectorHealthStore.upsert 1627 1628 // --- Connector Outbox --- 1629 const connectorOutboxStore = createCollectionStore('connector_outbox') 1630 export const loadConnectorOutbox = connectorOutboxStore.load 1631 export const upsertConnectorOutboxItem = connectorOutboxStore.upsert 1632 export const deleteConnectorOutboxItem = connectorOutboxStore.deleteItem 1633 1634 // --- Approvals --- 1635 const approvalsStore = createCollectionStore('approvals') 1636 export const loadApprovals = approvalsStore.load 1637 export const upsertApproval = approvalsStore.upsert 1638 export const deleteApproval = approvalsStore.deleteItem 1639 1640 // --- Browser Sessions --- 1641 const browserSessionsStore = createCollectionStore('browser_sessions') 1642 export const loadBrowserSessions = browserSessionsStore.load 1643 export const upsertBrowserSession = browserSessionsStore.upsert 1644 export const deleteBrowserSession = browserSessionsStore.deleteItem 1645 1646 // --- Watch Jobs --- 1647 const watchJobsStore = createCollectionStore('watch_jobs') 1648 export const loadWatchJobs = watchJobsStore.load 1649 export const upsertWatchJob = watchJobsStore.upsert 1650 export const upsertWatchJobs = watchJobsStore.upsertMany 1651 export const deleteWatchJob = watchJobsStore.deleteItem 1652 1653 // --- Delegation Jobs --- 1654 const delegationJobsStore = createCollectionStore('delegation_jobs', { ttlMs: 5_000 }) 1655 export const loadDelegationJobs = delegationJobsStore.load 1656 export const loadDelegationJobItem = delegationJobsStore.loadItem 1657 export const upsertDelegationJob = delegationJobsStore.upsert 1658 export const { patch: patchDelegationJob } = delegationJobsStore 1659 export const deleteDelegationJob = delegationJobsStore.deleteItem 1660 1661 // --- Main Loop States --- 1662 const mainLoopStatesStore = createCollectionStore('main_loop_states') 1663 export const loadPersistedMainLoopState = mainLoopStatesStore.loadItem 1664 export const upsertPersistedMainLoopState = mainLoopStatesStore.upsert 1665 export const deletePersistedMainLoopState = mainLoopStatesStore.deleteItem 1666 1667 // --- Working States --- 1668 const workingStatesStore = createCollectionStore('working_states') 1669 export const loadPersistedWorkingState = workingStatesStore.loadItem 1670 export const upsertPersistedWorkingState = workingStatesStore.upsert 1671 export const deletePersistedWorkingState = workingStatesStore.deleteItem 1672 1673 // --- Wallets --- 1674 const walletsStore = createCollectionStore('wallets') 1675 export const loadWallets = walletsStore.load 1676 export const saveWallets = walletsStore.save 1677 export const loadWallet = walletsStore.loadItem 1678 export const upsertWallet = walletsStore.upsert 1679 export const deleteWalletItem = walletsStore.deleteItem 1680 1681 // --- Wallet Transactions --- 1682 const walletTransactionsStore = createCollectionStore('wallet_transactions') 1683 export const loadWalletTransactions = walletTransactionsStore.load 1684 export const saveWalletTransactions = walletTransactionsStore.save 1685 export const loadWalletTransaction = walletTransactionsStore.loadItem 1686 export const upsertWalletTransaction = walletTransactionsStore.upsert 1687 export const deleteWalletTransaction = walletTransactionsStore.deleteItem 1688 1689 // --- Goals --- 1690 const goalsStore = createCollectionStore('goals') 1691 export const loadGoals = goalsStore.load 1692 export const loadGoal = goalsStore.loadItem 1693 export const upsertGoal = goalsStore.upsert 1694 export const deleteGoalItem = goalsStore.deleteItem 1695 1696 // --- Agent Missions (autonomous goal-driven runs) --- 1697 const agentMissionsStore = createCollectionStore<Mission>('agent_missions', { ttlMs: 5_000 }) 1698 export const loadAgentMissions = agentMissionsStore.load 1699 export const saveAgentMissions = agentMissionsStore.save 1700 export const loadAgentMission = agentMissionsStore.loadItem 1701 export const upsertAgentMission = agentMissionsStore.upsert 1702 export const patchAgentMission = agentMissionsStore.patch 1703 export const deleteAgentMission = agentMissionsStore.deleteItem 1704 1705 const missionReportsStore = createCollectionStore<MissionReport>('mission_reports') 1706 export const loadMissionReports = missionReportsStore.load 1707 export const saveMissionReports = missionReportsStore.save 1708 export const loadMissionReport = missionReportsStore.loadItem 1709 export const upsertMissionReport = missionReportsStore.upsert 1710 export const deleteMissionReport = missionReportsStore.deleteItem 1711 1712 const agentMissionEventsStore = createCollectionStore<MissionEvent>('agent_mission_events') 1713 export const loadAgentMissionEvents = agentMissionEventsStore.load 1714 export const saveAgentMissionEvents = agentMissionEventsStore.save 1715 export const loadAgentMissionEvent = agentMissionEventsStore.loadItem 1716 export const upsertAgentMissionEvent = agentMissionEventsStore.upsert 1717 export const deleteAgentMissionEvent = agentMissionEventsStore.deleteItem 1718 1719 function legacyMissionStatusToWorkingStatus(value: unknown): 'idle' | 'progress' | 'blocked' | 'completed' { 1720 const normalized = typeof value === 'string' ? value.trim().toLowerCase() : '' 1721 if (normalized === 'achieved' || normalized === 'completed' || normalized === 'ok') return 'completed' 1722 if (normalized === 'blocked' || normalized === 'waiting' || normalized === 'paused') return 'blocked' 1723 if (normalized === 'active' || normalized === 'executing' || normalized === 'progress') return 'progress' 1724 return 'idle' 1725 } 1726 1727 function buildGoalFromLegacyMission(id: string, mission: StoredObject): StoredObject { 1728 const objective = typeof mission.objective === 'string' && mission.objective.trim() 1729 ? mission.objective.trim() 1730 : typeof mission.title === 'string' && mission.title.trim() 1731 ? mission.title.trim() 1732 : 'Legacy mission objective' 1733 const title = typeof mission.title === 'string' && mission.title.trim() 1734 ? mission.title.trim() 1735 : objective.slice(0, 120) 1736 const status = typeof mission.status === 'string' && mission.status.trim().toLowerCase() === 'achieved' 1737 ? 'achieved' 1738 : typeof mission.status === 'string' && mission.status.trim().toLowerCase() === 'abandoned' 1739 ? 'abandoned' 1740 : 'active' 1741 1742 return { 1743 id, 1744 title, 1745 description: typeof mission.plannerSummary === 'string' && mission.plannerSummary.trim() 1746 ? mission.plannerSummary.trim() 1747 : typeof mission.description === 'string' && mission.description.trim() 1748 ? mission.description.trim() 1749 : undefined, 1750 level: mission.taskId ? 'task' : mission.agentId ? 'agent' : mission.projectId ? 'project' : 'organization', 1751 parentGoalId: typeof mission.parentMissionId === 'string' && mission.parentMissionId.trim() ? mission.parentMissionId.trim() : null, 1752 projectId: typeof mission.projectId === 'string' && mission.projectId.trim() ? mission.projectId.trim() : null, 1753 agentId: typeof mission.agentId === 'string' && mission.agentId.trim() ? mission.agentId.trim() : null, 1754 taskId: typeof mission.taskId === 'string' && mission.taskId.trim() ? mission.taskId.trim() : null, 1755 objective, 1756 constraints: Array.isArray(mission.constraints) 1757 ? mission.constraints.filter((entry): entry is string => typeof entry === 'string' && entry.trim().length > 0) 1758 : [], 1759 successMetric: typeof mission.successMetric === 'string' && mission.successMetric.trim() ? mission.successMetric.trim() : null, 1760 budgetUsd: typeof mission.budgetUsd === 'number' && Number.isFinite(mission.budgetUsd) ? mission.budgetUsd : null, 1761 deadlineAt: typeof mission.deadlineAt === 'number' && Number.isFinite(mission.deadlineAt) ? mission.deadlineAt : null, 1762 status, 1763 createdAt: typeof mission.createdAt === 'number' && Number.isFinite(mission.createdAt) ? mission.createdAt : Date.now(), 1764 updatedAt: typeof mission.updatedAt === 'number' && Number.isFinite(mission.updatedAt) ? mission.updatedAt : Date.now(), 1765 } 1766 } 1767 1768 function buildWorkingStateFromLegacyMission(mission: StoredObject): StoredObject | null { 1769 const sessionId = typeof mission.sessionId === 'string' && mission.sessionId.trim() 1770 ? mission.sessionId.trim() 1771 : '' 1772 if (!sessionId) return null 1773 const currentStep = typeof mission.currentStep === 'string' && mission.currentStep.trim() 1774 ? mission.currentStep.trim() 1775 : '' 1776 const blockerSummary = typeof mission.blockerSummary === 'string' && mission.blockerSummary.trim() 1777 ? mission.blockerSummary.trim() 1778 : '' 1779 return { 1780 sessionId, 1781 objective: typeof mission.objective === 'string' && mission.objective.trim() ? mission.objective.trim() : null, 1782 summary: typeof mission.plannerSummary === 'string' && mission.plannerSummary.trim() ? mission.plannerSummary.trim() : null, 1783 status: legacyMissionStatusToWorkingStatus(mission.status ?? mission.phase), 1784 nextAction: currentStep || null, 1785 planSteps: currentStep 1786 ? [{ id: `legacy-mission-${sessionId}`, text: currentStep, status: 'active', createdAt: Date.now(), updatedAt: Date.now() }] 1787 : [], 1788 blockers: blockerSummary 1789 ? [{ id: `legacy-mission-blocker-${sessionId}`, summary: blockerSummary, status: 'active', createdAt: Date.now(), updatedAt: Date.now() }] 1790 : [], 1791 confirmedFacts: [], 1792 artifacts: [], 1793 decisions: [], 1794 openQuestions: [], 1795 hypotheses: [], 1796 evidenceRefs: [], 1797 constraints: Array.isArray(mission.constraints) 1798 ? mission.constraints.filter((entry): entry is string => typeof entry === 'string' && entry.trim().length > 0) 1799 : [], 1800 successCriteria: Array.isArray(mission.successCriteria) 1801 ? mission.successCriteria.filter((entry): entry is string => typeof entry === 'string' && entry.trim().length > 0) 1802 : [], 1803 updatedAt: typeof mission.updatedAt === 'number' && Number.isFinite(mission.updatedAt) ? mission.updatedAt : Date.now(), 1804 } 1805 } 1806 1807 export function saveMissions(missions: Record<string, StoredObject>): void { 1808 for (const [id, mission] of Object.entries(missions || {})) { 1809 upsertGoal(id, buildGoalFromLegacyMission(id, mission)) 1810 const workingState = buildWorkingStateFromLegacyMission(mission) 1811 if (workingState) upsertPersistedWorkingState(String(workingState.sessionId), workingState) 1812 } 1813 } 1814 1815 export function loadMissions(): Record<string, StoredObject> { 1816 return loadGoals() as Record<string, StoredObject> 1817 } 1818 1819 export function getSessionMessages(sessionId: string): Message[] { 1820 const session = loadSession(sessionId) 1821 return Array.isArray(session?.messages) ? session.messages : [] 1822 }