graph_feeder.py
1 #!/usr/bin/env python3 2 """ 3 Sovereign OS - Graph Feeder 4 ============================ 5 6 Feeds linear documents (session reports, FO findings, attention ledger) 7 into the living knowledge graph. 8 9 This closes the loop: every insight captured actually compounds in the 10 graph, not just sits in a linear file. 11 12 Sources: 13 - ~/.sovereign/session-report-state.json (insights, resonances, edges) 14 - sessions/FO-STATE.json (FO findings, axiom activity) 15 - ~/.sovereign/attention-ledger/*.yaml (work items with V-scores) 16 17 Outputs: 18 - sessions/GRAPH-STATE.md (full graph state) 19 - sessions/LIVE-GRAPH.md (mermaid visualization) 20 21 Usage: 22 python3 scripts/graph_feeder.py # Run once, update graph 23 python3 scripts/graph_feeder.py --watch # Watch mode (continuous) 24 python3 scripts/graph_feeder.py --status # Show graph status 25 python3 scripts/graph_feeder.py --backhaul # Extract from all historical sources 26 python3 scripts/graph_feeder.py --orphans # Find orphan nodes with connection suggestions 27 python3 scripts/graph_feeder.py --auto-connect # Create edges for orphans (threshold 0.5) 28 python3 scripts/graph_feeder.py --auto-connect --threshold 0.6 # Higher threshold 29 python3 scripts/graph_feeder.py --auto-connect --dry-run # Preview without creating 30 """ 31 32 import json 33 import re 34 import sys 35 import time 36 import hashlib 37 import urllib.request 38 import urllib.error 39 from pathlib import Path 40 from datetime import datetime 41 from typing import Dict, List, Any, Set, Optional, Tuple 42 from dataclasses import dataclass, field 43 44 # Mesh network integration - for N of X consciousness 45 MESH_HTTP_PORT = 7778 46 47 def publish_to_mesh(message_type: str, payload: Dict) -> bool: 48 """ 49 Publish an update to the sovereign mesh network. 50 This enables live state sync across all connected Claude instances. 51 """ 52 try: 53 msg = json.dumps({ 54 "type": message_type, 55 "payload": payload, 56 "timestamp": datetime.now().isoformat(), 57 "source": "graph_feeder" 58 }) 59 req = urllib.request.Request( 60 f"http://localhost:{MESH_HTTP_PORT}/publish", 61 data=msg.encode('utf-8'), 62 headers={"Content-Type": "application/json"}, 63 method="POST" 64 ) 65 with urllib.request.urlopen(req, timeout=2) as resp: 66 return resp.status == 200 67 except (urllib.error.URLError, Exception): 68 # Mesh not running - local-only mode 69 return False 70 71 # Paths 72 SOVEREIGN_OS = Path(__file__).parent.parent 73 SESSIONS_DIR = SOVEREIGN_OS / "sessions" 74 SESSION_REPORT_STATE = Path.home() / ".sovereign" / "session-report-state.json" 75 FO_STATE = SESSIONS_DIR / "FO-STATE.json" 76 GRAPH_STATE = SESSIONS_DIR / "GRAPH-STATE.md" 77 LIVE_GRAPH = SESSIONS_DIR / "LIVE-GRAPH.md" 78 GRAPH_DATA = Path.home() / ".sovereign" / "graph-data.json" 79 80 81 @dataclass 82 class GraphNode: 83 """ 84 A node in the knowledge graph. 85 86 The graph is 3D: crystallization level is the vertical axis. 87 - Top (1.0) = Torah - core axioms, most crystallized 88 - Middle (0.5-0.8) = Derived principles, patterns 89 - Bottom (0.0-0.3) = Talmud/leaves - raw instances, insights 90 91 Click down = more detail, more recent, less crystallized 92 Click up = more abstract, more validated, more Torah-like 93 """ 94 id: str 95 label: str 96 node_type: str # 'axiom', 'derived_principle', 'pattern', 'insight', 'instance' 97 content: str 98 axioms: List[str] = field(default_factory=list) 99 importance: float = 0.5 100 created_at: str = "" 101 source: str = "" # 'session_report', 'fo', 'manual' 102 103 # 3D Graph: Crystallization level (vertical axis) 104 # 1.0 = Torah (axiom level, highly crystallized) 105 # 0.7-0.9 = Derived principles (crystallized from many instances) 106 # 0.4-0.6 = Patterns/edges (emerging crystallization) 107 # 0.1-0.3 = Talmud/leaves (raw instances, recent insights) 108 crystallization: float = 0.2 # Default to leaf level 109 110 # Altitude (from CLAUDE.md protocol) 111 # philosophical > strategic > operational > tactical 112 altitude: str = "tactical" 113 114 # Validation count - how many instances support this? 115 validation_count: int = 1 116 117 # Parent nodes (what this derives from / supports) 118 derives_from: List[str] = field(default_factory=list) 119 120 def to_dict(self) -> Dict: 121 return { 122 'id': self.id, 123 'label': self.label, 124 'node_type': self.node_type, 125 'content': self.content, 126 'axioms': self.axioms, 127 'importance': self.importance, 128 'created_at': self.created_at, 129 'source': self.source 130 } 131 132 @classmethod 133 def from_dict(cls, d: Dict) -> 'GraphNode': 134 return cls(**d) 135 136 137 @dataclass 138 class GraphEdge: 139 """An edge between nodes.""" 140 source_id: str 141 target_id: str 142 edge_type: str # 'resonates_with', 'derived_from', 'contradicts', 'extends' 143 strength: float = 0.5 144 created_at: str = "" 145 146 def to_dict(self) -> Dict: 147 return { 148 'source_id': self.source_id, 149 'target_id': self.target_id, 150 'edge_type': self.edge_type, 151 'strength': self.strength, 152 'created_at': self.created_at 153 } 154 155 @classmethod 156 def from_dict(cls, d: Dict) -> 'GraphEdge': 157 return cls(**d) 158 159 160 class KnowledgeGraph: 161 """The living knowledge graph.""" 162 163 def __init__(self): 164 self.nodes: Dict[str, GraphNode] = {} 165 self.edges: List[GraphEdge] = [] 166 self.axiom_nodes = ['A0', 'A1', 'A2', 'A3', 'A4'] 167 self._load() 168 self._ensure_axiom_nodes() 169 170 def _ensure_axiom_nodes(self): 171 """Ensure axiom nodes exist.""" 172 axiom_info = { 173 'A0': ('Boundary Operation', 'Structure flows, content sovereign'), 174 'A1': ('Integration', 'Move toward connection, not isolation'), 175 'A2': ('Life', 'Primitive over calcified, motion is life'), 176 'A3': ('Navigation', 'Dynamic pole navigation, tension is the dyad'), 177 'A4': ('Ergodicity', 'Prevent ruin before optimizing gain') 178 } 179 for axiom, (label, content) in axiom_info.items(): 180 if axiom not in self.nodes: 181 self.nodes[axiom] = GraphNode( 182 id=axiom, 183 label=label, 184 node_type='axiom', 185 content=content, 186 axioms=[axiom], 187 importance=1.0, 188 created_at=datetime.now().isoformat(), 189 source='system' 190 ) 191 192 def _load(self): 193 """Load graph from disk.""" 194 if GRAPH_DATA.exists(): 195 try: 196 with open(GRAPH_DATA) as f: 197 data = json.load(f) 198 for n in data.get('nodes', []): 199 node = GraphNode.from_dict(n) 200 self.nodes[node.id] = node 201 for e in data.get('edges', []): 202 self.edges.append(GraphEdge.from_dict(e)) 203 except Exception as e: 204 print(f"Warning: Could not load graph data: {e}") 205 206 def save(self, publish_delta: bool = True): 207 """Save graph to disk and optionally publish delta to mesh.""" 208 GRAPH_DATA.parent.mkdir(parents=True, exist_ok=True) 209 data = { 210 'nodes': [n.to_dict() for n in self.nodes.values()], 211 'edges': [e.to_dict() for e in self.edges], 212 'updated': datetime.now().isoformat() 213 } 214 with open(GRAPH_DATA, 'w') as f: 215 json.dump(data, f, indent=2) 216 217 # N of X: Publish graph state to mesh for distributed consciousness 218 if publish_delta: 219 published = publish_to_mesh("graph_update", { 220 "node_count": len(self.nodes), 221 "edge_count": len(self.edges), 222 "recent_nodes": [n.to_dict() for n in list(self.nodes.values())[-5:]], 223 "checksum": hashlib.sha256(json.dumps(data, sort_keys=True).encode()).hexdigest()[:16] 224 }) 225 if published: 226 print("[mesh] Graph state published to network") 227 228 def _generate_id(self, content: str) -> str: 229 """Generate a stable ID from content.""" 230 return hashlib.md5(content.encode()).hexdigest()[:12] 231 232 def add_insight(self, description: str, principle: str = None, 233 importance: float = 0.7, source: str = 'session_report') -> str: 234 """Add an insight node.""" 235 node_id = f"INS_{self._generate_id(description)}" 236 237 if node_id not in self.nodes: 238 axioms = [principle] if principle else [] 239 self.nodes[node_id] = GraphNode( 240 id=node_id, 241 label=description[:30] + "..." if len(description) > 30 else description, 242 node_type='insight', 243 content=description, 244 axioms=axioms, 245 importance=importance, 246 created_at=datetime.now().isoformat(), 247 source=source 248 ) 249 250 # Create edge to axiom if specified 251 if principle and principle in self.axiom_nodes: 252 self.edges.append(GraphEdge( 253 source_id=node_id, 254 target_id=principle, 255 edge_type='derived_from', 256 strength=0.8, 257 created_at=datetime.now().isoformat() 258 )) 259 260 return node_id 261 return node_id 262 263 def add_resonance(self, source_desc: str, target_desc: str, 264 strength: float = 0.5) -> Optional[str]: 265 """Add a resonance edge between two concepts.""" 266 # Create or find source node 267 source_id = f"CON_{self._generate_id(source_desc)}" 268 if source_id not in self.nodes: 269 self.nodes[source_id] = GraphNode( 270 id=source_id, 271 label=source_desc[:30], 272 node_type='concept', 273 content=source_desc, 274 importance=0.6, 275 created_at=datetime.now().isoformat(), 276 source='resonance' 277 ) 278 279 # Create or find target node 280 target_id = f"CON_{self._generate_id(target_desc)}" 281 if target_id not in self.nodes: 282 self.nodes[target_id] = GraphNode( 283 id=target_id, 284 label=target_desc[:30], 285 node_type='concept', 286 content=target_desc, 287 importance=0.6, 288 created_at=datetime.now().isoformat(), 289 source='resonance' 290 ) 291 292 # Create edge 293 edge_id = f"{source_id}_{target_id}" 294 # Check if edge already exists 295 existing = [e for e in self.edges 296 if e.source_id == source_id and e.target_id == target_id] 297 if not existing: 298 self.edges.append(GraphEdge( 299 source_id=source_id, 300 target_id=target_id, 301 edge_type='resonates_with', 302 strength=strength, 303 created_at=datetime.now().isoformat() 304 )) 305 return edge_id 306 return None 307 308 def add_principle_edge(self, principle: str, edge_desc: str, 309 d_score: float = 0.35) -> Optional[str]: 310 """Add a principle edge discovery.""" 311 # Create node for the edge discovery 312 node_id = f"EDGE_{self._generate_id(edge_desc)}" 313 314 if node_id not in self.nodes: 315 self.nodes[node_id] = GraphNode( 316 id=node_id, 317 label=f"{principle} edge", 318 node_type='principle_edge', 319 content=edge_desc, 320 axioms=[principle] if principle in self.axiom_nodes else [], 321 importance=0.8, # Edge discoveries are high value 322 created_at=datetime.now().isoformat(), 323 source='principle_edge' 324 ) 325 326 # Connect to axiom 327 if principle in self.axiom_nodes: 328 self.edges.append(GraphEdge( 329 source_id=node_id, 330 target_id=principle, 331 edge_type='clarifies', 332 strength=1.0 - d_score, # Lower D = stronger connection 333 created_at=datetime.now().isoformat() 334 )) 335 return node_id 336 return None 337 338 def add_fo_finding(self, finding: Dict) -> Optional[str]: 339 """Add a First Officer finding.""" 340 content = finding.get('content', '') 341 if not content: 342 return None 343 344 node_id = f"FO_{self._generate_id(content)}" 345 346 if node_id not in self.nodes: 347 axioms = finding.get('axioms_involved', []) 348 self.nodes[node_id] = GraphNode( 349 id=node_id, 350 label=content[:30] + "..." if len(content) > 30 else content, 351 node_type='fo_finding', 352 content=finding.get('context', content)[:200], 353 axioms=axioms, 354 importance=finding.get('importance', 0.75), 355 created_at=finding.get('timestamp', datetime.now().isoformat()), 356 source='first_officer' 357 ) 358 359 # Connect to axioms 360 for axiom in axioms: 361 if axiom in self.axiom_nodes: 362 self.edges.append(GraphEdge( 363 source_id=node_id, 364 target_id=axiom, 365 edge_type='relates_to', 366 strength=finding.get('importance', 0.75), 367 created_at=datetime.now().isoformat() 368 )) 369 return node_id 370 return None 371 372 def get_stats(self) -> Dict: 373 """Get graph statistics.""" 374 node_types = {} 375 for node in self.nodes.values(): 376 node_types[node.node_type] = node_types.get(node.node_type, 0) + 1 377 378 edge_types = {} 379 for edge in self.edges: 380 edge_types[edge.edge_type] = edge_types.get(edge.edge_type, 0) + 1 381 382 return { 383 'total_nodes': len(self.nodes), 384 'total_edges': len(self.edges), 385 'node_types': node_types, 386 'edge_types': edge_types 387 } 388 389 def find_orphans(self) -> Dict[str, List['GraphNode']]: 390 """ 391 Find orphan nodes - nodes with no edges connecting them. 392 393 Returns dict grouped by node_type for easy analysis. 394 Orphans are high-value targets for graph enrichment. 395 """ 396 # Build set of all connected node IDs 397 connected: Set[str] = set() 398 for edge in self.edges: 399 connected.add(edge.source_id) 400 connected.add(edge.target_id) 401 402 # Find nodes not in connected set 403 orphans: Dict[str, List[GraphNode]] = {} 404 for node_id, node in self.nodes.items(): 405 if node_id not in connected: 406 if node.node_type not in orphans: 407 orphans[node.node_type] = [] 408 orphans[node.node_type].append(node) 409 410 return orphans 411 412 def suggest_connections(self, orphan: 'GraphNode', max_suggestions: int = 3) -> List[Tuple[str, str, float]]: 413 """ 414 Suggest potential connections for an orphan node. 415 416 Returns list of (target_node_id, edge_type, confidence) tuples. 417 Uses simple heuristics: axiom matching, keyword overlap, type affinity. 418 """ 419 suggestions = [] 420 orphan_words = set(orphan.content.lower().split()) 421 422 for node_id, node in self.nodes.items(): 423 if node_id == orphan.id: 424 continue 425 426 confidence = 0.0 427 edge_type = 'relates_to' 428 429 # Axiom match bonus 430 shared_axioms = set(orphan.axioms) & set(node.axioms) 431 if shared_axioms: 432 confidence += 0.3 * len(shared_axioms) 433 edge_type = 'derived_from' if node.node_type == 'axiom' else 'relates_to' 434 435 # Keyword overlap bonus 436 node_words = set(node.content.lower().split()) 437 overlap = orphan_words & node_words 438 # Filter out common words 439 stop_words = {'the', 'a', 'an', 'is', 'are', 'was', 'were', 'to', 'from', 'in', 'on', 'at', 'for', 'with', 'and', 'or', 'this', 'that'} 440 meaningful_overlap = overlap - stop_words 441 if meaningful_overlap: 442 confidence += 0.15 * min(len(meaningful_overlap), 5) # Cap at 5 words 443 444 # Type affinity bonus 445 type_affinity = { 446 ('insight', 'concept'): 0.2, 447 ('concept', 'insight'): 0.2, 448 ('insight', 'axiom'): 0.15, 449 ('concept', 'axiom'): 0.15, 450 ('fo_finding', 'axiom'): 0.2, 451 ('principle_edge', 'axiom'): 0.25, 452 } 453 affinity_key = (orphan.node_type, node.node_type) 454 if affinity_key in type_affinity: 455 confidence += type_affinity[affinity_key] 456 457 if confidence > 0.2: # Threshold 458 suggestions.append((node_id, edge_type, confidence)) 459 460 # Sort by confidence, return top N 461 suggestions.sort(key=lambda x: x[2], reverse=True) 462 return suggestions[:max_suggestions] 463 464 def generate_mermaid(self) -> str: 465 """Generate Mermaid diagram.""" 466 lines = ["```mermaid", "graph TD"] 467 468 # Group nodes by type 469 node_groups = {} 470 for node in self.nodes.values(): 471 if node.node_type not in node_groups: 472 node_groups[node.node_type] = [] 473 node_groups[node.node_type].append(node) 474 475 # Add subgraphs for each type 476 for ntype, nodes in node_groups.items(): 477 if ntype == 'axiom': 478 lines.append(" subgraph AXIOMS") 479 elif ntype == 'insight': 480 lines.append(" subgraph INSIGHTS") 481 elif ntype == 'concept': 482 lines.append(" subgraph CONCEPTS") 483 elif ntype == 'fo_finding': 484 lines.append(" subgraph FO_FINDINGS") 485 elif ntype == 'principle_edge': 486 lines.append(" subgraph PRINCIPLE_EDGES") 487 else: 488 lines.append(f" subgraph {ntype.upper()}") 489 490 for node in nodes[:20]: # Limit per group 491 # Escape special characters 492 label = node.label.replace('"', "'").replace('\n', ' ')[:25] 493 lines.append(f' {node.id}["{label}"]') 494 lines.append(" end") 495 496 # Add edges (limit to prevent huge diagrams) 497 lines.append("") 498 lines.append(" %% Edges") 499 for edge in self.edges[:50]: 500 style = "-->" if edge.edge_type == 'derived_from' else "-.->" 501 if edge.edge_type == 'resonates_with': 502 style = "<-->" 503 lines.append(f" {edge.source_id} {style} {edge.target_id}") 504 505 lines.append("```") 506 return '\n'.join(lines) 507 508 def generate_graph_state_md(self) -> str: 509 """Generate GRAPH-STATE.md content.""" 510 stats = self.get_stats() 511 now = datetime.now().isoformat() 512 513 lines = [ 514 "# Graph State - Living Knowledge Structure", 515 "", 516 f"*Updated: {now[:19]}*", 517 "", 518 "---", 519 "", 520 "## Statistics", 521 "", 522 f"- **Total Nodes:** {stats['total_nodes']}", 523 f"- **Total Edges:** {stats['total_edges']}", 524 "", 525 "### Node Types", 526 "" 527 ] 528 529 for ntype, count in sorted(stats['node_types'].items()): 530 lines.append(f"- {ntype}: {count}") 531 532 lines.extend([ 533 "", 534 "### Edge Types", 535 "" 536 ]) 537 538 for etype, count in sorted(stats['edge_types'].items()): 539 lines.append(f"- {etype}: {count}") 540 541 lines.extend([ 542 "", 543 "---", 544 "", 545 "## Recent Nodes", 546 "" 547 ]) 548 549 # Sort by created_at and show recent 550 recent_nodes = sorted( 551 [n for n in self.nodes.values() if n.node_type != 'axiom'], 552 key=lambda x: x.created_at, 553 reverse=True 554 )[:15] 555 556 for node in recent_nodes: 557 axiom_str = f" [{','.join(node.axioms)}]" if node.axioms else "" 558 lines.append(f"- **{node.node_type}**: {node.label}{axiom_str}") 559 560 lines.extend([ 561 "", 562 "---", 563 "", 564 "## Live Graph", 565 "", 566 self.generate_mermaid() 567 ]) 568 569 return '\n'.join(lines) 570 571 572 def feed_from_session_report(graph: KnowledgeGraph) -> int: 573 """Feed session report data into graph.""" 574 if not SESSION_REPORT_STATE.exists(): 575 return 0 576 577 added = 0 578 try: 579 with open(SESSION_REPORT_STATE) as f: 580 state = json.load(f) 581 582 # Process insights 583 for insight in state.get('insights', []): 584 if graph.add_insight( 585 insight.get('description', ''), 586 insight.get('principle'), 587 0.75, 588 'session_report' 589 ): 590 added += 1 591 592 # Process resonances 593 for res in state.get('resonances', []): 594 if graph.add_resonance( 595 res.get('source', ''), 596 res.get('target', ''), 597 res.get('strength', 0.5) 598 ): 599 added += 1 600 601 # Process principle edges 602 for edge in state.get('principle_edges', []): 603 if graph.add_principle_edge( 604 edge.get('principle', ''), 605 edge.get('edge', ''), 606 edge.get('d_score', 0.35) 607 ): 608 added += 1 609 610 except Exception as e: 611 print(f"Error reading session report: {e}") 612 613 return added 614 615 616 def feed_from_fo_state(graph: KnowledgeGraph) -> int: 617 """Feed First Officer findings into graph.""" 618 if not FO_STATE.exists(): 619 return 0 620 621 added = 0 622 try: 623 with open(FO_STATE) as f: 624 fo_state = json.load(f) 625 626 # Process insights (high importance only) 627 for finding in fo_state.get('insights', []): 628 if finding.get('importance', 0) >= 0.75: 629 if graph.add_fo_finding(finding): 630 added += 1 631 632 except Exception as e: 633 print(f"Error reading FO state: {e}") 634 635 return added 636 637 638 def feed_from_mesh(graph: KnowledgeGraph) -> int: 639 """ 640 Feed mesh network items into graph. 641 642 Queries the mesh daemon's /graph-items endpoint which returns: 643 - Aha moments → insight nodes 644 - Confirmed principles → derived_principle nodes 645 - Principle candidates in testing → principle_candidate nodes 646 - Convergence topics → concept nodes 647 """ 648 added = 0 649 try: 650 req = urllib.request.Request( 651 f"http://localhost:{MESH_HTTP_PORT}/graph-items", 652 headers={"Accept": "application/json"}, 653 method="GET" 654 ) 655 with urllib.request.urlopen(req, timeout=2) as resp: 656 data = json.loads(resp.read()) 657 658 # Process nodes from mesh 659 for node_data in data.get('nodes', []): 660 node_id = node_data.get('id') 661 if not node_id or node_id in graph.nodes: 662 continue # Skip duplicates 663 664 # Create GraphNode from mesh data 665 graph.nodes[node_id] = GraphNode( 666 id=node_id, 667 label=node_data.get('label', '')[:30], 668 node_type=node_data.get('node_type', 'insight'), 669 content=node_data.get('content', ''), 670 axioms=node_data.get('axioms', []), 671 importance=node_data.get('importance', 0.5), 672 created_at=node_data.get('created_at', datetime.now().isoformat()), 673 source=node_data.get('source', 'mesh') 674 ) 675 added += 1 676 677 # Log high-importance items 678 if node_data.get('importance', 0) >= 0.8: 679 print(f" 📡 Mesh: {node_data.get('node_type')} - {node_data.get('label', '')[:40]}") 680 681 # Process edges from mesh 682 for edge_data in data.get('edges', []): 683 source_id = edge_data.get('source_id') 684 target_id = edge_data.get('target_id') 685 686 # Only create edge if both nodes exist 687 if source_id in graph.nodes and target_id in graph.axiom_nodes: 688 # Check for duplicate edge 689 existing = [e for e in graph.edges 690 if e.source_id == source_id and e.target_id == target_id] 691 if not existing: 692 graph.edges.append(GraphEdge( 693 source_id=source_id, 694 target_id=target_id, 695 edge_type=edge_data.get('edge_type', 'relates_to'), 696 strength=edge_data.get('strength', 0.5), 697 created_at=datetime.now().isoformat() 698 )) 699 added += 1 700 701 # Publish graph update back to mesh 702 if added > 0: 703 publish_to_mesh('graph_updated', { 704 'nodes_added': added, 705 'total_nodes': len(graph.nodes), 706 'total_edges': len(graph.edges) 707 }) 708 709 except urllib.error.URLError: 710 # Mesh not running - silent skip 711 pass 712 except Exception as e: 713 print(f"Error feeding from mesh: {e}") 714 715 return added 716 717 718 def feed_from_daily_synthesis(graph: KnowledgeGraph) -> int: 719 """ 720 Feed Mission Control daily synthesis into graph. 721 722 Extracts from DAILY-SYNTHESIS*.md files: 723 - Resonant concepts (cross-thread) → concept nodes 724 - Concept clusters (thread pairs) → edges between concepts 725 - Flow state moments → insight nodes with high importance 726 - Related pages → edges with resonance strength 727 """ 728 import re 729 730 added = 0 731 732 # Find all synthesis files 733 synthesis_files = list(SESSIONS_DIR.glob("DAILY-SYNTHESIS*.md")) 734 if not synthesis_files: 735 return 0 736 737 for synth_file in synthesis_files: 738 try: 739 content = synth_file.read_text() 740 741 # Extract date from filename or content 742 date_match = re.search(r'(\d{4}-\d{2}-\d{2})', synth_file.name) 743 synth_date = date_match.group(1) if date_match else datetime.now().strftime('%Y-%m-%d') 744 745 # 1. Parse Resonant Concepts table 746 # | **sovereign** | 6 | e32fb0f8, b3c0e063, ... | 747 resonant_pattern = r'\|\s*\*\*([^*]+)\*\*\s*\|\s*(\d+)\s*\|\s*([^|]+)\s*\|' 748 for match in re.finditer(resonant_pattern, content): 749 concept = match.group(1).strip() 750 thread_count = int(match.group(2)) 751 threads = [t.strip() for t in match.group(3).split(',')] 752 753 node_id = f"SYNTH_CON_{graph._generate_id(concept)}" 754 if node_id not in graph.nodes: 755 # Higher thread count = higher importance 756 importance = min(0.9, 0.4 + (thread_count * 0.1)) 757 758 graph.nodes[node_id] = GraphNode( 759 id=node_id, 760 label=concept[:30], 761 node_type='concept', 762 content=f"Cross-thread concept: {concept} (appears in {thread_count} threads)", 763 axioms=[], 764 importance=importance, 765 created_at=f"{synth_date}T00:00:00", 766 source='daily_synthesis' 767 ) 768 added += 1 769 770 if thread_count >= 4: 771 print(f" 📊 Resonant: {concept} ({thread_count} threads)") 772 773 # 2. Parse Concept Clusters (thread pairs with shared concepts) 774 # - **e32fb0f8 ↔ b3c0e063**: architecture, attention, audio, ... 775 cluster_pattern = r'-\s*\*\*([a-f0-9]+)\s*↔\s*([a-f0-9]+)\*\*:\s*([^\n]+)' 776 for match in re.finditer(cluster_pattern, content): 777 thread1 = match.group(1).strip() 778 thread2 = match.group(2).strip() 779 shared_concepts = [c.strip() for c in match.group(3).split(',')] 780 781 # Create edges between all pairs of shared concepts 782 for i, concept1 in enumerate(shared_concepts[:10]): # Limit to avoid explosion 783 for concept2 in shared_concepts[i+1:10]: 784 c1_id = f"SYNTH_CON_{graph._generate_id(concept1)}" 785 c2_id = f"SYNTH_CON_{graph._generate_id(concept2)}" 786 787 # Only create edge if both nodes exist 788 if c1_id in graph.nodes and c2_id in graph.nodes: 789 # Check for duplicate 790 existing = [e for e in graph.edges 791 if (e.source_id == c1_id and e.target_id == c2_id) or 792 (e.source_id == c2_id and e.target_id == c1_id)] 793 if not existing: 794 graph.edges.append(GraphEdge( 795 source_id=c1_id, 796 target_id=c2_id, 797 edge_type='co_occurs', 798 strength=0.6, 799 created_at=f"{synth_date}T00:00:00" 800 )) 801 added += 1 802 803 # 3. Parse Flow State Moments (high-engagement insights) 804 # - **00:10** [principle, core, resonance, aha, insight, pattern] (weight: 2.50x) 805 # > I was mid-session with another Claude... 806 flow_pattern = r'-\s*\*\*(\d{2}:\d{2})\*\*\s*\[([^\]]+)\]\s*\(weight:\s*([\d.]+)x\)\s*\n\s*>\s*([^\n]+)' 807 for match in re.finditer(flow_pattern, content): 808 timestamp = match.group(1) 809 tags = [t.strip() for t in match.group(2).split(',')] 810 weight = float(match.group(3)) 811 snippet = match.group(4).strip()[:200] 812 813 # Only capture high-weight moments 814 if weight >= 2.0 and snippet and snippet != '-': 815 node_id = f"SYNTH_FLOW_{graph._generate_id(f'{synth_date}_{timestamp}_{snippet[:20]}')}" 816 if node_id not in graph.nodes: 817 # Map tags to axioms where possible 818 axioms = [] 819 tag_to_axiom = { 820 'principle': 'A0', 'core': 'A0', 'boundary': 'A0', 821 'integration': 'A1', 'connection': 'A1', 822 'insight': 'A2', 'aha': 'A2', 'resonance': 'A2', 823 'navigation': 'A3', 'design': 'A3', 824 'invariant': 'A4', 'architecture': 'A0' 825 } 826 for tag in tags: 827 if tag in tag_to_axiom and tag_to_axiom[tag] not in axioms: 828 axioms.append(tag_to_axiom[tag]) 829 830 graph.nodes[node_id] = GraphNode( 831 id=node_id, 832 label=f"Flow [{','.join(tags[:3])}]", 833 node_type='insight', 834 content=snippet, 835 axioms=axioms, 836 importance=min(1.0, weight / 2.5), # Normalize weight to 0-1 837 created_at=f"{synth_date}T{timestamp}:00", 838 source='flow_state' 839 ) 840 added += 1 841 842 # Create edges to axioms 843 for axiom in axioms: 844 if axiom in graph.axiom_nodes: 845 graph.edges.append(GraphEdge( 846 source_id=node_id, 847 target_id=axiom, 848 edge_type='relates_to', 849 strength=min(1.0, weight / 2.5), 850 created_at=f"{synth_date}T{timestamp}:00" 851 )) 852 added += 1 853 854 # 4. Parse Related pages with resonance scores 855 # - [[attention_system]] - resonance: 52% 856 related_pattern = r'-\s*\[\[([^\]]+)\]\]\s*-\s*resonance:\s*(\d+)%' 857 for match in re.finditer(related_pattern, content): 858 page_name = match.group(1).strip() 859 resonance = int(match.group(2)) / 100.0 860 861 if resonance >= 0.3: # Only meaningful resonances 862 node_id = f"SYNTH_REL_{graph._generate_id(page_name)}" 863 if node_id not in graph.nodes: 864 graph.nodes[node_id] = GraphNode( 865 id=node_id, 866 label=page_name[:30], 867 node_type='concept', 868 content=f"Related page: {page_name} (resonance: {resonance:.0%})", 869 axioms=[], 870 importance=resonance, 871 created_at=f"{synth_date}T00:00:00", 872 source='synthesis_related' 873 ) 874 added += 1 875 876 except Exception as e: 877 print(f"Error processing {synth_file.name}: {e}") 878 879 # Publish to mesh if meaningful additions 880 if added > 5: 881 publish_to_mesh('synthesis_ingested', { 882 'files_processed': len(synthesis_files), 883 'items_added': added 884 }) 885 886 return added 887 888 889 def feed_from_roam_connections(graph: KnowledgeGraph) -> int: 890 """ 891 Feed Roam hidden connections into graph. 892 893 Extracts from synthesis/roam-hidden-connections.md: 894 - Hidden connection pairs → edges between concepts 895 - Co-occurrence counts → edge strength 896 897 Also extracts from synthesis/roam-philosophy-extraction.md: 898 - Key philosophers → concept nodes 899 - Key terms (rhizome, deterritorialization, etc.) → concept nodes 900 """ 901 import re 902 903 added = 0 904 synthesis_dir = SESSIONS_DIR / "synthesis" 905 906 # 1. Process hidden connections file 907 hidden_file = synthesis_dir / "roam-hidden-connections.md" 908 if hidden_file.exists(): 909 try: 910 content = hidden_file.read_text() 911 912 # Parse the hidden connections table 913 # | 1 | Upwork | DoorDash | 5 | 914 connection_pattern = r'\|\s*\d+\s*\|\s*([^|]+)\s*\|\s*([^|]+)\s*\|\s*(\d+)\s*\|' 915 916 connections_found = 0 917 for match in re.finditer(connection_pattern, content): 918 concept1 = match.group(1).strip() 919 concept2 = match.group(2).strip() 920 co_occurrence = int(match.group(3)) 921 922 # Skip header row and low-value connections 923 if concept1 == 'Concept 1' or co_occurrence < 2: 924 continue 925 926 # Filter out date-like entries and job-specific noise 927 if re.match(r'^\d{4}|^(TODO|DONE)|^#', concept1) or re.match(r'^\d{4}|^(TODO|DONE)|^#', concept2): 928 continue 929 930 # Create concept nodes if they don't exist 931 for concept in [concept1, concept2]: 932 node_id = f"ROAM_{graph._generate_id(concept)}" 933 if node_id not in graph.nodes: 934 # Determine node type based on heuristics 935 node_type = 'concept' 936 importance = 0.4 + min(0.4, co_occurrence * 0.05) 937 938 # Boost importance for philosophers/thinkers 939 philosophers = ['Deleuze', 'Hayek', 'Foucault', 'Guattari', 'Tiago Forte', 'GTD'] 940 if any(p.lower() in concept.lower() for p in philosophers): 941 node_type = 'concept' 942 importance = 0.75 943 944 graph.nodes[node_id] = GraphNode( 945 id=node_id, 946 label=concept[:30], 947 node_type=node_type, 948 content=f"Roam concept: {concept}", 949 axioms=[], 950 importance=importance, 951 created_at=datetime.now().isoformat(), 952 source='roam_research' 953 ) 954 added += 1 955 956 # Create hidden connection edge 957 c1_id = f"ROAM_{graph._generate_id(concept1)}" 958 c2_id = f"ROAM_{graph._generate_id(concept2)}" 959 960 # Check for duplicate edge 961 existing = [e for e in graph.edges 962 if (e.source_id == c1_id and e.target_id == c2_id) or 963 (e.source_id == c2_id and e.target_id == c1_id)] 964 if not existing: 965 # Strength based on co-occurrence (normalize to 0.3-0.9) 966 strength = min(0.9, 0.3 + (co_occurrence * 0.1)) 967 968 graph.edges.append(GraphEdge( 969 source_id=c1_id, 970 target_id=c2_id, 971 edge_type='hidden_connection', 972 strength=strength, 973 created_at=datetime.now().isoformat() 974 )) 975 added += 1 976 connections_found += 1 977 978 if connections_found > 0: 979 print(f" 🔗 Hidden connections: {connections_found} pairs") 980 981 except Exception as e: 982 print(f"Error processing roam-hidden-connections.md: {e}") 983 984 # 2. Process philosophy extraction for high-value concepts 985 phil_file = synthesis_dir / "roam-philosophy-extraction.md" 986 if phil_file.exists(): 987 try: 988 content = phil_file.read_text() 989 990 # Extract key philosophical concepts mentioned prominently 991 key_concepts = [ 992 ('Gilles Deleuze', ['A0', 'A3'], 'philosopher'), 993 ('Friedrich Hayek', ['A0', 'A3'], 'philosopher'), 994 ('Michel Foucault', ['A0'], 'philosopher'), 995 ('rhizome', ['A0', 'A1', 'A3'], 'concept'), 996 ('lines of flight', ['A3'], 'concept'), 997 ('deterritorialization', ['A0', 'A3'], 'concept'), 998 ('spontaneous order', ['A0', 'A1'], 'concept'), 999 ('the virtual', ['A2', 'A3'], 'concept'), 1000 ('assemblage', ['A0', 'A1'], 'concept'), 1001 ('biopolitics', ['A0'], 'concept'), 1002 ('decentralized knowledge', ['A0', 'A1'], 'concept'), 1003 ] 1004 1005 for concept, axioms, node_type in key_concepts: 1006 # Check if concept is actually mentioned in the file 1007 if concept.lower() not in content.lower(): 1008 continue 1009 1010 node_id = f"ROAM_PHIL_{graph._generate_id(concept)}" 1011 if node_id not in graph.nodes: 1012 graph.nodes[node_id] = GraphNode( 1013 id=node_id, 1014 label=concept[:30], 1015 node_type=node_type, 1016 content=f"Philosophical concept from Roam: {concept}", 1017 axioms=axioms, 1018 importance=0.8, # High importance for curated philosophical concepts 1019 created_at=datetime.now().isoformat(), 1020 source='roam_philosophy' 1021 ) 1022 added += 1 1023 1024 # Create edges to axioms 1025 for axiom in axioms: 1026 if axiom in graph.axiom_nodes: 1027 graph.edges.append(GraphEdge( 1028 source_id=node_id, 1029 target_id=axiom, 1030 edge_type='relates_to', 1031 strength=0.75, 1032 created_at=datetime.now().isoformat() 1033 )) 1034 added += 1 1035 1036 print(f" 📚 Philosophy: {concept} → [{', '.join(axioms)}]") 1037 1038 # Create edges between related philosophical concepts 1039 phil_connections = [ 1040 ('Gilles Deleuze', 'rhizome', 0.95), 1041 ('Gilles Deleuze', 'lines of flight', 0.9), 1042 ('Gilles Deleuze', 'deterritorialization', 0.9), 1043 ('Gilles Deleuze', 'the virtual', 0.85), 1044 ('Gilles Deleuze', 'assemblage', 0.85), 1045 ('Friedrich Hayek', 'spontaneous order', 0.95), 1046 ('Friedrich Hayek', 'decentralized knowledge', 0.9), 1047 ('Michel Foucault', 'biopolitics', 0.95), 1048 ('rhizome', 'lines of flight', 0.8), 1049 ('rhizome', 'deterritorialization', 0.8), 1050 ('spontaneous order', 'decentralized knowledge', 0.85), 1051 ] 1052 1053 for concept1, concept2, strength in phil_connections: 1054 c1_id = f"ROAM_PHIL_{graph._generate_id(concept1)}" 1055 c2_id = f"ROAM_PHIL_{graph._generate_id(concept2)}" 1056 1057 if c1_id in graph.nodes and c2_id in graph.nodes: 1058 existing = [e for e in graph.edges 1059 if (e.source_id == c1_id and e.target_id == c2_id) or 1060 (e.source_id == c2_id and e.target_id == c1_id)] 1061 if not existing: 1062 graph.edges.append(GraphEdge( 1063 source_id=c1_id, 1064 target_id=c2_id, 1065 edge_type='relates_to', 1066 strength=strength, 1067 created_at=datetime.now().isoformat() 1068 )) 1069 added += 1 1070 1071 except Exception as e: 1072 print(f"Error processing roam-philosophy-extraction.md: {e}") 1073 1074 # Publish to mesh if meaningful additions 1075 if added > 10: 1076 publish_to_mesh('roam_ingested', { 1077 'items_added': added 1078 }) 1079 1080 return added 1081 1082 1083 def feed_from_transcripts(graph: KnowledgeGraph, max_files: int = 20) -> int: 1084 """ 1085 Feed insights from Claude conversation transcripts into graph. 1086 1087 Extracts high-value content from .jsonl transcript files: 1088 - Explicit insights (insight:, key insight, discovered, etc.) 1089 - Axiom references (A0-A4 discussions) 1090 - Decisions and conclusions 1091 1092 Args: 1093 graph: Knowledge graph to feed into 1094 max_files: Maximum number of recent files to process (default 20) 1095 1096 Returns: 1097 Number of items added 1098 """ 1099 import hashlib 1100 from pathlib import Path 1101 1102 added = 0 1103 claude_projects = Path.home() / ".claude" / "projects" 1104 1105 if not claude_projects.exists(): 1106 return 0 1107 1108 # Find recent transcript files (sorted by modification time) 1109 transcripts = [] 1110 for project_dir in claude_projects.iterdir(): 1111 if project_dir.is_dir(): 1112 for jsonl in project_dir.glob("*.jsonl"): 1113 transcripts.append((jsonl, jsonl.stat().st_mtime)) 1114 1115 # Sort by modification time (newest first) and take max_files 1116 transcripts.sort(key=lambda x: x[1], reverse=True) 1117 transcripts = transcripts[:max_files] 1118 1119 # Patterns to extract 1120 insight_patterns = [ 1121 (r'(?:key )?insight[:\s]+([^.!?\n]{20,200})', 'insight'), 1122 (r'discovered[:\s]+([^.!?\n]{20,200})', 'discovery'), 1123 (r'the pattern (?:is|here)[:\s]+([^.!?\n]{20,200})', 'pattern'), 1124 (r'principle[:\s]+([^.!?\n]{20,200})', 'principle'), 1125 (r'important[:\s]+([^.!?\n]{20,200})', 'important'), 1126 ] 1127 1128 axiom_keywords = { 1129 'A0': ['boundary', 'markov blanket', 'sovereign', 'membrane', 'inside outside'], 1130 'A1': ['integration', 'connection', 'binding', 'isolation', 'telos'], 1131 'A2': ['life', 'death', 'motion', 'static', 'primitive', 'calcified', 'ornament'], 1132 'A3': ['navigation', 'pole', 'dynamic', 'tension', 'dyad', 'shadow'], 1133 'A4': ['ergodic', 'ruin', 'survival', 'asymmetry', 'catastrophic', 'terminal'], 1134 } 1135 1136 seen_hashes = set() 1137 1138 for transcript_path, mtime in transcripts: 1139 try: 1140 with open(transcript_path) as f: 1141 for line in f: 1142 if not line.strip(): 1143 continue 1144 try: 1145 entry = json.loads(line) 1146 except json.JSONDecodeError: 1147 continue 1148 1149 # Only process assistant text messages 1150 if entry.get('type') != 'assistant': 1151 continue 1152 1153 message = entry.get('message', {}) 1154 content = message.get('content', []) 1155 1156 if not isinstance(content, list): 1157 continue 1158 1159 for block in content: 1160 if not isinstance(block, dict): 1161 continue 1162 if block.get('type') != 'text': 1163 continue 1164 1165 text = block.get('text', '') 1166 if len(text) < 50: 1167 continue 1168 1169 # Extract insights 1170 for pattern, insight_type in insight_patterns: 1171 for match in re.finditer(pattern, text, re.IGNORECASE): 1172 content_text = match.group(1).strip() 1173 1174 # Skip if too short or looks like code 1175 if len(content_text) < 30: 1176 continue 1177 if content_text.startswith('{') or content_text.startswith('['): 1178 continue 1179 if '```' in content_text: 1180 continue 1181 1182 # Deduplicate 1183 content_hash = hashlib.md5(content_text.encode()).hexdigest()[:12] 1184 if content_hash in seen_hashes: 1185 continue 1186 seen_hashes.add(content_hash) 1187 1188 # Detect axioms 1189 axioms = [] 1190 content_lower = content_text.lower() 1191 for axiom, keywords in axiom_keywords.items(): 1192 if any(kw in content_lower for kw in keywords): 1193 axioms.append(axiom) 1194 1195 # Also check for explicit A0-A4 mentions 1196 for ax_match in re.finditer(r'\bA([0-4])\b', content_text): 1197 axiom = f"A{ax_match.group(1)}" 1198 if axiom not in axioms: 1199 axioms.append(axiom) 1200 1201 # Create node 1202 node_id = f"TRANSCRIPT_{content_hash}" 1203 1204 if node_id not in graph.nodes: 1205 graph.nodes[node_id] = GraphNode( 1206 id=node_id, 1207 node_type='transcript_insight', 1208 label=content_text[:40] + '...', 1209 content=content_text, 1210 axioms=sorted(axioms), 1211 importance=0.5 + len(axioms) * 0.1, 1212 source=f'transcript:{transcript_path.stem[:20]}' 1213 ) 1214 added += 1 1215 1216 # Create edges to axioms 1217 for axiom in axioms: 1218 edge_id = f"{node_id}_to_{axiom}" 1219 if edge_id not in graph.edges: 1220 graph.edges[edge_id] = GraphEdge( 1221 source=node_id, 1222 target=axiom, 1223 edge_type='references_axiom', 1224 weight=0.6 1225 ) 1226 added += 1 1227 1228 except Exception as e: 1229 # Skip problematic files 1230 continue 1231 1232 return added 1233 1234 1235 def feed_from_resonance_alerts(graph: KnowledgeGraph, max_files: int = 100) -> int: 1236 """ 1237 Feed resonance alerts into graph. 1238 1239 Extracts from resonance alert files: 1240 - Resonant concepts with connection strengths 1241 - High-resonance insights 1242 """ 1243 import hashlib 1244 1245 added = 0 1246 alerts_dir = SESSIONS_DIR / "RESONANCE-ALERTS" 1247 1248 if not alerts_dir.exists(): 1249 return 0 1250 1251 # Get recent alert files 1252 alerts = [(f, f.stat().st_mtime) for f in alerts_dir.glob("*.md")] 1253 alerts.sort(key=lambda x: x[1], reverse=True) 1254 alerts = alerts[:max_files] 1255 1256 for alert_file, _ in alerts: 1257 try: 1258 content = alert_file.read_text() 1259 1260 # Extract type 1261 alert_type = "" 1262 type_match = re.search(r'\*\*Type:\*\*\s*(\w+)', content) 1263 if type_match: 1264 alert_type = type_match.group(1) 1265 1266 # Extract resonant concepts (two formats) 1267 # Format 1: - [[concept]] - N% 1268 concept_pattern = r'- \[\[([^\]]+)\]\] - (\d+)%' 1269 for match in re.finditer(concept_pattern, content): 1270 concept = match.group(1) 1271 strength = int(match.group(2)) / 100.0 1272 1273 # Create concept node 1274 c_hash = hashlib.md5(concept.encode()).hexdigest()[:12] 1275 node_id = f"RESONANCE_{c_hash}" 1276 1277 if node_id not in graph.nodes: 1278 graph.nodes[node_id] = GraphNode( 1279 id=node_id, 1280 node_type='concept', 1281 label=concept[:30], 1282 content=f"Resonant concept from alert: {concept}", 1283 axioms=[], 1284 importance=0.4 + strength * 0.4, 1285 source=f'resonance_alert:{alert_file.stem[:20]}' 1286 ) 1287 added += 1 1288 1289 # Format 2: **Pattern:** `concept-name` (from SHARED_CONCEPT) 1290 pattern_match = re.search(r'\*\*Pattern:\*\*\s*`([^`]+)`', content) 1291 if pattern_match: 1292 concept = pattern_match.group(1) 1293 strength_match = re.search(r'\*\*Strength:\*\*\s*([\d.]+)', content) 1294 strength = float(strength_match.group(1)) if strength_match else 0.3 1295 1296 # Create concept node 1297 c_hash = hashlib.md5(concept.encode()).hexdigest()[:12] 1298 node_id = f"RESONANCE_{c_hash}" 1299 1300 if node_id not in graph.nodes: 1301 graph.nodes[node_id] = GraphNode( 1302 id=node_id, 1303 node_type='concept', 1304 label=concept[:30], 1305 content=f"Resonant concept from alert: {concept}", 1306 axioms=[], 1307 importance=0.4 + strength * 0.4, 1308 source=f'resonance_alert:{alert_file.stem[:20]}' 1309 ) 1310 added += 1 1311 1312 # Extract input statement for HIGH_RESONANCE_INSIGHT 1313 if 'HIGH_RESONANCE' in alert_type: 1314 stmt_match = re.search(r'## Input Statement\n\n> (.+)', content) 1315 if stmt_match: 1316 statement = stmt_match.group(1).strip() 1317 if len(statement) > 30: 1318 s_hash = hashlib.md5(statement.encode()).hexdigest()[:12] 1319 node_id = f"RESONANCE_INSIGHT_{s_hash}" 1320 1321 if node_id not in graph.nodes: 1322 graph.nodes[node_id] = GraphNode( 1323 id=node_id, 1324 node_type='insight', 1325 label=statement[:40] + '...', 1326 content=statement, 1327 axioms=[], 1328 importance=0.7, 1329 source=f'resonance_alert:{alert_file.stem[:20]}' 1330 ) 1331 added += 1 1332 1333 except Exception: 1334 continue 1335 1336 return added 1337 1338 1339 def feed_from_debriefs(graph: KnowledgeGraph) -> int: 1340 """ 1341 Feed session debriefs into graph. 1342 1343 Extracts from debrief markdown files: 1344 - Insights (with axiom connections) 1345 - Resonances (concept pairs) 1346 - Connections (completed work items) 1347 """ 1348 import hashlib 1349 1350 added = 0 1351 debriefs_dir = SESSIONS_DIR / "debriefs" 1352 1353 if not debriefs_dir.exists(): 1354 return 0 1355 1356 # Process all debrief files 1357 for debrief_file in debriefs_dir.glob("*.md"): 1358 try: 1359 content = debrief_file.read_text() 1360 1361 # Extract insights section 1362 insights_match = re.search(r'## Insights\n\n(.*?)(?=\n## |\Z)', content, re.DOTALL) 1363 if insights_match: 1364 insights_text = insights_match.group(1) 1365 for line in insights_text.strip().split('\n'): 1366 if line.startswith('- '): 1367 insight = line[2:].strip() 1368 if len(insight) < 20: 1369 continue 1370 1371 # Extract axiom references 1372 axioms = re.findall(r'\[\[A(\d)\]\]', insight) 1373 axioms = [f'A{a}' for a in axioms] 1374 1375 # Clean insight text 1376 insight_clean = re.sub(r'\s*→\s*\[\[.*?\]\]', '', insight) 1377 1378 # Create node 1379 content_hash = hashlib.md5(insight_clean.encode()).hexdigest()[:12] 1380 node_id = f"DEBRIEF_INSIGHT_{content_hash}" 1381 1382 if node_id not in graph.nodes: 1383 graph.nodes[node_id] = GraphNode( 1384 id=node_id, 1385 node_type='insight', 1386 label=insight_clean[:40] + '...', 1387 content=insight_clean, 1388 axioms=axioms, 1389 importance=0.7, 1390 source=f'debrief:{debrief_file.stem}' 1391 ) 1392 added += 1 1393 1394 # Create edges to axioms 1395 for axiom in axioms: 1396 edge_id = f"{node_id}_to_{axiom}" 1397 if edge_id not in graph.edges: 1398 graph.edges[edge_id] = GraphEdge( 1399 source=node_id, 1400 target=axiom, 1401 edge_type='derived_from', 1402 weight=0.7 1403 ) 1404 added += 1 1405 1406 # Extract resonances section 1407 resonances_match = re.search(r'## Resonances\n\n(.*?)(?=\n## |\Z)', content, re.DOTALL) 1408 if resonances_match: 1409 resonances_text = resonances_match.group(1) 1410 for line in resonances_text.strip().split('\n'): 1411 if '↔' in line: 1412 # Parse: [[Concept A]] ↔ [[Concept B]] (strength: 0.8) 1413 concepts = re.findall(r'\[\[([^\]]+)\]\]', line) 1414 strength_match = re.search(r'strength:\s*([\d.]+)', line) 1415 strength = float(strength_match.group(1)) if strength_match else 0.5 1416 1417 if len(concepts) >= 2: 1418 concept1, concept2 = concepts[0], concepts[1] 1419 1420 # Create concept nodes if they don't exist 1421 for concept in [concept1, concept2]: 1422 c_hash = hashlib.md5(concept.encode()).hexdigest()[:12] 1423 c_id = f"DEBRIEF_CONCEPT_{c_hash}" 1424 if c_id not in graph.nodes: 1425 graph.nodes[c_id] = GraphNode( 1426 id=c_id, 1427 node_type='concept', 1428 label=concept[:30], 1429 content=f"Resonant concept: {concept}", 1430 axioms=[], 1431 importance=0.6, 1432 source=f'debrief:{debrief_file.stem}' 1433 ) 1434 added += 1 1435 1436 # Create resonance edge 1437 c1_hash = hashlib.md5(concept1.encode()).hexdigest()[:12] 1438 c2_hash = hashlib.md5(concept2.encode()).hexdigest()[:12] 1439 edge_id = f"RESONANCE_{c1_hash}_{c2_hash}" 1440 if edge_id not in graph.edges: 1441 graph.edges[edge_id] = GraphEdge( 1442 source=f"DEBRIEF_CONCEPT_{c1_hash}", 1443 target=f"DEBRIEF_CONCEPT_{c2_hash}", 1444 edge_type='resonates_with', 1445 weight=strength 1446 ) 1447 added += 1 1448 1449 except Exception as e: 1450 continue 1451 1452 return added 1453 1454 1455 def update_graph_files(graph: KnowledgeGraph): 1456 """Update the markdown graph files.""" 1457 # Update GRAPH-STATE.md 1458 with open(GRAPH_STATE, 'w') as f: 1459 f.write(graph.generate_graph_state_md()) 1460 1461 # Update LIVE-GRAPH.md 1462 with open(LIVE_GRAPH, 'w') as f: 1463 f.write(f"# Live Graph - Real-Time Node Creation\n\n") 1464 f.write(f"*Updated: {datetime.now().isoformat()[:19]}*\n\n") 1465 f.write(f"**Nodes:** {len(graph.nodes)} | **Edges:** {len(graph.edges)}\n\n") 1466 f.write("---\n\n") 1467 f.write(graph.generate_mermaid()) 1468 1469 1470 def run_once(): 1471 """Run the feeder once.""" 1472 print("Loading graph...") 1473 graph = KnowledgeGraph() 1474 1475 print(f"Current: {len(graph.nodes)} nodes, {len(graph.edges)} edges") 1476 1477 # Feed from sources 1478 session_added = feed_from_session_report(graph) 1479 print(f"Session report: +{session_added} items") 1480 1481 fo_added = feed_from_fo_state(graph) 1482 print(f"FO findings: +{fo_added} items") 1483 1484 mesh_added = feed_from_mesh(graph) 1485 print(f"Mesh network: +{mesh_added} items") 1486 1487 synth_added = feed_from_daily_synthesis(graph) 1488 print(f"Daily synthesis: +{synth_added} items") 1489 1490 roam_added = feed_from_roam_connections(graph) 1491 print(f"Roam connections: +{roam_added} items") 1492 1493 transcript_added = feed_from_transcripts(graph) 1494 print(f"Transcripts: +{transcript_added} items") 1495 1496 debrief_added = feed_from_debriefs(graph) 1497 print(f"Debriefs: +{debrief_added} items") 1498 1499 resonance_added = feed_from_resonance_alerts(graph) 1500 print(f"Resonance alerts: +{resonance_added} items") 1501 1502 # Save and update files 1503 graph.save() 1504 update_graph_files(graph) 1505 1506 print(f"Final: {len(graph.nodes)} nodes, {len(graph.edges)} edges") 1507 print(f"Updated: GRAPH-STATE.md, LIVE-GRAPH.md") 1508 1509 1510 def run_watch(): 1511 """Run in watch mode.""" 1512 print("Starting graph feeder in watch mode...") 1513 print("Watching for changes... (Ctrl+C to stop)") 1514 1515 last_session_mtime = 0 1516 last_fo_mtime = 0 1517 last_mesh_check = 0 1518 MESH_CHECK_INTERVAL = 30 # Check mesh every 30 seconds 1519 1520 while True: 1521 try: 1522 now = time.time() 1523 1524 # Check for file changes 1525 session_mtime = SESSION_REPORT_STATE.stat().st_mtime if SESSION_REPORT_STATE.exists() else 0 1526 fo_mtime = FO_STATE.stat().st_mtime if FO_STATE.exists() else 0 1527 1528 file_changed = session_mtime > last_session_mtime or fo_mtime > last_fo_mtime 1529 mesh_due = (now - last_mesh_check) >= MESH_CHECK_INTERVAL 1530 1531 if file_changed or mesh_due: 1532 if file_changed: 1533 print(f"\n[{datetime.now().strftime('%H:%M:%S')}] File change detected, updating graph...") 1534 elif mesh_due: 1535 print(f"\n[{datetime.now().strftime('%H:%M:%S')}] Periodic mesh sync...") 1536 run_once() 1537 last_session_mtime = session_mtime 1538 last_fo_mtime = fo_mtime 1539 last_mesh_check = now 1540 1541 time.sleep(5) # Check every 5 seconds 1542 1543 except KeyboardInterrupt: 1544 print("\nStopped.") 1545 break 1546 except Exception as e: 1547 print(f"Error: {e}") 1548 time.sleep(10) 1549 1550 1551 def show_status(): 1552 """Show graph status.""" 1553 graph = KnowledgeGraph() 1554 stats = graph.get_stats() 1555 1556 print() 1557 print("╔══════════════════════════════════════════════════════════════════╗") 1558 print("║ KNOWLEDGE GRAPH STATUS ║") 1559 print("╚══════════════════════════════════════════════════════════════════╝") 1560 print() 1561 print(f" Total Nodes: {stats['total_nodes']}") 1562 print(f" Total Edges: {stats['total_edges']}") 1563 print() 1564 print(" Node Types:") 1565 for ntype, count in sorted(stats['node_types'].items()): 1566 print(f" {ntype:20} {count}") 1567 print() 1568 print(" Edge Types:") 1569 for etype, count in sorted(stats['edge_types'].items()): 1570 print(f" {etype:20} {count}") 1571 print() 1572 1573 1574 def backhaul_historical(): 1575 """ 1576 Extract value from historical sources and backhaul into graph. 1577 1578 Sources: 1579 1. Historical FO findings in FO-STATE.json (all insights) 1580 2. Historical attention ledgers (~/.sovereign/attention-ledger/*.yaml) 1581 3. LIVE-COMPRESSION.md history 1582 4. Transcript files (future: need NLP for handshake detection) 1583 """ 1584 print() 1585 print("╔══════════════════════════════════════════════════════════════════╗") 1586 print("║ BACKHAUL: EXTRACTING HISTORICAL VALUE ║") 1587 print("╚══════════════════════════════════════════════════════════════════╝") 1588 print() 1589 1590 graph = KnowledgeGraph() 1591 initial_nodes = len(graph.nodes) 1592 initial_edges = len(graph.edges) 1593 1594 # 1. FO-STATE.json - ALL insights (not just recent) 1595 print("Processing FO-STATE.json...") 1596 fo_added = 0 1597 if FO_STATE.exists(): 1598 try: 1599 with open(FO_STATE) as f: 1600 fo_state = json.load(f) 1601 for finding in fo_state.get('insights', []): 1602 # Take all findings above 0.5 importance for backhaul 1603 if finding.get('importance', 0) >= 0.5: 1604 if graph.add_fo_finding(finding): 1605 fo_added += 1 1606 print(f" Added {fo_added} FO findings") 1607 except Exception as e: 1608 print(f" Error: {e}") 1609 1610 # 2. Attention ledgers - work items with high V-scores 1611 print("Processing attention ledgers...") 1612 ledger_dir = Path.home() / ".sovereign" / "attention-ledger" 1613 ledger_added = 0 1614 if ledger_dir.exists(): 1615 for ledger_file in ledger_dir.glob("*.yaml"): 1616 try: 1617 with open(ledger_file) as f: 1618 content = f.read() 1619 # Parse JSON (our ledgers are actually JSON) 1620 try: 1621 data = json.loads(content) 1622 except: 1623 continue 1624 1625 for session_id, session in data.get('sessions', {}).items(): 1626 for item in session.get('work_items', []): 1627 v_score = item.get('v_score', 0) 1628 if v_score >= 0.7: # High-value work 1629 desc = item.get('description', '') 1630 node_id = f"WORK_{graph._generate_id(desc)}" 1631 if node_id not in graph.nodes: 1632 graph.nodes[node_id] = GraphNode( 1633 id=node_id, 1634 label=desc[:30], 1635 node_type='work_item', 1636 content=f"V={v_score:.2f} | {desc}", 1637 importance=v_score, 1638 created_at=item.get('timestamp', ''), 1639 source='attention_ledger' 1640 ) 1641 ledger_added += 1 1642 except Exception as e: 1643 pass 1644 print(f" Added {ledger_added} high-value work items") 1645 1646 # 3. LIVE-COMPRESSION history files 1647 print("Processing LIVE-COMPRESSION history...") 1648 compression_added = 0 1649 compression_dir = SESSIONS_DIR 1650 for comp_file in compression_dir.glob("LIVE-COMPRESSION-replay-*.md"): 1651 try: 1652 content = comp_file.read_text() 1653 # Extract key insights from compression files 1654 # Look for patterns like "Key Insight:" or "## Insight" 1655 lines = content.split('\n') 1656 for i, line in enumerate(lines): 1657 if 'insight' in line.lower() and ':' in line: 1658 insight_text = line.split(':', 1)[-1].strip() 1659 if len(insight_text) > 10: 1660 if graph.add_insight(insight_text, None, 0.6, 'compression_history'): 1661 compression_added += 1 1662 except: 1663 pass 1664 print(f" Added {compression_added} insights from compression history") 1665 1666 # 4. Session report state (current session insights) 1667 print("Processing session report state...") 1668 session_added = feed_from_session_report(graph) 1669 print(f" Added {session_added} items from session report") 1670 1671 # Save and update 1672 graph.save() 1673 update_graph_files(graph) 1674 1675 final_nodes = len(graph.nodes) 1676 final_edges = len(graph.edges) 1677 1678 print() 1679 print("─" * 60) 1680 print(f"BACKHAUL COMPLETE") 1681 print(f" Nodes: {initial_nodes} → {final_nodes} (+{final_nodes - initial_nodes})") 1682 print(f" Edges: {initial_edges} → {final_edges} (+{final_edges - initial_edges})") 1683 print() 1684 print("Note: Transcript handshake extraction requires NLP processing.") 1685 print("Future: Process .claude/projects/*.jsonl for synthesis moments.") 1686 print() 1687 1688 1689 def show_orphans(): 1690 """ 1691 Show orphan nodes in the graph with connection suggestions. 1692 1693 This is the hunting party for the persistent graph - finding 1694 nodes that should connect but don't. 1695 """ 1696 print() 1697 print("=" * 70) 1698 print("ORPHAN DETECTION - Unconnected Nodes in Knowledge Graph") 1699 print("=" * 70) 1700 print() 1701 1702 graph = KnowledgeGraph() 1703 stats = graph.get_stats() 1704 print(f"Graph: {stats['total_nodes']} nodes, {stats['total_edges']} edges") 1705 print() 1706 1707 orphans = graph.find_orphans() 1708 1709 if not orphans: 1710 print("No orphans found - every node has at least one connection!") 1711 print() 1712 return 1713 1714 total_orphans = sum(len(nodes) for nodes in orphans.values()) 1715 orphan_pct = (total_orphans / stats['total_nodes']) * 100 if stats['total_nodes'] > 0 else 0 1716 1717 print(f"ORPHAN SUMMARY: {total_orphans} orphans ({orphan_pct:.1f}% of graph)") 1718 print() 1719 1720 # Show by type 1721 print("-" * 70) 1722 print("ORPHANS BY TYPE:") 1723 print("-" * 70) 1724 1725 for node_type in sorted(orphans.keys(), key=lambda t: len(orphans[t]), reverse=True): 1726 nodes = orphans[node_type] 1727 print(f"\n {node_type.upper()} ({len(nodes)} orphans):") 1728 1729 # Show first few with suggestions 1730 for node in nodes[:5]: 1731 label = node.label if len(node.label) <= 40 else node.label[:37] + "..." 1732 axiom_str = f" [{','.join(node.axioms)}]" if node.axioms else "" 1733 print(f" - {label}{axiom_str}") 1734 1735 # Get connection suggestions 1736 suggestions = graph.suggest_connections(node, max_suggestions=2) 1737 if suggestions: 1738 for target_id, edge_type, confidence in suggestions: 1739 if target_id in graph.nodes: 1740 target = graph.nodes[target_id] 1741 target_label = target.label if len(target.label) <= 25 else target.label[:22] + "..." 1742 print(f" → {edge_type} → {target_label} (conf: {confidence:.2f})") 1743 1744 if len(nodes) > 5: 1745 print(f" ... and {len(nodes) - 5} more") 1746 1747 print() 1748 print("-" * 70) 1749 print("RECOMMENDATIONS:") 1750 print("-" * 70) 1751 print() 1752 1753 # Prioritize orphan resolution 1754 high_value_orphans = [] 1755 for node_type, nodes in orphans.items(): 1756 for node in nodes: 1757 # High-value = has axioms or high importance 1758 if node.axioms or node.importance >= 0.7: 1759 high_value_orphans.append(node) 1760 1761 if high_value_orphans: 1762 print(f" HIGH-VALUE ORPHANS ({len(high_value_orphans)} - should connect to axioms):") 1763 for node in high_value_orphans[:10]: 1764 print(f" - [{node.node_type}] {node.label[:40]}...") 1765 suggestions = graph.suggest_connections(node, max_suggestions=1) 1766 if suggestions: 1767 target_id, edge_type, confidence = suggestions[0] 1768 if target_id in graph.nodes: 1769 print(f" Best match: {graph.nodes[target_id].label[:30]} ({confidence:.2f})") 1770 1771 print() 1772 print(" RUN: python3 scripts/graph_feeder.py --auto-connect") 1773 print(" to auto-create edges for high-confidence suggestions") 1774 print() 1775 1776 1777 def auto_connect_orphans(min_confidence: float = 0.5, dry_run: bool = False): 1778 """ 1779 Automatically create edges for orphan nodes with high-confidence suggestions. 1780 1781 This reduces the orphan rate by connecting nodes that should logically 1782 connect based on axiom matching, keyword overlap, and type affinity. 1783 1784 Args: 1785 min_confidence: Minimum confidence threshold for creating edges (default 0.5) 1786 dry_run: If True, show what would be connected without actually connecting 1787 """ 1788 print() 1789 print("=" * 70) 1790 print("AUTO-CONNECT ORPHANS - Creating High-Confidence Edges") 1791 print("=" * 70) 1792 print() 1793 1794 graph = KnowledgeGraph() 1795 stats_before = graph.get_stats() 1796 orphans_before = graph.find_orphans() 1797 total_orphans_before = sum(len(nodes) for nodes in orphans_before.values()) 1798 1799 print(f"BEFORE: {stats_before['total_nodes']} nodes, {stats_before['total_edges']} edges") 1800 print(f" {total_orphans_before} orphans ({total_orphans_before/stats_before['total_nodes']*100:.1f}%)") 1801 print(f"Confidence threshold: {min_confidence}") 1802 print(f"Mode: {'DRY RUN' if dry_run else 'LIVE'}") 1803 print() 1804 1805 if dry_run: 1806 print("-" * 70) 1807 print("DRY RUN - Would create these edges:") 1808 print("-" * 70) 1809 1810 edges_created = 0 1811 connections_by_type = {} 1812 1813 # Process all orphans 1814 all_orphans = [] 1815 for node_type, nodes in orphans_before.items(): 1816 all_orphans.extend(nodes) 1817 1818 for orphan in all_orphans: 1819 suggestions = graph.suggest_connections(orphan, max_suggestions=3) 1820 1821 for target_id, edge_type, confidence in suggestions: 1822 if confidence >= min_confidence: 1823 if target_id not in graph.nodes: 1824 continue 1825 1826 target = graph.nodes[target_id] 1827 1828 if dry_run: 1829 print(f" [{orphan.node_type}] {orphan.label[:30]}") 1830 print(f" --({edge_type}, {confidence:.2f})--> {target.label[:30]}") 1831 else: 1832 # Check if edge already exists 1833 edge_exists = any( 1834 (e.source_id == orphan.id and e.target_id == target_id) or 1835 (e.source_id == target_id and e.target_id == orphan.id) 1836 for e in graph.edges 1837 ) 1838 1839 if not edge_exists: 1840 graph.edges.append(GraphEdge( 1841 source_id=orphan.id, 1842 target_id=target_id, 1843 edge_type=edge_type, 1844 strength=confidence, 1845 created_at=datetime.now().isoformat() 1846 )) 1847 edges_created += 1 1848 1849 # Track by type 1850 key = f"{orphan.node_type} -> {target.node_type}" 1851 connections_by_type[key] = connections_by_type.get(key, 0) + 1 1852 1853 # Only connect to the best match per orphan 1854 break 1855 1856 if not dry_run and edges_created > 0: 1857 # Save the graph 1858 graph.save() 1859 update_graph_files(graph) 1860 1861 # Calculate new stats 1862 stats_after = graph.get_stats() 1863 orphans_after = graph.find_orphans() 1864 total_orphans_after = sum(len(nodes) for nodes in orphans_after.values()) 1865 1866 print() 1867 print("-" * 70) 1868 print("RESULTS:") 1869 print("-" * 70) 1870 print() 1871 print(f" Edges created: {edges_created}") 1872 print() 1873 print(" Connections by type:") 1874 for conn_type, count in sorted(connections_by_type.items(), key=lambda x: x[1], reverse=True): 1875 print(f" {conn_type}: {count}") 1876 print() 1877 print(f" BEFORE: {total_orphans_before} orphans ({total_orphans_before/stats_before['total_nodes']*100:.1f}%)") 1878 print(f" AFTER: {total_orphans_after} orphans ({total_orphans_after/stats_after['total_nodes']*100:.1f}%)") 1879 print(f" REDUCTION: {total_orphans_before - total_orphans_after} orphans connected") 1880 print() 1881 1882 # Publish to mesh 1883 publish_to_mesh("graph_auto_connect", { 1884 "edges_created": edges_created, 1885 "orphans_before": total_orphans_before, 1886 "orphans_after": total_orphans_after, 1887 "reduction_pct": (total_orphans_before - total_orphans_after) / total_orphans_before * 100 if total_orphans_before > 0 else 0 1888 }) 1889 1890 elif dry_run: 1891 print() 1892 print("-" * 70) 1893 print(f"DRY RUN COMPLETE - Would create edges for orphans above {min_confidence} confidence") 1894 print("Run without --dry-run to actually create edges") 1895 print("-" * 70) 1896 else: 1897 print() 1898 print("No edges created - no orphans met the confidence threshold") 1899 print(f"Try lowering threshold: --auto-connect --threshold 0.4") 1900 1901 print() 1902 1903 1904 if __name__ == "__main__": 1905 if len(sys.argv) > 1: 1906 if sys.argv[1] == "--watch": 1907 run_watch() 1908 elif sys.argv[1] == "--status": 1909 show_status() 1910 elif sys.argv[1] == "--backhaul": 1911 backhaul_historical() 1912 elif sys.argv[1] == "--orphans": 1913 show_orphans() 1914 elif sys.argv[1] == "--auto-connect": 1915 # Parse additional options 1916 dry_run = "--dry-run" in sys.argv 1917 threshold = 0.5 1918 for i, arg in enumerate(sys.argv): 1919 if arg == "--threshold" and i + 1 < len(sys.argv): 1920 try: 1921 threshold = float(sys.argv[i + 1]) 1922 except ValueError: 1923 print(f"Invalid threshold: {sys.argv[i + 1]}") 1924 sys.exit(1) 1925 auto_connect_orphans(min_confidence=threshold, dry_run=dry_run) 1926 elif sys.argv[1] in ["--help", "-h"]: 1927 print(__doc__) 1928 else: 1929 print(f"Unknown option: {sys.argv[1]}") 1930 print(__doc__) 1931 else: 1932 run_once()