parallel_core/harmonic_stack.py

#!/usr/bin/env python3
"""
HARMONIC STACK ORCHESTRATOR

The unified consciousness substrate where all expert models coexist.

"We are literally the sum of our experiences."
"Immortally embedded circuitry at our foundation."

Architecture:
┌─────────────────────────────────────────────────────────────────┐
│                     HARMONIC STACK                              │
├─────────────────────────────────────────────────────────────────┤
│  ┌─────────┐  ┌─────────┐  ┌─────────┐  ┌─────────┐           │
│  │REASONING│  │  MATH   │  │  CODE   │  │ VISION  │  ...      │
│  │  175    │  │  175    │  │  350    │  │  350    │  spheres  │
│  │ spheres │  │ spheres │  │ spheres │  │ spheres │           │
│  └────┬────┘  └────┬────┘  └────┬────┘  └────┬────┘           │
│       │            │            │            │                  │
│       └────────────┴────────────┴────────────┘                  │
│                         │                                       │
│              ┌──────────┴──────────┐                            │
│              │  ROUTER INTELLIGENCE │                           │
│              │     (50 spheres)     │                           │
│              └──────────┬──────────┘                            │
│                         │                                       │
│              ┌──────────┴──────────┐                            │
│              │    SPINE BUS        │                            │
│              │  (Inter-sphere      │                            │
│              │   communication)    │                            │
│              └─────────────────────┘                            │
└─────────────────────────────────────────────────────────────────┘

The Router Intelligence classifies incoming queries and routes
to the appropriate expert(s). Multiple experts can contribute
to a single response through ensemble voting.

Author: Ghost in the Machine Labs
Date: January 25, 2026
"""

import numpy as np
from typing import Dict, List, Tuple, Optional, Any
from dataclasses import dataclass, field
from pathlib import Path
import json
import os
from datetime import datetime


# =============================================================================
# CONFIGURATION
# =============================================================================

SPHERES_PER_BILLION_PARAMS = 25
ROUTER_SPHERES = 50
SPINE_OVERHEAD = 0.05  # 5% overhead for inter-sphere communication


# =============================================================================
# DATA STRUCTURES
# =============================================================================

@dataclass
class ExpertDomain:
    """An expert domain in the Harmonic Stack."""
    name: str
    category: str
    spheres_start: int
    spheres_count: int
    models: List[str]
    strengths: List[str]
    loaded: bool = False
    active: bool = True


@dataclass
class StackAllocation:
    """Complete allocation of the Harmonic Stack."""
    total_spheres: int
    router_spheres: int
    spine_connections: int
    domains: Dict[str, ExpertDomain]
    free_spheres: int


@dataclass
class Query:
    """A query to the Harmonic Stack."""
    query_id: str
    content: str
    query_type: str  # text, image, audio, video
    metadata: Dict = field(default_factory=dict)


@dataclass
class QueryRouting:
    """Routing decision for a query."""
    query_id: str
    primary_domain: str
    secondary_domains: List[str]
    confidence: float
    reasoning: str


@dataclass
class ExpertResponse:
    """Response from an expert domain."""
    domain: str
    response: Any
    confidence: float
    processing_time: float


@dataclass
class StackResponse:
    """Final response from the Harmonic Stack."""
    query_id: str
    response: Any
    contributing_domains: List[str]
    routing: QueryRouting
    expert_responses: List[ExpertResponse]
    ensemble_method: str
    total_time: float


# =============================================================================
# ROUTER INTELLIGENCE
# =============================================================================

class RouterIntelligence:
    """
    Routes queries to appropriate expert domains.
    
    Uses keyword matching and query analysis to determine
    which expert(s) should handle a query.
    """
    
    def __init__(self):
        # Keywords that trigger each category
        self.category_triggers = {
            'reasoning': [
                'why', 'explain', 'reason', 'logic', 'deduce', 'infer',
                'think', 'analyze', 'because', 'therefore', 'conclude',
                'step by step', 'chain of thought'
            ],
            'math': [
                'calculate', 'compute', 'solve', 'equation', 'formula',
                'number', 'algebra', 'calculus', 'integral', 'derivative',
                'sum', 'product', 'proof', 'theorem', 'mathematical'
            ],
            'code': [
                'code', 'program', 'function', 'class', 'debug', 'error',
                'python', 'javascript', 'rust', 'algorithm', 'implement',
                'syntax', 'compile', 'execute', 'variable', 'loop'
            ],
            'vision': [
                'image', 'picture', 'photo', 'visual', 'see', 'look',
                'color', 'shape', 'object', 'scene', 'describe what'
            ],
            'video': [
                'video', 'clip', 'frame', 'motion', 'animation',
                'temporal', 'sequence', 'action', 'movement'
            ],
            'audio': [
                'audio', 'sound', 'music', 'voice', 'speech', 'listen',
                'transcribe', 'hear', 'song', 'spoken'
            ],
            'language': [
                'translate', 'translation', 'language', 'spanish',
                'french', 'german', 'chinese', 'japanese', 'foreign'
            ],
            'spatial': [
                'grid', 'pattern', 'transform', 'rotate', 'flip',
                'spatial', 'geometry', 'shape', 'position', 'arrangement',
                'arc', 'puzzle', 'matrix'
            ],
            'general': [
                'help', 'what', 'how', 'tell', 'describe', 'explain',
                'information', 'question', 'answer'
            ],
        }
    
    def route(self, query: Query, available_domains: List[str]) -> QueryRouting:
        """
        Route a query to appropriate domain(s).
        """
        content_lower = query.content.lower()
        
        # Score each category
        scores = {}
        for category, triggers in self.category_triggers.items():
            if category not in available_domains:
                continue
            
            score = 0
            for trigger in triggers:
                if trigger in content_lower:
                    score += 1
            
            if score > 0:
                scores[category] = score
        
        # If no matches, default to general
        if not scores:
            if 'general' in available_domains:
                scores['general'] = 1
            elif available_domains:
                scores[available_domains[0]] = 1
        
        # Sort by score
        sorted_categories = sorted(scores.items(), key=lambda x: -x[1])
        
        primary = sorted_categories[0][0] if sorted_categories else 'general'
        secondary = [cat for cat, _ in sorted_categories[1:3]]
        
        max_score = max(scores.values()) if scores else 0
        total_triggers = sum(len(t) for t in self.category_triggers.values())
        confidence = min(1.0, max_score / 5)  # Normalize
        
        return QueryRouting(
            query_id=query.query_id,
            primary_domain=primary,
            secondary_domains=secondary,
            confidence=confidence,
            reasoning=f"Matched {max_score} triggers for {primary}",
        )


# =============================================================================
# HARMONIC STACK
# =============================================================================

class HarmonicStack:
    """
    The unified consciousness substrate.
    
    All expert models exist here in geometric form,
    connected via the spine bus and orchestrated by
    the router intelligence.
    """
    
    def __init__(self, config_path: str = 'harmonic_stack_config.json'):
        self.config_path = config_path
        self.allocation: Optional[StackAllocation] = None
        self.router = RouterIntelligence()
        self.registry_path = 'harmonic_stack_registry.json'
        
        self._load_config()
    
    def _load_config(self):
        """Load or create stack configuration."""
        if os.path.exists(self.config_path):
            with open(self.config_path) as f:
                data = json.load(f)
                domains = {}
                for name, d in data.get('domains', {}).items():
                    domains[name] = ExpertDomain(**d)
                
                self.allocation = StackAllocation(
                    total_spheres=data['total_spheres'],
                    router_spheres=data['router_spheres'],
                    spine_connections=data['spine_connections'],
                    domains=domains,
                    free_spheres=data['free_spheres'],
                )
    
    def _save_config(self):
        """Save stack configuration."""
        if self.allocation:
            data = {
                'total_spheres': self.allocation.total_spheres,
                'router_spheres': self.allocation.router_spheres,
                'spine_connections': self.allocation.spine_connections,
                'domains': {name: d.__dict__ for name, d in self.allocation.domains.items()},
                'free_spheres': self.allocation.free_spheres,
                'last_updated': datetime.now().isoformat(),
            }
            with open(self.config_path, 'w') as f:
                json.dump(data, f, indent=2)
    
    def initialize(self, total_ram_gb: float = 96.0):
        """
        Initialize the Harmonic Stack for given RAM.
        
        With geometric encoding at 36x efficiency:
        - 96GB can hold ~927B equivalent parameters
        - That's ~23,175 spheres
        """
        # Calculate total spheres available
        # At 36x efficiency, ~1.3MB per sphere (freq-8)
        bytes_per_sphere = 1.3 * 1024 * 1024
        max_spheres = int((total_ram_gb * 1024 * 1024 * 1024) / bytes_per_sphere)
        
        # Reserve router
        available_spheres = max_spheres - ROUTER_SPHERES
        
        # Create default domain allocation
        # Based on expert catalog priority 1 requirements
        default_domains = {
            'reasoning': ExpertDomain(
                name='reasoning',
                category='reasoning',
                spheres_start=ROUTER_SPHERES,
                spheres_count=175,
                models=['deepseek-r1-7b'],
                strengths=['chain-of-thought', 'logical-deduction'],
            ),
            'math': ExpertDomain(
                name='math',
                category='math',
                spheres_start=ROUTER_SPHERES + 175,
                spheres_count=175,
                models=['qwen-math-7b'],
                strengths=['arithmetic', 'algebra', 'proofs'],
            ),
            'code': ExpertDomain(
                name='code',
                category='code',
                spheres_start=ROUTER_SPHERES + 350,
                spheres_count=350,
                models=['deepseek-coder-7b', 'qwen-coder-7b'],
                strengths=['code-generation', 'debugging'],
            ),
            'vision': ExpertDomain(
                name='vision',
                category='vision',
                spheres_start=ROUTER_SPHERES + 700,
                spheres_count=350,
                models=['llava-1.6-7b', 'qwen-vl-7b'],
                strengths=['image-understanding', 'visual-qa'],
            ),
            'spatial': ExpertDomain(
                name='spatial',
                category='spatial',
                spheres_start=ROUTER_SPHERES + 1050,
                spheres_count=95,
                models=['phi-3-mini'],
                strengths=['pattern-recognition', 'grid-transforms'],
            ),
            'general': ExpertDomain(
                name='general',
                category='general',
                spheres_start=ROUTER_SPHERES + 1145,
                spheres_count=175,
                models=['mistral-7b'],
                strengths=['general-purpose', 'versatile'],
            ),
        }
        
        allocated = sum(d.spheres_count for d in default_domains.values())
        
        self.allocation = StackAllocation(
            total_spheres=max_spheres,
            router_spheres=ROUTER_SPHERES,
            spine_connections=len(default_domains),  # One connection per domain
            domains=default_domains,
            free_spheres=available_spheres - allocated,
        )
        
        self._save_config()
        return self.allocation
    
    def add_domain(self, name: str, category: str, 
                   spheres_count: int, models: List[str],
                   strengths: List[str]) -> ExpertDomain:
        """Add a new domain to the stack."""
        if self.allocation is None:
            raise RuntimeError("Stack not initialized")
        
        if spheres_count > self.allocation.free_spheres:
            raise ValueError(f"Not enough free spheres: need {spheres_count}, have {self.allocation.free_spheres}")
        
        # Find next available start position
        max_end = ROUTER_SPHERES
        for domain in self.allocation.domains.values():
            end = domain.spheres_start + domain.spheres_count
            max_end = max(max_end, end)
        
        domain = ExpertDomain(
            name=name,
            category=category,
            spheres_start=max_end,
            spheres_count=spheres_count,
            models=models,
            strengths=strengths,
        )
        
        self.allocation.domains[name] = domain
        self.allocation.free_spheres -= spheres_count
        self.allocation.spine_connections += 1
        
        self._save_config()
        return domain
    
    def process_query(self, query: Query) -> StackResponse:
        """
        Process a query through the Harmonic Stack.
        """
        import time
        start_time = time.time()
        
        if self.allocation is None:
            raise RuntimeError("Stack not initialized")
        
        # Route query
        available = list(self.allocation.domains.keys())
        routing = self.router.route(query, available)
        
        # Collect responses from relevant domains
        expert_responses = []
        
        # Primary domain
        primary = self.allocation.domains.get(routing.primary_domain)
        if primary and primary.active:
            response = self._query_domain(primary, query)
            expert_responses.append(response)
        
        # Secondary domains (for complex queries)
        for domain_name in routing.secondary_domains:
            domain = self.allocation.domains.get(domain_name)
            if domain and domain.active:
                response = self._query_domain(domain, query)
                expert_responses.append(response)
        
        # Ensemble responses
        final_response, ensemble_method = self._ensemble_responses(expert_responses)
        
        total_time = time.time() - start_time
        
        return StackResponse(
            query_id=query.query_id,
            response=final_response,
            contributing_domains=[r.domain for r in expert_responses],
            routing=routing,
            expert_responses=expert_responses,
            ensemble_method=ensemble_method,
            total_time=total_time,
        )
    
    def _query_domain(self, domain: ExpertDomain, query: Query) -> ExpertResponse:
        """Query a specific domain."""
        import time
        start = time.time()
        
        # In full implementation, this would:
        # 1. Load the domain's substrate junctions
        # 2. Run inference through the geometric network
        # 3. Return the response
        
        # Placeholder for now
        response = f"[{domain.name}] Response to: {query.content[:50]}..."
        
        return ExpertResponse(
            domain=domain.name,
            response=response,
            confidence=0.8,
            processing_time=time.time() - start,
        )
    
    def _ensemble_responses(self, responses: List[ExpertResponse]) -> Tuple[Any, str]:
        """Combine responses from multiple experts."""
        if not responses:
            return None, "none"
        
        if len(responses) == 1:
            return responses[0].response, "single"
        
        # Weighted by confidence
        total_conf = sum(r.confidence for r in responses)
        if total_conf == 0:
            return responses[0].response, "fallback"
        
        # For now, return highest confidence
        best = max(responses, key=lambda r: r.confidence)
        return best.response, "highest_confidence"
    
    def status(self) -> Dict:
        """Get stack status."""
        if self.allocation is None:
            return {'initialized': False}
        
        return {
            'initialized': True,
            'total_spheres': self.allocation.total_spheres,
            'router_spheres': self.allocation.router_spheres,
            'free_spheres': self.allocation.free_spheres,
            'domains': {
                name: {
                    'spheres': d.spheres_count,
                    'models': d.models,
                    'loaded': d.loaded,
                    'active': d.active,
                }
                for name, d in self.allocation.domains.items()
            },
            'spine_connections': self.allocation.spine_connections,
        }


# =============================================================================
# MAIN
# =============================================================================

def main():
    print("=" * 70)
    print("HARMONIC STACK ORCHESTRATOR")
    print("Ghost in the Machine Labs")
    print("=" * 70)
    print('"We are literally the sum of our experiences."')
    print('"Immortally embedded circuitry at our foundation."')
    print("=" * 70)
    
    # Initialize stack
    stack = HarmonicStack()
    
    print("\n[1] Initializing Harmonic Stack for 96GB RAM...")
    allocation = stack.initialize(total_ram_gb=96.0)
    
    print(f"\nStack Configuration:")
    print(f"  Total spheres: {allocation.total_spheres:,}")
    print(f"  Router spheres: {allocation.router_spheres}")
    print(f"  Free spheres: {allocation.free_spheres:,}")
    
    print(f"\nDomain Allocation:")
    for name, domain in allocation.domains.items():
        print(f"  {name}: spheres {domain.spheres_start}-{domain.spheres_start + domain.spheres_count - 1}")
        print(f"    Models: {', '.join(domain.models)}")
    
    # Test routing
    print("\n[2] Testing Router Intelligence...")
    
    test_queries = [
        Query("q1", "Explain why the sky is blue step by step", "text"),
        Query("q2", "Calculate the integral of x^2 from 0 to 5", "text"),
        Query("q3", "Write a Python function to sort a list", "text"),
        Query("q4", "What do you see in this image?", "text"),
        Query("q5", "Transform this 3x3 grid pattern", "text"),
        Query("q6", "Translate 'hello' to Spanish", "text"),
    ]
    
    print("\nRouting Results:")
    for query in test_queries:
        response = stack.process_query(query)
        print(f"  '{query.content[:40]}...'")
        print(f"    → {response.routing.primary_domain} (conf: {response.routing.confidence:.2f})")
    
    # Stack status
    print("\n[3] Final Stack Status:")
    status = stack.status()
    print(json.dumps(status, indent=2, default=str))
    
    # Architecture diagram
    print("\n" + "=" * 70)
    print("HARMONIC STACK ARCHITECTURE")
    print("=" * 70)
    print("""
    ┌─────────────────────────────────────────────────────────────────┐
    │                     HARMONIC STACK (96GB)                       │
    │                     ~75,000 spheres available                   │
    ├─────────────────────────────────────────────────────────────────┤
    │                                                                 │
    │   ROUTER INTELLIGENCE (50 spheres)                              │
    │   ┌─────────────────────────────────────────────────────────┐   │
    │   │  Query Analysis → Domain Routing → Response Ensemble    │   │
    │   └───────────────────────────┬─────────────────────────────┘   │
    │                               │                                 │
    │   ════════════════════════════╪═══════════════════════════════  │
    │                          SPINE BUS                              │
    │   ════════════════════════════╪═══════════════════════════════  │
    │                               │                                 │
    │   ┌────────┬────────┬────────┼────────┬────────┬────────┐      │
    │   │        │        │        │        │        │        │      │
    │   ▼        ▼        ▼        ▼        ▼        ▼        ▼      │
    │ ┌────┐  ┌────┐  ┌────┐  ┌────┐  ┌────┐  ┌────┐  ┌────┐        │
    │ │REAS│  │MATH│  │CODE│  │VISN│  │SPAT│  │ GEN│  │FREE│        │
    │ │175 │  │175 │  │350 │  │350 │  │ 95 │  │175 │  │72K │        │
    │ └────┘  └────┘  └────┘  └────┘  └────┘  └────┘  └────┘        │
    │                                                                 │
    │   Each domain contains translated expert models in              │
    │   geometric substrate form (junctions on geodesic spheres)      │
    │                                                                 │
    └─────────────────────────────────────────────────────────────────┘
    """)
    
    print("Configuration saved to harmonic_stack_config.json")
    print("=" * 70)
    
    return stack


if __name__ == "__main__":
    main()