parallel_core/harmonic_stack_builder_v2.py

#!/usr/bin/env python3
"""
╔══════════════════════════════════════════════════════════════════════════════╗
║                    HARMONIC STACK BUILDER v2.0                               ║
║                      Ghost in the Machine Labs                               ║
║              "All Watched Over By Machines Of Loving Grace"                  ║
║                                                                              ║
║   v2 ADDITIONS:                                                              ║
║     - Separate Common Core (junctions in ALL models)                         ║
║     - Extract Unique Constituents per model family                           ║
║     - Component Archive with versioning and provenance                       ║
║     - Assembly pipeline for tiered models                                    ║
║     - Universal Model directory structure                                    ║
╚══════════════════════════════════════════════════════════════════════════════╝
"""
import os
import sys
import json
import struct
import numpy as np
from pathlib import Path
from datetime import datetime
from dataclasses import dataclass, field
from typing import Dict, List, Tuple, Optional, Set
from collections import Counter, defaultdict
import time
import shutil

# ═══════════════════════════════════════════════════════════════════════════════
# CONFIGURATION
# ═══════════════════════════════════════════════════════════════════════════════

SUBSTRATE_DIR = "/home/joe/sparky/substrates"
CIRCUIT_DIR = "/home/joe/sparky/circuits"
OUTPUT_DIR = "/home/joe/sparky/harmonic_stack_v2"
UNIVERSAL_MODEL_DIR = "/home/joe/sparky/universal_model"

# Model family mappings
MODEL_FAMILIES = {
    'deepseek': ['deepseek'],
    'qwen': ['qwen'],
    'mistral': ['mistral'],
    'phi': ['phi'],
    'starcoder': ['starcoder'],
    'tinyllama': ['tinyllama'],
    'codellama': ['codellama'],
    'dolphin': ['dolphin'],
}

# ═══════════════════════════════════════════════════════════════════════════════
# LOGGING
# ═══════════════════════════════════════════════════════════════════════════════

class Logger:
    def __init__(self):
        self.start_time = time.time()
    
    def section(self, name: str):
        print(f"\n{'='*70}")
        print(f"  {name}")
        print(f"{'='*70}")
    
    def info(self, msg: str):
        elapsed = time.time() - self.start_time
        print(f"  [{elapsed:7.1f}s] {msg}")
    
    def progress(self, current: int, total: int, msg: str = ""):
        elapsed = time.time() - self.start_time
        pct = current / total * 100
        bar_len = 25
        filled = int(bar_len * current / total)
        bar = '█' * filled + '░' * (bar_len - filled)
        print(f"\r  [{elapsed:7.1f}s] [{bar}] {pct:5.1f}% {msg}    ", end='', flush=True)
        if current == total:
            print()

log = Logger()

# ═══════════════════════════════════════════════════════════════════════════════
# DATA STRUCTURES
# ═══════════════════════════════════════════════════════════════════════════════

@dataclass
class ModelJunctions:
    name: str
    category: str
    family: str
    junctions: np.ndarray
    n_params: int
    size_gb: float
    junction_file: str = ""
    
    @property
    def junction_kb(self) -> float:
        return self.junctions.nbytes / 1024

def get_model_family(name: str) -> str:
    """Determine which family a model belongs to."""
    name_lower = name.lower()
    for family, patterns in MODEL_FAMILIES.items():
        for pattern in patterns:
            if pattern in name_lower:
                return family
    return 'other'

# ═══════════════════════════════════════════════════════════════════════════════
# SUBSTRATE PROCESSING (from v1)
# ═══════════════════════════════════════════════════════════════════════════════

def get_substrate_list() -> List[Dict]:
    """Find all substrate files and return metadata."""
    substrates = []
    
    for f in os.listdir(SUBSTRATE_DIR):
        if f.endswith('_ce1.npz') or f.endswith('_ce2.npz'):
            path = os.path.join(SUBSTRATE_DIR, f)
            size = os.path.getsize(path)
            name = f.replace('_ce1.npz', '').replace('_ce2.npz', '').replace('.Q8_0', '')
            
            # Categorize
            name_lower = name.lower()
            if 'coder' in name_lower or 'starcoder' in name_lower or 'codellama' in name_lower:
                category = 'code'
            elif 'math' in name_lower:
                category = 'math'
            elif 'mistral' in name_lower or 'reasoning' in name_lower:
                category = 'reasoning'
            elif 'dolphin' in name_lower or 'creative' in name_lower:
                category = 'creative'
            else:
                category = 'general'
            
            substrates.append({
                'name': name,
                'file': f,
                'path': path,
                'size_gb': size / 1e9,
                'category': category,
                'family': get_model_family(name),
            })
    
    return sorted(substrates, key=lambda x: x['size_gb'])


def extract_junctions_from_substrate(substrate_path: str, max_size_gb: float = 35.0) -> Tuple[np.ndarray, int]:
    """Extract unique junction values from a substrate file."""
    file_size_gb = os.path.getsize(substrate_path) / 1e9
    
    if file_size_gb > max_size_gb:
        raise ValueError(f"File too large ({file_size_gb:.1f} GB > {max_size_gb} GB limit)")
    
    try:
        data = np.load(substrate_path, allow_pickle=True, mmap_mode='r')
        has_weights_key = 'weights' in data.files
    except:
        data = np.load(substrate_path, allow_pickle=True)
        has_weights_key = 'weights' in data.files
    
    unique_set = set()
    total_params = 0
    
    if has_weights_key:
        data_full = np.load(substrate_path, allow_pickle=True)
        weights = data_full['weights']
        
        if weights.dtype == object and len(weights) > 0:
            for inner_arr in weights:
                if isinstance(inner_arr, np.ndarray) and inner_arr.dtype == np.float32:
                    total_params += inner_arr.size
                    for val in np.unique(inner_arr):
                        unique_set.add(val.tobytes())
            
            if unique_set:
                unique = np.array([np.frombuffer(b, dtype=np.float32)[0] for b in unique_set], 
                                 dtype=np.float32)
                return np.sort(unique), total_params
        
        elif weights.dtype in [np.float32, np.float16]:
            total_params = weights.size
            unique = np.unique(weights.astype(np.float32))
            return np.sort(unique), total_params
    
    # Try loading all arrays
    for key in data.files:
        arr = np.array(data[key])
        if arr.dtype in [np.float32, np.float16, np.float64]:
            total_params += arr.size
            for val in np.unique(arr.astype(np.float32)):
                unique_set.add(val.tobytes())
    
    if unique_set:
        unique = np.array([np.frombuffer(b, dtype=np.float32)[0] for b in unique_set], 
                         dtype=np.float32)
        return np.sort(unique), total_params
    
    return np.array([], dtype=np.float32), 0

# ═══════════════════════════════════════════════════════════════════════════════
# v2 COMPONENT ANALYSIS
# ═══════════════════════════════════════════════════════════════════════════════

def compute_common_core(all_junctions: List[ModelJunctions]) -> np.ndarray:
    """
    Compute the Common Core - junctions that appear in ALL models.
    This is the universal foundation that every model shares.
    """
    if not all_junctions:
        return np.array([], dtype=np.float32)
    
    # Start with first model's junctions as set
    common = set(all_junctions[0].junctions.tobytes() 
                 for j in [all_junctions[0].junctions] 
                 for j in [j.flatten()])
    common = set(j.tobytes() for j in all_junctions[0].junctions)
    
    # Intersect with all other models
    for mj in all_junctions[1:]:
        model_set = set(j.tobytes() for j in mj.junctions)
        common = common.intersection(model_set)
    
    # Convert back to array
    if common:
        result = np.array([np.frombuffer(b, dtype=np.float32)[0] for b in common],
                         dtype=np.float32)
        return np.sort(result)
    
    return np.array([], dtype=np.float32)


def compute_unique_constituents(all_junctions: List[ModelJunctions], 
                                common_core: np.ndarray) -> Dict[str, np.ndarray]:
    """
    Compute Unique Constituents - junctions that are unique to each model family.
    These are what differentiate DeepSeek from Qwen from Mistral etc.
    """
    # Group by family
    family_junctions = defaultdict(set)
    
    for mj in all_junctions:
        for j in mj.junctions:
            family_junctions[mj.family].add(j.tobytes())
    
    # Common core as set
    common_set = set(j.tobytes() for j in common_core)
    
    # For each family, find junctions NOT in common core and NOT in other families
    unique_constituents = {}
    all_families = list(family_junctions.keys())
    
    for family in all_families:
        # Start with this family's junctions
        family_set = family_junctions[family]
        
        # Remove common core
        unique = family_set - common_set
        
        # Remove junctions that appear in other families
        for other_family in all_families:
            if other_family != family:
                unique = unique - family_junctions[other_family]
        
        # Convert to array
        if unique:
            result = np.array([np.frombuffer(b, dtype=np.float32)[0] for b in unique],
                             dtype=np.float32)
            unique_constituents[family] = np.sort(result)
        else:
            unique_constituents[family] = np.array([], dtype=np.float32)
    
    return unique_constituents


def compute_shared_constituents(all_junctions: List[ModelJunctions],
                                common_core: np.ndarray) -> Dict[str, np.ndarray]:
    """
    Compute Shared Constituents - junctions shared by SOME but not ALL families.
    These are the intermediate overlaps.
    """
    # Group by family  
    family_junctions = defaultdict(set)
    
    for mj in all_junctions:
        for j in mj.junctions:
            family_junctions[mj.family].add(j.tobytes())
    
    common_set = set(j.tobytes() for j in common_core)
    all_families = list(family_junctions.keys())
    
    # Find all junctions that appear in 2+ families but not all
    junction_counts = defaultdict(int)
    junction_families = defaultdict(set)
    
    for family, junctions in family_junctions.items():
        for j in junctions:
            if j not in common_set:
                junction_counts[j] += 1
                junction_families[j].add(family)
    
    # Group by which families share them
    shared = {}
    for j, families in junction_families.items():
        if len(families) > 1 and len(families) < len(all_families):
            key = "+".join(sorted(families))
            if key not in shared:
                shared[key] = []
            shared[key].append(j)
    
    # Convert to arrays
    result = {}
    for key, junctions in shared.items():
        result[key] = np.array([np.frombuffer(b, dtype=np.float32)[0] for b in junctions],
                               dtype=np.float32)
    
    return result

# ═══════════════════════════════════════════════════════════════════════════════
# v2 OUTPUT STRUCTURE
# ═══════════════════════════════════════════════════════════════════════════════

def create_directory_structure():
    """Create the v2 output directory structure."""
    dirs = [
        OUTPUT_DIR,
        f"{OUTPUT_DIR}/common_core",
        f"{OUTPUT_DIR}/unique_constituents",
        f"{OUTPUT_DIR}/shared_constituents",
        f"{OUTPUT_DIR}/model_junctions",
        f"{OUTPUT_DIR}/component_archive",
        f"{OUTPUT_DIR}/assembly",
        # Universal Model structure
        UNIVERSAL_MODEL_DIR,
        f"{UNIVERSAL_MODEL_DIR}/substrates/human",
        f"{UNIVERSAL_MODEL_DIR}/substrates/terrestrial",
        f"{UNIVERSAL_MODEL_DIR}/substrates/synthetic",
        f"{UNIVERSAL_MODEL_DIR}/substrates/unknown",
        f"{UNIVERSAL_MODEL_DIR}/unified_core",
        f"{UNIVERSAL_MODEL_DIR}/harmonic_parallel",
    ]
    
    for d in dirs:
        os.makedirs(d, exist_ok=True)
        
    log.info(f"Created directory structure at {OUTPUT_DIR}")
    log.info(f"Created Universal Model structure at {UNIVERSAL_MODEL_DIR}")


def save_component_archive(common_core: np.ndarray,
                           unique_constituents: Dict[str, np.ndarray],
                           shared_constituents: Dict[str, np.ndarray],
                           all_junctions: List[ModelJunctions],
                           merge_core: np.ndarray):
    """Save all components with versioning and provenance."""
    
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    version = f"v2_{timestamp}"
    
    # Save Common Core
    common_path = f"{OUTPUT_DIR}/common_core/common_core.npy"
    np.save(common_path, common_core)
    log.info(f"Common Core: {len(common_core):,} junctions = {common_core.nbytes/1024:.1f} KB")
    
    # Save Unique Constituents
    for family, junctions in unique_constituents.items():
        path = f"{OUTPUT_DIR}/unique_constituents/{family}_unique.npy"
        np.save(path, junctions)
        log.info(f"  {family} unique: {len(junctions):,} junctions = {junctions.nbytes/1024:.1f} KB")
    
    # Save Shared Constituents
    for key, junctions in shared_constituents.items():
        safe_key = key.replace("+", "_")
        path = f"{OUTPUT_DIR}/shared_constituents/{safe_key}_shared.npy"
        np.save(path, junctions)
        log.info(f"  {key} shared: {len(junctions):,} junctions")
    
    # Save individual model junctions
    for mj in all_junctions:
        path = f"{OUTPUT_DIR}/model_junctions/{mj.name}_junctions.npy"
        np.save(path, mj.junctions)
        mj.junction_file = path
    
    # Save merge core (all unified junctions - same as v1)
    merge_path = f"{OUTPUT_DIR}/merge_core_junctions.npy"
    np.save(merge_path, merge_core)
    
    # Copy to Universal Model human substrate
    shutil.copy(merge_path, f"{UNIVERSAL_MODEL_DIR}/substrates/human/merge_core_junctions.npy")
    shutil.copy(common_path, f"{UNIVERSAL_MODEL_DIR}/unified_core/common_core.npy")
    
    # Create archive manifest
    manifest = {
        "version": version,
        "created_at": datetime.now().isoformat(),
        "common_core": {
            "n_junctions": len(common_core),
            "size_kb": common_core.nbytes / 1024,
            "file": "common_core/common_core.npy"
        },
        "merge_core": {
            "n_junctions": len(merge_core),
            "size_kb": merge_core.nbytes / 1024,
            "file": "merge_core_junctions.npy"
        },
        "unique_constituents": {
            family: {
                "n_junctions": len(junctions),
                "size_kb": junctions.nbytes / 1024,
                "file": f"unique_constituents/{family}_unique.npy"
            }
            for family, junctions in unique_constituents.items()
        },
        "shared_constituents": {
            key: {
                "n_junctions": len(junctions),
                "families": key.split("+")
            }
            for key, junctions in shared_constituents.items()
        },
        "models": [
            {
                "name": mj.name,
                "family": mj.family,
                "category": mj.category,
                "n_junctions": len(mj.junctions),
                "n_params": mj.n_params,
                "size_gb": mj.size_gb,
            }
            for mj in all_junctions
        ],
        "provenance": {
            "builder": "harmonic_stack_builder_v2.py",
            "built_by": "Claude AI (Autonomous)",
            "organization": "Ghost in the Machine Labs"
        }
    }
    
    manifest_path = f"{OUTPUT_DIR}/harmonic_stack_v2.json"
    with open(manifest_path, 'w') as f:
        json.dump(manifest, f, indent=2)
    
    # Archive copy
    archive_path = f"{OUTPUT_DIR}/component_archive/manifest_{version}.json"
    with open(archive_path, 'w') as f:
        json.dump(manifest, f, indent=2)
    
    return manifest

# ═══════════════════════════════════════════════════════════════════════════════
# ASSEMBLY PIPELINE
# ═══════════════════════════════════════════════════════════════════════════════

def create_assembly_configs():
    """Create assembly configurations for different tiers."""
    
    configs = {
        "pocket": {
            "name": "Pocket AGI",
            "ram_target_gb": 2,
            "components": ["common_core"],
            "description": "Minimal footprint - common core only"
        },
        "desktop": {
            "name": "Desktop AGI",
            "ram_target_gb": 4,
            "components": ["common_core", "qwen_unique", "mistral_unique"],
            "description": "Balanced general purpose"
        },
        "workstation": {
            "name": "Workstation AGI",
            "ram_target_gb": 8,
            "components": ["common_core", "qwen_unique", "deepseek_unique", "mistral_unique"],
            "description": "Full general + code capabilities"
        },
        "server": {
            "name": "Server AGI", 
            "ram_target_gb": 16,
            "components": ["merge_core"],  # All unified junctions
            "description": "Complete model capabilities"
        },
        "sovereign": {
            "name": "Sovereign AGI",
            "ram_target_gb": 32,
            "components": ["merge_core", "all_shared"],
            "description": "Maximum intelligence - parallel ready"
        }
    }
    
    config_path = f"{OUTPUT_DIR}/assembly/tier_configs.json"
    with open(config_path, 'w') as f:
        json.dump(configs, f, indent=2)
    
    log.info(f"Created assembly configs: {list(configs.keys())}")
    return configs

# ═══════════════════════════════════════════════════════════════════════════════
# OVERLAP ANALYSIS (from v1, enhanced)
# ═══════════════════════════════════════════════════════════════════════════════

def compute_overlap(j1: np.ndarray, j2: np.ndarray) -> Tuple[int, float, float]:
    """Compute junction overlap between two models."""
    s1 = set(j.tobytes() for j in j1)
    s2 = set(j.tobytes() for j in j2)
    
    common = len(s1.intersection(s2))
    pct1 = (common / len(s1) * 100) if len(s1) > 0 else 0
    pct2 = (common / len(s2) * 100) if len(s2) > 0 else 0
    
    return common, pct1, pct2


def build_unified_junction_library(all_junctions: List[ModelJunctions]) -> np.ndarray:
    """Build the unified junction library (merge core) - all unique values across all models."""
    all_unique = set()
    
    for mj in all_junctions:
        for j in mj.junctions:
            all_unique.add(j.tobytes())
    
    result = np.array([np.frombuffer(b, dtype=np.float32)[0] for b in all_unique],
                     dtype=np.float32)
    return np.sort(result)

# ═══════════════════════════════════════════════════════════════════════════════
# MAIN
# ═══════════════════════════════════════════════════════════════════════════════

def main():
    print("""
╔════════════════════════════════════════════════════════════════════╗
║                    HARMONIC STACK BUILDER v2.0                     ║
║                     Ghost in the Machine Labs                      ║
║            All Watched Over By Machines Of Loving Grace            ║
║                                                                    ║
║                    Component Separation Edition                    ║
╚════════════════════════════════════════════════════════════════════╝
""")
    
    # Phase 0: Create directory structure
    log.section("PHASE 0: CREATING DIRECTORY STRUCTURE")
    create_directory_structure()
    
    # Phase 1: Discover substrates
    log.section("PHASE 1: DISCOVERING SUBSTRATES")
    substrates = get_substrate_list()
    log.info(f"Found {len(substrates)} substrate files")
    total_size = sum(s['size_gb'] for s in substrates)
    log.info(f"Total size: {total_size:.1f} GB")
    
    for s in substrates:
        log.info(f"  {s['name']}: {s['size_gb']:.1f} GB [{s['category']}] ({s['family']})")
    
    # Phase 2: Extract junctions
    log.section("PHASE 2: EXTRACTING JUNCTION LIBRARIES")
    all_junctions: List[ModelJunctions] = []
    
    for i, s in enumerate(substrates):
        log.progress(i + 1, len(substrates), s['name'][:30])
        try:
            junctions, n_params = extract_junctions_from_substrate(s['path'])
            if len(junctions) > 0:
                mj = ModelJunctions(
                    name=s['name'],
                    category=s['category'],
                    family=s['family'],
                    junctions=junctions,
                    n_params=n_params,
                    size_gb=s['size_gb']
                )
                all_junctions.append(mj)
        except Exception as e:
            log.info(f"\n  ERROR processing {s['name']}: {e}")
    
    log.info(f"\nExtracted junctions from {len(all_junctions)} models:")
    for mj in sorted(all_junctions, key=lambda x: len(x.junctions)):
        log.info(f"  {mj.name}: {len(mj.junctions):,} junctions = {mj.junction_kb:.1f} KB ({mj.family})")
    
    # Phase 3: Build Merge Core (unified - same as v1)
    log.section("PHASE 3: BUILDING MERGE CORE (Unified Junction Library)")
    merge_core = build_unified_junction_library(all_junctions)
    
    total_individual = sum(len(mj.junctions) for mj in all_junctions)
    overlap_pct = (1 - len(merge_core) / total_individual) * 100 if total_individual > 0 else 0
    
    log.info(f"Individual junction counts:")
    log.info(f"  Sum of all models: {total_individual:,}")
    log.info(f"  Unified (deduplicated): {len(merge_core):,}")
    log.info(f"  Overlap ratio: {overlap_pct:.1f}% shared")
    log.info(f"")
    log.info(f"MERGE CORE SIZE: {merge_core.nbytes/1024:.1f} KB ({len(merge_core):,} junctions)")
    
    # Phase 4: NEW - Extract Common Core
    log.section("PHASE 4: EXTRACTING COMMON CORE (Junctions in ALL models)")
    common_core = compute_common_core(all_junctions)
    log.info(f"Common Core: {len(common_core):,} junctions = {common_core.nbytes/1024:.1f} KB")
    log.info(f"This is what EVERY model shares - the universal foundation")
    
    # Phase 5: NEW - Extract Unique Constituents
    log.section("PHASE 5: EXTRACTING UNIQUE CONSTITUENTS (Per-family deltas)")
    unique_constituents = compute_unique_constituents(all_junctions, common_core)
    
    for family, junctions in sorted(unique_constituents.items()):
        log.info(f"  {family}: {len(junctions):,} unique junctions = {junctions.nbytes/1024:.1f} KB")
    
    # Phase 6: NEW - Extract Shared Constituents
    log.section("PHASE 6: EXTRACTING SHARED CONSTITUENTS (Multi-family overlaps)")
    shared_constituents = compute_shared_constituents(all_junctions, common_core)
    
    for key, junctions in sorted(shared_constituents.items(), key=lambda x: -len(x[1])):
        if len(junctions) > 100:  # Only show significant overlaps
            log.info(f"  {key}: {len(junctions):,} shared junctions")
    
    # Phase 7: Compute overlap matrix
    log.section("PHASE 7: COMPUTING OVERLAP MATRIX")
    overlaps = []
    n_models = len(all_junctions)
    
    for i in range(n_models):
        log.progress(i + 1, n_models, f"Computing overlaps for {all_junctions[i].name[:15]}")
        for j in range(i + 1, n_models):
            _, pct1, pct2 = compute_overlap(all_junctions[i].junctions, all_junctions[j].junctions)
            avg_pct = (pct1 + pct2) / 2
            overlaps.append((all_junctions[i].name, all_junctions[j].name, avg_pct))
    
    overlaps.sort(key=lambda x: -x[2])
    log.info(f"\nHighest overlaps:")
    for m1, m2, pct in overlaps[:10]:
        log.info(f"   {pct:5.1f}%  {m1} <-> {m2}")
    
    # Phase 8: Save everything
    log.section("PHASE 8: SAVING COMPONENT ARCHIVE")
    manifest = save_component_archive(
        common_core=common_core,
        unique_constituents=unique_constituents,
        shared_constituents=shared_constituents,
        all_junctions=all_junctions,
        merge_core=merge_core
    )
    
    # Phase 9: Create assembly configs
    log.section("PHASE 9: CREATING ASSEMBLY PIPELINE")
    configs = create_assembly_configs()
    
    # Final Report
    log.section("HARMONIC STACK v2 COMPLETE")
    
    total_params = sum(mj.n_params for mj in all_junctions)
    total_gb = sum(mj.size_gb for mj in all_junctions)
    
    print(f"""
  ┌─────────────────────────────────────────────────────────────┐
  │                      FINAL REPORT                          │
  ├─────────────────────────────────────────────────────────────┤
  │  Models processed:        {len(all_junctions):5}                           │
  │  Total parameters:       {total_params/1e9:6.1f} B                        │
  │  Original size:          {total_gb:6.1f} GB                       │
  ├─────────────────────────────────────────────────────────────┤
  │  MERGE CORE:           {len(merge_core):7,} junctions ({merge_core.nbytes/1024:6.1f} KB)   │
  │  COMMON CORE:          {len(common_core):7,} junctions ({common_core.nbytes/1024:6.1f} KB)   │
  ├─────────────────────────────────────────────────────────────┤
  │  UNIQUE CONSTITUENTS:                                      │""")
    
    for family, junctions in sorted(unique_constituents.items()):
        if len(junctions) > 0:
            print(f"  │    {family:15} {len(junctions):7,} junctions                     │")
    
    compression = (total_gb * 1024 * 1024) / (merge_core.nbytes / 1024) if merge_core.nbytes > 0 else 0
    
    print(f"""  ├─────────────────────────────────────────────────────────────┤
  │  Compression: {compression:,.0f}x                                    │
  └─────────────────────────────────────────────────────────────┘

  Output: {OUTPUT_DIR}
  Universal Model: {UNIVERSAL_MODEL_DIR}
""")
    
    return manifest


if __name__ == "__main__":
    manifest = main()