Upload folder using huggingface_hub

.clang-format

@@ -0,0 +1,98 @@
+# NeuroFlow C++ 代码风格配置
+# 基于 Google C++ Style Guide (C++20)
+#
+# 用法:
+#   clang-format -i <file.cpp>          # 就地格式化
+#   clang-format --dry-run <file.cpp>   # 检查而不修改
+#   find src/ include/ -name '*.cpp' -o -name '*.hpp' | xargs clang-format -i
+
+---
+BasedOnStyle: Google
+
+Language: Cpp
+Standard: c++20
+
+# ──── 缩进与空格 ────
+IndentWidth: 4
+UseTab: Never
+TabWidth: 4
+ContinuationIndentWidth: 4
+AccessModifierOffset: -4          # public:/private: 左移1个缩进
+
+# ──── 行宽与换行 ────
+ColumnLimit: 100                   # Google推荐80，NeuroFlow用100（模板/长类型名）
+BreakBeforeBraces: Attach
+AllowShortFunctionsOnASingleLine: Inline
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+
+# ──── 空行 ────
+MaxEmptyLinesToKeep: 1
+SeparateDefinitionBlocks: Always
+
+# ──── Include 排序 ────
+# Google 要求的5层分组:
+#   1. 关联头文件 (.hpp)
+#   2. C 系统头
+#   3. C++ 标准库头
+#   4. 第三方库头
+#   5. 项目头
+SortIncludes: true
+IncludeBlocks: Regroup
+IncludeCategories:
+  # 关联头文件 (对应 .cpp 的同名 .hpp)
+  - Regex: '^".*\.hpp"$'
+    Priority: 1
+  # C 系统头
+  - Regex: '^<.*\.h>$'
+    Priority: 2
+  # C++ 标准库
+  - Regex: '^<(algorithm|chrono|cmath|cstdint|cstdio|cstdlib|cstring|exception|filesystem|fstream|functional|iostream|memory|numeric|random|sstream|stdexcept|string|thread|unordered_map|utility|vector)>$'
+    Priority: 3
+  # POSIX / 平台头
+  - Regex: '^<(fcntl|sys/|unistd|windows|omp)'
+    Priority: 4
+  # CBLAS
+  - Regex: '^<cblas\.h>$'
+    Priority: 4
+  # 第三方库 / 项目内部
+  - Regex: '^"(neuroflow|weight_io)/'
+    Priority: 5
+  - Regex: '.*'
+    Priority: 6
+
+# ──── 指针对齐 ────
+DerivePointerAlignment: false
+PointerAlignment: Left              # int* p (类型对齐)
+
+# ──── 空格 ────
+SpaceAfterCStyleCast: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpacesInAngles: false               # vector<int> 而非 vector< int >
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+
+# ──── 大括号 ────
+BreakBeforeBraces: Custom
+BraceWrapping:
+  AfterClass: false
+  AfterControlStatement: Never
+  AfterEnum: false
+  AfterFunction: false
+  AfterNamespace: false
+  AfterStruct: false
+  AfterUnion: false
+  BeforeCatch: false
+  BeforeElse: false
+  IndentBraces: false
+  AfterExternBlock: false
+
+# ──── 函数声明 ────
+AlwaysBreakAfterReturnType: None
+AlwaysBreakTemplateDeclarations: false
+
+# ──── 其他 ────
+ReflowComments: false               # 不自动重排注释（含中文）
+FixNamespaceComments: true          # }  // namespace neuroflow

.clang-tidy

@@ -0,0 +1,95 @@
+# NeuroFlow C++ 静态分析检查配置
+# 基于 Google C++ Style Guide — 只启用不与现有代码大面积冲突的规则
+#
+# 用法:
+#   clang-tidy <file.cpp> -- -Iinclude -std=c++20
+#   # 或通过 CMake:
+#   cmake -DCMAKE_CXX_CLANG_TIDY="clang-tidy" ..
+
+---
+# ──── Google 核心规范检查 ────
+Checks: >
+  -*,
+  # 命名规范（仅新代码警告，旧代码不阻塞）
+  google-readability-*,
+  google-runtime-*,
+  google-build-*,
+  google-explicit-constructor,
+  google-global-names-in-headers,
+  
+  # C++ 现代写法检查
+  modernize-use-override,
+  modernize-use-nullptr,
+  modernize-use-using,
+  modernize-make-unique,
+  modernize-make-shared,
+  modernize-use-noexcept,
+  modernize-use-emplace,
+  modernize-loop-convert,
+  modernize-return-braced-init-list,
+  modernize-use-auto,
+  modernize-use-default-member-init,
+  
+  # 性能检查
+  performance-unnecessary-value-param,
+  performance-move-const-arg,
+  performance-for-range-copy,
+  performance-unnecessary-copy-initialization,
+  
+  # 可读性检查
+  readability-redundant-member-init,
+  readability-container-size-empty,
+  readability-simplify-boolean-expr,
+  readability-redundant-string-cstr,
+  readability-implicit-bool-conversion,
+  readability-make-member-function-const,
+  readability-avoid-const-params-in-decls,
+  
+  # Bug 预防
+  bugprone-unused-raii,
+  bugprone-use-after-move,
+  bugprone-suspicious-include,
+  bugprone-misplaced-widening-cast,
+  bugprone-integer-division,
+  bugprone-suspicious-memset-usage,
+
+  # 杂项
+  misc-unused-parameters,
+  misc-unused-using-decls,
+  misc-include-cleaner,
+  misc-const-correctness
+
+# ──── 故意不启用的检查（与现有代码冲突严重，将来逐步开启）────
+#   google-readability-casting        → 需大量 static_cast 替换
+#   readability-identifier-naming     → 等重命名完成后再开
+#   modernize-pass-by-value           → 需逐文件评估
+#   cppcoreguidelines-pro-type-const-cast → 太严格
+#   clang-analyzer-*                  → 太慢，CI 阶段用
+
+# ──── 配置项 ────
+CheckOptions:
+  # modernize-use-auto: MinTypeNameLength=5（长类型名才建议 auto）
+  - key: modernize-use-auto.MinTypeNameLength
+    value: '5'
+  
+  # modernize-use-auto: 移除星号时用 auto（而非 auto*）
+  - key: modernize-use-auto.RemoveStars
+    value: 'true'
+  
+  # misc-include-cleaner: 忽略标准库头（编译器内置路径）
+  - key: misc-include-cleaner.IgnoreHeaders
+    value: '^<.*\.h>$;^<.*>$'
+  
+  # readability-implicit-bool-conversion: 允许条件表达式中的隐式转换
+  - key: readability-implicit-bool-conversion.AllowIntegerConditions
+    value: 'true'
+  - key: readability-implicit-bool-conversion.AllowPointerConditions
+    value: 'true'
+  
+  # performance-unnecessary-value-param: 允许的值类型上限
+  - key: performance-unnecessary-value-param.AllowedTypes
+    value: 'std::string;std::vector'
+
+# ──── WarningsAsErrors ────
+# 把它注释掉表示不阻塞编译，只在 CI 中打开
+# WarningsAsErrors: '*'

.gitattributes

@@ -1,35 +1,5 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
+*.nfv1 filter=lfs diff=lfs merge=lfs -text
+*.nfv2 filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+scripts/test_generate filter=lfs diff=lfs merge=lfs -text

CMakeLists.txt

@@ -0,0 +1,255 @@
+cmake_minimum_required(VERSION 3.14)
+
+option(NEUROFLOW_USE_CUDA "Enable CUDA/cuBLAS backend" OFF)
+
+if(NEUROFLOW_USE_CUDA)
+    project(neuroflow_core VERSION 1.0.0 LANGUAGES CXX CUDA)
+else()
+    project(neuroflow_core VERSION 1.0.0 LANGUAGES CXX)
+endif()
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+option(NEUROFLOW_BUILD_TESTS "Build unit tests" OFF)
+option(NEUROFLOW_BUILD_PYTHON "Build Python bindings" OFF)
+option(NEUROFLOW_USE_AVX2 "Enable AVX2 SIMD" ON)
+option(NEUROFLOW_USE_BLAS "Use external BLAS (OpenBLAS/MKL) for GEMM" ON)
+option(NEUROFLOW_USE_AMP "Enable mixed precision training (FP16/BF16)" OFF)
+
+if(MSVC)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W3 /GR /EHsc /utf-8")
+    if(NEUROFLOW_USE_AVX2)
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
+        message(STATUS "AVX2 SIMD enabled (MSVC)")
+    endif()
+    set(CMAKE_CXX_FLAGS_RELEASE "/O2 /DNDEBUG /GL")
+    set(CMAKE_CXX_FLAGS_DEBUG "/Od /Zi /DDEBUG")
+    find_package(OpenMP)
+else()
+    find_package(OpenMP REQUIRED)
+
+    if(NEUROFLOW_USE_AVX2)
+        include(CheckCXXCompilerFlag)
+        check_cxx_compiler_flag("-mavx2" COMPILER_SUPPORTS_AVX2)
+        if(COMPILER_SUPPORTS_AVX2)
+            set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2")
+            message(STATUS "AVX2 SIMD enabled")
+        else()
+            message(STATUS "AVX2 not supported by compiler")
+        endif()
+    endif()
+
+    set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG -march=native -mtune=native -funroll-loops -ffast-math -ftree-vectorize")
+    set(CMAKE_CXX_FLAGS_DEBUG "-g -O0 -DDEBUG")
+endif()
+
+
+add_library(neuroflow_core STATIC
+    src/tensor.cpp
+    src/model.cpp
+    src/weight_io.cpp
+    src/tokenizer.cpp
+    src/sampling.cpp
+    src/causal_lm.cpp
+    src/tensor_ops.cpp
+    src/generative_model.cpp
+    src/rope.cpp
+    src/swiglu.cpp
+    src/rms_norm.cpp
+    src/adamw.cpp
+    src/scheduler.cpp
+    src/train_lm.cpp
+    src/grad_scaler.cpp
+    src/sft_train.cpp
+    src/dpo_train.cpp
+)
+
+if(NEUROFLOW_USE_CUDA)
+    find_package(CUDAToolkit REQUIRED)
+    if(CUDAToolkit_VERSION VERSION_LESS "11.0")
+        message(FATAL_ERROR "CUDA >= 11.0 required, found ${CUDAToolkit_VERSION}")
+    endif()
+    target_compile_definitions(neuroflow_core PUBLIC USE_CUDA)
+    target_include_directories(neuroflow_core PUBLIC ${CUDAToolkit_INCLUDE_DIRS})
+    target_link_libraries(neuroflow_core PUBLIC CUDA::cudart CUDA::cublas)
+    target_sources(neuroflow_core PRIVATE src/cuda_context.cpp)
+    set_source_files_properties(src/cuda_context.cpp PROPERTIES LANGUAGE CUDA)
+    target_compile_options(neuroflow_core PRIVATE
+        $<$<COMPILE_LANGUAGE:CUDA>:--generate-code=arch=compute_80,code=[sm_80,compute_80]>
+        $<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr>
+        $<$<COMPILE_LANGUAGE:CUDA>:--extended-lambda>
+    )
+    set(NEUROFLOW_CUDA_FOUND TRUE)
+    message(STATUS "CUDA backend enabled: ${CUDAToolkit_VERSION}")
+else()
+    set(NEUROFLOW_CUDA_FOUND FALSE)
+endif()
+
+target_include_directories(neuroflow_core PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+    ${CMAKE_CURRENT_SOURCE_DIR}/src
+)
+
+target_compile_definitions(neuroflow_core PRIVATE
+    $<$<CONFIG:DEBUG>:DEBUG>
+)
+
+if(NEUROFLOW_USE_BLAS)
+    find_package(BLAS)
+    if(BLAS_FOUND)
+        target_compile_definitions(neuroflow_core PUBLIC USE_CBLAS)
+        target_link_libraries(neuroflow_core PUBLIC ${BLAS_LIBRARIES})
+        if(BLAS_INCLUDE_DIRS)
+            target_include_directories(neuroflow_core PUBLIC ${BLAS_INCLUDE_DIRS})
+        endif()
+        set(NEUROFLOW_BLAS_FOUND TRUE)
+        message(STATUS "External BLAS found: ${BLAS_LIBRARIES}")
+        message(STATUS "  NOTE: Set OPENBLAS_NUM_THREADS=1 at runtime to avoid thread oversubscription with OpenMP")
+    else()
+        set(OPENBLAS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/openblas")
+        if(EXISTS "${OPENBLAS_DIR}/include/cblas.h" AND EXISTS "${OPENBLAS_DIR}/lib/libscipy_openblas.lib")
+            target_compile_definitions(neuroflow_core PUBLIC USE_CBLAS)
+            target_include_directories(neuroflow_core PUBLIC "${OPENBLAS_DIR}/include")
+            target_link_libraries(neuroflow_core PUBLIC "${OPENBLAS_DIR}/lib/libscipy_openblas.lib")
+            set(NEUROFLOW_BLAS_FOUND TRUE)
+            message(STATUS "Using bundled OpenBLAS from: ${OPENBLAS_DIR}")
+            message(STATUS "  NOTE: Set OPENBLAS_NUM_THREADS=1 at runtime to avoid thread oversubscription with OpenMP")
+        else()
+            set(NEUROFLOW_BLAS_FOUND FALSE)
+            message(STATUS "External BLAS not found, using built-in GEMM")
+        endif()
+    endif()
+else()
+    set(NEUROFLOW_BLAS_FOUND FALSE)
+endif()
+
+if(NEUROFLOW_USE_AMP)
+    if(NOT NEUROFLOW_USE_CUDA)
+        message(WARNING "NEUROFLOW_USE_AMP requires CUDA, enabling CUDA automatically")
+        set(NEUROFLOW_USE_CUDA ON)
+    endif()
+    target_compile_definitions(neuroflow_core PUBLIC NEUROFLOW_USE_AMP)
+    message(STATUS "Mixed precision training (AMP) enabled")
+endif()
+
+if(OpenMP_CXX_FOUND)
+    target_link_libraries(neuroflow_core PUBLIC OpenMP::OpenMP_CXX)
+endif()
+
+add_executable(neuroflow_train_v2 src/train_v2.cpp)
+target_link_libraries(neuroflow_train_v2 PRIVATE neuroflow_core)
+if(OpenMP_CXX_FOUND)
+    target_link_libraries(neuroflow_train_v2 PRIVATE OpenMP::OpenMP_CXX)
+endif()
+
+
+add_executable(neuroflow_infer_v2 src/infer_v2.cpp)
+target_link_libraries(neuroflow_infer_v2 PRIVATE neuroflow_core)
+if(OpenMP_CXX_FOUND)
+    target_link_libraries(neuroflow_infer_v2 PRIVATE OpenMP::OpenMP_CXX)
+endif()
+
+add_executable(neuroflow_sft src/sft_main.cpp)
+target_link_libraries(neuroflow_sft PRIVATE neuroflow_core)
+if(OpenMP_CXX_FOUND)
+    target_link_libraries(neuroflow_sft PRIVATE OpenMP::OpenMP_CXX)
+endif()
+
+add_executable(neuroflow_dpo src/dpo_main.cpp)
+target_link_libraries(neuroflow_dpo PRIVATE neuroflow_core)
+if(OpenMP_CXX_FOUND)
+    target_link_libraries(neuroflow_dpo PRIVATE OpenMP::OpenMP_CXX)
+endif()
+
+if(NEUROFLOW_BUILD_TESTS)
+    enable_testing()
+
+    set(NEUROFLOW_TEST_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/tests)
+
+    add_executable(neuroflow_test_lm tests/test_lm_pathway.cpp)
+    target_link_libraries(neuroflow_test_lm PRIVATE neuroflow_core)
+    target_include_directories(neuroflow_test_lm PRIVATE ${NEUROFLOW_TEST_INCLUDE})
+    add_test(NAME lm_pathway_test COMMAND neuroflow_test_lm)
+
+    if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_tensor.cpp)
+        add_executable(neuroflow_tensor_test tests/test_tensor.cpp)
+        target_link_libraries(neuroflow_tensor_test PRIVATE neuroflow_core)
+        target_include_directories(neuroflow_tensor_test PRIVATE ${NEUROFLOW_TEST_INCLUDE})
+        add_test(NAME tensor_test COMMAND neuroflow_tensor_test)
+    endif()
+
+    if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_model.cpp)
+        add_executable(neuroflow_model_test tests/test_model.cpp)
+        target_link_libraries(neuroflow_model_test PRIVATE neuroflow_core)
+        target_include_directories(neuroflow_model_test PRIVATE ${NEUROFLOW_TEST_INCLUDE})
+        add_test(NAME model_test COMMAND neuroflow_model_test)
+    endif()
+
+    if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_rope.cpp)
+        add_executable(neuroflow_test_rope tests/test_rope.cpp)
+        target_link_libraries(neuroflow_test_rope PRIVATE neuroflow_core)
+        target_include_directories(neuroflow_test_rope PRIVATE ${NEUROFLOW_TEST_INCLUDE})
+        add_test(NAME rope_test COMMAND neuroflow_test_rope)
+    endif()
+
+    if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_swiglu.cpp)
+        add_executable(neuroflow_test_swiglu tests/test_swiglu.cpp)
+        target_link_libraries(neuroflow_test_swiglu PRIVATE neuroflow_core)
+        target_include_directories(neuroflow_test_swiglu PRIVATE ${NEUROFLOW_TEST_INCLUDE})
+        add_test(NAME swiglu_test COMMAND neuroflow_test_swiglu)
+    endif()
+
+    if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_rms_norm.cpp)
+        add_executable(neuroflow_test_rms_norm tests/test_rms_norm.cpp)
+        target_link_libraries(neuroflow_test_rms_norm PRIVATE neuroflow_core)
+        target_include_directories(neuroflow_test_rms_norm PRIVATE ${NEUROFLOW_TEST_INCLUDE})
+        add_test(NAME rms_norm_test COMMAND neuroflow_test_rms_norm)
+    endif()
+
+    if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_adamw.cpp)
+        add_executable(neuroflow_test_adamw tests/test_adamw.cpp)
+        target_link_libraries(neuroflow_test_adamw PRIVATE neuroflow_core)
+        target_include_directories(neuroflow_test_adamw PRIVATE ${NEUROFLOW_TEST_INCLUDE})
+        add_test(NAME adamw_test COMMAND neuroflow_test_adamw)
+    endif()
+
+    if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_scheduler.cpp)
+        add_executable(neuroflow_test_scheduler tests/test_scheduler.cpp)
+        target_link_libraries(neuroflow_test_scheduler PRIVATE neuroflow_core)
+        target_include_directories(neuroflow_test_scheduler PRIVATE ${NEUROFLOW_TEST_INCLUDE})
+        add_test(NAME scheduler_test COMMAND neuroflow_test_scheduler)
+    endif()
+
+    if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_gqa.cpp)
+        add_executable(neuroflow_test_gqa tests/test_gqa.cpp)
+        target_link_libraries(neuroflow_test_gqa PRIVATE neuroflow_core)
+        target_include_directories(neuroflow_test_gqa PRIVATE ${NEUROFLOW_TEST_INCLUDE})
+        add_test(NAME gqa_test COMMAND neuroflow_test_gqa)
+    endif()
+
+    message(STATUS "Unit tests enabled (built-in test framework)")
+endif()
+
+install(TARGETS neuroflow_core
+    ARCHIVE DESTINATION lib
+    LIBRARY DESTINATION lib
+)
+
+install(DIRECTORY include/neuroflow
+    DESTINATION include
+)
+
+message(STATUS "")
+message(STATUS "NeuroFlow Core Build Configuration:")
+message(STATUS "  Version:        ${PROJECT_VERSION}")
+message(STATUS "  C++ Standard:   ${CMAKE_CXX_STANDARD}")
+message(STATUS "  Build Type:     ${CMAKE_BUILD_TYPE}")
+message(STATUS "  AVX2:           ${NEUROFLOW_USE_AVX2}")
+message(STATUS "  AMP:            ${NEUROFLOW_USE_AMP}")
+message(STATUS "  Tests:          ${NEUROFLOW_BUILD_TESTS}")
+message(STATUS "  Python:         ${NEUROFLOW_BUILD_PYTHON}")
+message(STATUS "  OpenMP:         ${OpenMP_CXX_FOUND}")
+message(STATUS "  BLAS:           ${NEUROFLOW_BLAS_FOUND}")
+message(STATUS "  CUDA:           ${NEUROFLOW_CUDA_FOUND}")

README.md

@@ -0,0 +1,113 @@
+---
+language:
+- zh
+- en
+- code
+license: apache-2.0
+library_name: cpp
+tags:
+- neuroflow
+- causal-lm
+- sn
+- ecn
+- dmn
+- memory-augmented
+- transformer
+- cpp
+- llm
+pipeline_tag: text-generation
+---
+
+# NeuroFlow C++ LLM
+
+NeuroFlow is a **memory-augmented causal language model** implemented entirely in **C++17**, featuring a three-brain architecture (SN/ECN/DMN) inspired by cognitive neuroscience. Designed for efficient training and inference on consumer GPUs.
+
+## Model Architecture
+
+| Parameter | Value | Description |
+|-----------|-------|-------------|
+| `d_model` | 512 | Model dimension |
+| `hidden_dim` | 2048 | FFN hidden dimension |
+| `memory_dim` | 512 | Memory dimension |
+| `num_layers` | 12 | ECN layers |
+| `memory_slots` | 64 | Memory slots |
+| `num_associations` | 8 | DMN association heads |
+| `vocab_size` | 128,000 | 128K multilingual BPE tokenizer |
+| `max_seq_len` | 512 | Maximum sequence length |
+| `causal_window_size` | 64 | Causal attention window |
+| `lm_num_attn_layers` | 2 | Causal LM attention layers |
+| `params` | ~120M | Total parameters |
+
+### Three-Brain Architecture
+- **SN (Sensory Network)**: Input encoding and feature extraction
+- **ECN (Executive Control Network)**: Core reasoning and processing layers
+- **DMN (Default Mode Network)**: Memory-augmented association and retrieval
+
+## Files
+
+| File | Size | Description |
+|------|------|-------------|
+| `output/checkpoint_step1000/model.nfv1` | 431 MB | Checkpoint at 1000 steps |
+| `output/checkpoint_step2000/model.nfv1` | 431 MB | Checkpoint at 2000 steps |
+| `output/lm_head_lmh1.nfv1` | 255 MB | LM head weights (native format v1) |
+| `configs/config.json` | — | Model architecture configuration |
+| `configs/tokenizer_128k.json` | — | 128K BPE tokenizer |
+| `configs/huggingface/` | — | HuggingFace-compatible tokenizer files (vocab.json, merges.txt) |
+
+### Source Code
+The full C++ source is included under `src/` and `include/` directories:
+- **Core**: `tensor.hpp/cpp`, `model.hpp/cpp`, `tokenizer.hpp/cpp`
+- **Architecture**: `causal_lm.hpp/cpp`, `generative_model.hpp/cpp`, `networks.hpp`
+- **Training**: `train_lm.hpp/cpp`, `train_v2.cpp`, `sft_train.cpp`, `dpo_train.cpp`
+- **CUDA**: `cuda_context.hpp/cpp`, `cuda_kernels.hpp`, `tensor_ops.cpp`
+- **Optimizers**: `adamw.hpp/cpp`, `scheduler.hpp/cpp`, `grad_scaler.hpp/cpp`
+
+## Build & Train
+
+### Prerequisites
+- CMake ≥ 3.15
+- C++17 compiler (GCC ≥ 9, MSVC 2019+)
+- CUDA Toolkit ≥ 11.4 (optional, for GPU training)
+- BLAS (OpenBLAS recommended)
+
+### Build
+```bash
+# CPU only
+mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Release
+make -j$(nproc)
+
+# With CUDA
+mkdir build_cuda && cd build_cuda
+cmake .. -DNEUROFLOW_USE_CUDA=ON -DCMAKE_BUILD_TYPE=Release
+make -j$(nproc)
+```
+
+### Train
+```bash
+./build_cuda/neuroflow_train_v2 \
+  --config configs/config_distill.json \
+  --data data/distill_train.txt \
+  --output output \
+  --epochs 20 \
+  --batch-size 64 \
+  --lr 0.0003 \
+  --use-cuda --adam
+```
+
+## Training Scripts
+
+Key Python scripts in `scripts/`:
+- `train_distill.py` — Knowledge distillation training pipeline
+- `preprocess_distill.py` — Data preprocessing for distillation
+- `deploy_dsw.sh` — One-click deployment for Alibaba Cloud DSW (A10 GPU)
+- `train_optimized.sh` — Optimized multi-stage training
+
+## License
+
+Apache 2.0
+
+## Links
+
+- [GitHub Repository](https://github.com/chenzhiwenhphp12-afk/neuroflow-model)
+- [HuggingFace Mirror](https://hf-mirror.com/cwenzi/neuroflow-cpp)

README_CPP.md

@@ -0,0 +1,80 @@
+# NeuroFlow C++ Core
+
+高性能C++底层实现，提供Python绑定接口。
+
+## 特点
+
+- **轻量化**: 相比原Python版本减少80%+内存占用
+- **SIMD优化**: AVX2/ARM NEON加速矩阵运算
+- **量化支持**: INT8/FP8量化，4x内存压缩
+- **MLA技术**: DeepSeek KV压缩，87.5%+内存节省
+- **长记忆**: 分页记忆系统，支持磁盘溢出
+- **零依赖**: 单静态库，无外部依赖
+
+## 构建
+
+```bash
+cd cpp_core
+chmod +x build.sh
+./build.sh build    # 构建核心库
+./build.sh test     # 构建并测试
+./build.sh python   # 构建Python绑定
+```
+
+## 目录结构
+
+```
+cpp_core/
+├── include/neuroflow/
+│   ├── tensor.hpp      # 张量运算引擎 (SIMD)
+│   ├── networks.hpp    # ECN/DMN/SN网络
+│   ├── memory.hpp      # MLA/记忆模块
+│   └── model.hpp       # 主模型类
+├── src/
+│   ├── tensor.cpp
+│   └── model.cpp
+├── bindings/
+│   └── python_bindings.cpp  # pybind11绑定
+├── tests/
+│   ├── test_tensor.cpp
+│   └── test_model.cpp
+├── CMakeLists.txt
+└── build.sh
+```
+
+## Python使用
+
+```python
+import neuroflow_cpp as nf
+
+# 创建模型
+config = nf.ModelConfig(
+    input_dim=512,
+    hidden_dim=256,
+    output_dim=10,
+    use_quantization=True
+)
+model = nf.NeuroFlowModel(config)
+
+# 前向传播
+import numpy as np
+x = np.random.randn(32, 512).astype(np.float32)
+output = model.forward(x)
+
+print(output.output.shape)  # (32, 10)
+print(output.decision.shape)
+print(output.manifold.shape)  # 如果return_manifold=True
+
+# 性能对比
+stats = nf.benchmark()
+print(f"Size reduction: {stats['size_reduction']*100:.1f}%")
+```
+
+## 性能对比
+
+| 版本 | 参数量 | 内存 | 推理时间 | 相比原版 |
+|------|--------|------|----------|----------|
+| C++ Original | 1.25M | 5 MB | 2.5 ms | baseline |
+| C++ Optimized | 171K | 0.7 MB | 1.2 ms | 2x加速 |
+| C++ Lite | 79K | 0.3 MB | 0.8 ms | 3x加速 |
+| C++ Quantized | 79K | 0.08 MB | 0.6 ms | 4x加速 |

README_MULTIMODAL.md

@@ -0,0 +1,199 @@
+# NeuroFlow MultiModal - 多模态类脑神经网络
+
+## 概述
+
+NeuroFlow MultiModal 是一个融合**类脑模块化设计**与**多模态能力**的轻量化神经网络，支持文本+图像的视觉-语言理解任务。
+
+## 核心特性
+
+### 1. 类脑模块化架构 (Brain-Inspired Modular Architecture)
+- **ECN (Executive Control Network)** - 执行控制网络，模拟前额叶，处理推理决策
+- **DMN (Default Mode Network)** - 默认模式网络，模拟后扣带回，处理联想记忆
+- **SN (Salience Network)** - 显著性网络，模拟前岛叶，处理注意力分配
+
+### 2. 多模态能力 (MultiModal Capabilities)
+- **Vision Encoder** - 轻量ViT风格图像编码器
+- **Cross-Modal Fusion** - 文本-图像跨模态融合层
+- **MultiModal Attention** - 跨模态注意力机制
+
+### 3. 技术亮点
+- SIMD优化 (AVX2 + ARM NEON)
+- MLA KV压缩 (87.5%内存节省)
+- INT8量化 (81%模型缩减)
+- LTP记忆巩固机制
+- 分页长记忆系统
+
+## 性能数据
+
+| 模型 | 参数量 | 推理时间 | 特点 |
+|------|--------|----------|------|
+| Full MultiModal | 231,705 | 39.81 ms | 完整功能 |
+| Lite MultiModal | 43,177 | 0.40 ms | 量化+压缩 |
+| **Speedup** | **81%↓** | **98x↑** | 超轻量 |
+
+## 架构图
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                   NeuroFlow MultiModal                      │
+├─────────────────────────────────────────────────────────────┤
+│                                                             │
+│  ┌──────────────┐    ┌──────────────┐                      │
+│  │ Text Input   │    │ Image Input  │                      │
+│  │ (batch, dim) │    │ (batch,C,H,W)│                      │
+│  └──────┬───────┘    └──────┬───────┘                      │
+│         │                   │                              │
+│         ▼                   ▼                              │
+│  ┌──────────────┐    ┌──────────────┐                      │
+│  │ Text Project │    │Vision Encoder│                      │
+│  │   (Linear)   │    │ (ViT-style)  │                      │
+│  └──────┬───────┘    └──────┬───────┘                      │
+│         │                   │                              │
+│         └───────────────────┼───────────────────┐          │
+│                             │                   │          │
+│                             ▼                   │          │
+│                   ┌───────────────────┐         │          │
+│                   │Cross-Modal Fusion │         │          │
+│                   │ (Text+Image Align)│         │          │
+│                   └───────┬───────────┘         │          │
+│                           │                     │          │
+│                           ▼                     │          │
+│              ┌─────────────────────────────┐    │          │
+│              │    MultiModal Attention     │    │          │
+│              │  (Text attends to Image)    │    │          │
+│              └──────────────┬──────────────┘    │          │
+│                             │                   │          │
+│                             ▼                   ▼          │
+│              ┌─────────────────────────────────────────┐  │
+│              │              Fused Features             │  │
+│              └──────────────────┬──────────────────────┘  │
+│                                 │                         │
+│         ┌───────────────────────┼───────────────────────┐ │
+│         │                       │                       │ │
+│         ▼                       ▼                       ▼ │
+│  ┌────────────┐         ┌────────────┐         ┌────────────┐
+│  │    SN      │         │    ECN     │         │    DMN     │
+│  │(Salience)  │────────►│(Executive) │◄───────►│(Default)   │
+│  │Attention   │         │ Control    │         │Mode Memory │
+│  └──────┬─────┘         └──────┬─────┘         └──────┬─────┘
+│         │                      │                      │    │
+│         │    ┌─────────────────┼──────────────────────┘    │
+│         │    │                 │                           │
+│         ▼    ▼                 ▼                           │
+│  ┌─────────────────────────────────────────────────────┐  │
+│  │              Memory Consolidation (LTP)             │  │
+│  │            (Long-term Memory Storage)               │  │
+│  └─────────────────────────┬───────────────────────────┘  │
+│                            │                              │
+│                            ▼                              │
+│                   ┌─────────────────┐                     │
+│                   │   Output Layer  │                     │
+│                   │   (Decision)    │                     │
+│                   └─────────────────┘                     │
+│                                                            │
+└─────────────────────────────────────────────────────────────┘
+```
+
+## 使用方法
+
+### C++ 接口
+
+```cpp
+#include "neuroflow/multimodal_model.hpp"
+
+using namespace neuroflow;
+
+// 创建配置
+NeuroFlowMultiModal::Config cfg;
+cfg.text_dim = 512;
+cfg.image_size = 224;
+cfg.patch_size = 16;
+cfg.vision_dim = 256;
+cfg.fusion_dim = 256;
+cfg.hidden_dim = 256;
+cfg.output_dim = 10;
+
+// 创建模型
+NeuroFlowMultiModal model(cfg);
+
+// 文本输入
+Tensor text({batch, text_dim});
+
+// 图像输入 (batch, channels, height, width)
+Tensor image({batch, 3, 224, 224});
+
+// 多模态推理
+auto output = model.forward_multimodal(text, image);
+
+// 纯文本推理
+auto output = model.forward_text(text);
+
+// 纯图像推理
+auto output = model.forward_image_only(image);
+```
+
+### 输出结构
+
+```cpp
+struct Output {
+    Tensor output;              // 最终决策输出
+    Tensor decision;            // ECN推理决策
+    Tensor value;               // OFC价值评估
+    Tensor saliency;            // SN显著性评分
+    Tensor text_image_sim;      // 文本-图像相似度
+    Tensor vision_feat;         // 视觉特征
+    Tensor text_feat;           // 文本特征
+    Tensor fused_feat;          // 融合特征
+    Tensor retrieved_mem;       // 检索记忆
+    Tensor manifold;            // 神经流形
+};
+```
+
+## 编译
+
+```bash
+cd cpp_core
+mkdir build && cd build
+cmake ..
+make -j$(nproc)
+
+# 运行测试
+./neuroflow_multimodal_test
+```
+
+## 文件结构
+
+```
+cpp_core/
+├── include/neuroflow/
+│   ├── tensor.hpp          # SIMD张量运算
+│   ├── networks.hpp        # ECN/DMN/SN类脑网络
+│   ├── memory.hpp          # MLA+分页记忆
+│   ├── multimodal.hpp      # Vision Encoder + Cross-Modal
+│   ├── multimodal_model.hpp # 多模态模型整合
+│   └── model.hpp           # 原版单模态模型
+├── tests/
+│   ├── test_tensor.cpp
+│   ├── test_model.cpp
+│   └── test_multimodal.cpp
+└── CMakeLists.txt
+```
+
+## 10项要求检测
+
+| 要求 | 状态 | 说明 |
+|------|------|------|
+| 1. 轻量化 | ✓ | 纯C++17，无外部依赖，Lite版43K参数 |
+| 2. 架构先进 | ✓ | ViT+类脑模块+MLA+Cross-Modal Attention |
+| 3. 执行效率高 | ✓ | SIMD优化，98x加速 |
+| 4. 低算力需求 | ✓ | INT8量化，81%缩减，CPU可运行 |
+| 5. 运行速度快 | ✓ | Lite版0.4ms，98x加速 |
+| 6. 长记忆 | ✓ | MLA KV+分页内存+LTP巩固 |
+| 7. 准确度高 | ✓ | 所有测试通过，量化误差<0.02 |
+| 8. 自我升级 | ✓ | consolidate()在线学习，LTP更新 |
+| 9. 简单易部署 | ✓ | CMake一键编译，pybind11绑定 |
+| 10. 易维护 | ✓ | 模块化设计，完整测试套件 |
+
+## License
+
+MIT

SKILL.md

@@ -0,0 +1,161 @@
+---
+name: llm-alignment-training
+description: Implement post-training alignment (SFT, DPO, RLHF) for language models — from base model to conversational assistant. Covers masked instruction loss, preference optimization, reward modeling, and integration with custom training loops.
+tags: [llm, alignment, sft, dpo, rlhf, fine-tuning, post-training]
+---
+
+# LLM Alignment Training
+
+Implement SFT (Supervised Fine-Tuning), DPO (Direct Preference Optimization), and RLHF (Reinforcement Learning from Human Feedback) for language models. This skill covers the full pipeline from base pre-trained model to conversational, aligned assistant.
+
+## When to Use
+
+- User wants to add conversational ability to a base language model
+- User wants to implement instruction-following training
+- User wants to add preference-based alignment (DPO/RLHF)
+- User has a custom model implementation and needs alignment training code
+- User wants to understand masked loss strategies for SFT
+
+## Core Concepts
+
+### Training Stages
+
+```
+Pretrained Model → SFT → DPO/RLHF → Aligned Model
+     (next-token)    (instruction)  (preference)
+```
+
+| Stage | Data | Loss Function | Purpose |
+|:------|:-----|:--------------|:--------|
+| SFT | (instruction, response) pairs | Masked cross-entropy | Learn to follow instructions |
+| DPO | (instruction, chosen, rejected) triples | DPO loss | Learn to prefer better responses |
+| RLHF | (instruction, chosen, rejected) + reward model | PPO/REINFORCE | Optimize for human preference |
+
+### SFT: Masked Instruction Loss
+
+The key insight: only compute loss on the **response** tokens, not the instruction tokens. This prevents the model from "re-learning" the question itself.
+
+```
+Input:  [bos] Question [ANS] Answer tokens...[eos]
+Mask:   [0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1]
+                                       ↑ loss starts here
+```
+
+Implementation:
+1. Tokenize `instruction + response` as a single sequence
+2. Build loss mask: 0 for instruction positions, 1 for response positions
+3. Compute cross-entropy only where mask = 1
+4. Normalize by number of response tokens
+
+### DPO: Direct Preference Optimization
+
+DPO eliminates the need for a separate reward model by using the policy model itself as the reward signal relative to a frozen reference model.
+
+```
+L_DPO = -log(σ(β × (log π(y_w)/π_ref(y_w) - log π(y_l)/π_ref(y_l))))
+```
+
+Where:
+- `π` = policy model (being optimized)
+- `π_ref` = reference model (frozen copy of SFT model)
+- `y_w` = chosen (preferred) response
+- `y_l` = rejected response
+- `β` = temperature parameter (typically 0.1-0.5)
+
+Key implementation details:
+- Reference model is a frozen copy — never updated
+- Both models share the same architecture
+- Log probabilities are computed per-token and summed over the response
+- The implicit reward is `r(y) = β × (log π(y) - log π_ref(y))`
+
+### RLHF: Full Pipeline (when DPO is insufficient)
+
+1. Train reward model on preference data
+2. Optimize policy against reward model using PPO
+3. KL penalty to prevent policy from diverging too far from reference
+
+## Implementation Patterns
+
+### For Custom C++ Implementations
+
+When implementing alignment for a custom model (not PyTorch/HuggingFace):
+
+1. **Reuse existing infrastructure**:
+   - Model forward pass
+   - Optimizer (AdamW)
+   - Learning rate scheduler
+   - Model serialization format
+
+2. **Add alignment-specific components**:
+   - Loss mask computation for SFT
+   - Reference model copy for DPO
+   - Log probability computation for DPO
+
+3. **Data format**:
+   - JSONL is the standard: `{"instruction": "...", "response": "..."}`
+   - For DPO: `{"instruction": "...", "chosen": "...", "rejected": "..."}`
+   - Alternative: plain text with delimiter (e.g., `|||`)
+
+4. **Checkpoint format**:
+   - Reuse existing model serialization
+   - Save both policy and reference models for DPO
+
+### Training Hyperparameters
+
+| Parameter | SFT | DPO | RLHF |
+|:----------|:----|:----|:-----|
+| Learning rate | 1e-5 to 5e-5 | 1e-6 to 5e-6 | 1e-6 to 1e-5 |
+| Epochs | 1-5 | 1-5 | 1-3 |
+| Batch size | 4-32 | 4-32 | 4-16 |
+| Max seq length | 512-4096 | 512-4096 | 512-2048 |
+| Warmup ratio | 0.03-0.1 | 0.03-0.1 | 0.03-0.1 |
+| β (DPO temp) | — | 0.1-0.5 | — |
+| KL penalty | — | — | 0.01-0.1 |
+
+## Common Pitfalls
+
+### SFT Pitfalls
+
+1. **Not masking instruction loss**: Model learns to "repeat" the question rather than answer it
+2. **Too high learning rate**: Destroys pre-trained knowledge (catastrophic forgetting)
+3. **Insufficient data diversity**: Model overfits to specific response patterns
+4. **Wrong tokenization**: Instruction and response must use the same tokenizer as pre-training
+
+### DPO Pitfalls
+
+1. **Reference model not frozen**: Both models update, destroying the preference signal
+2. **β too large**: Policy diverges too far from reference, quality degrades
+3. **β too small**: No meaningful preference learning occurs
+4. **Poor quality preference data**: Chosen/rejected pairs must have clear quality difference
+5. **Not using SFT model as starting point**: DPO works best when initialized from SFT, not base model
+
+### General Pitfalls
+
+1. **Forgetting to call `model.train()` / `model.eval()`**: Batch norm and dropout behave differently
+2. **Not saving checkpoints**: Alignment training can be unstable; save frequently
+3. **No evaluation**: Always hold out eval data to monitor overfitting
+4. **Ignoring sequence length**: Long sequences need more memory; consider gradient accumulation
+
+## Verification Checklist
+
+After implementing alignment training:
+
+- [ ] SFT loss decreases over training
+- [ ] DPO loss decreases (or accuracy of preference prediction increases)
+- [ ] Model generates coherent responses to unseen instructions
+- [ ] Model prefers chosen over rejected responses (DPO)
+- [ ] Reference model remains unchanged (DPO)
+- [ ] Checkpoints can be loaded and used for inference
+- [ ] No NaN/Inf in loss or gradients
+- [ ] Eval loss tracks training loss (no severe overfitting)
+
+## Related Skills
+
+- `huggingface-hub` — for using HuggingFace models/datasets
+- `evaluating-llms-harness` — for benchmarking aligned models
+- `serving-llms-vllm` — for serving aligned models
+- `dspy` — for declarative LM program optimization
+
+## References
+
+- See `references/neuroflow-sft-dpo-implementation.md` for a concrete example of implementing SFT+DPO for a custom C++ LLM (NeuroFlow model with SN/ECN/DMN architecture)

bindings/python_bindings.cpp

@@ -0,0 +1,406 @@
+/**
+ * NeuroFlow Python Bindings
+ * 
+ * 使用pybind11绑定C++核心到Python
+ * 保持与原Python API兼容
+ */
+
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include <pybind11/numpy.h>
+#include <pybind11/operators.h>
+
+#include "neuroflow/model.hpp"
+#include "neuroflow/tensor.hpp"
+#include "neuroflow/networks.hpp"
+#include "neuroflow/memory.hpp"
+#include "neuroflow/multimodal_model.hpp"
+
+namespace py = pybind11;
+using namespace neuroflow;
+
+// numpy数组转换为Tensor
+Tensor numpy_to_tensor(py::array_t<float> arr) {
+    py::buffer_info buf = arr.request();
+    
+    std::vector<size_t> shape;
+    for (auto dim : buf.shape) shape.push_back(static_cast<size_t>(dim));
+    
+    Tensor t(shape, QuantType::FP32);
+    float* data = t.as_fp32();  // 新创建的Tensor，可以用非const指针
+    float* src = static_cast<float*>(buf.ptr);
+    
+    memcpy(data, src, t.data_size);
+    
+    return t;
+}
+
+// Tensor转换为numpy数组
+py::array_t<float> tensor_to_numpy(const Tensor& t) {
+    if (t.dtype != QuantType::FP32) {
+        throw std::runtime_error("Only FP32 tensors can be converted to numpy");
+    }
+    
+    std::vector<ssize_t> shape;
+    for (auto dim : t.shape) shape.push_back(static_cast<ssize_t>(dim));
+    
+    const float* data = t.as_fp32();
+    
+    // 创建numpy数组并拷贝数据
+    py::array_t<float> arr(shape);
+    py::buffer_info buf = arr.request();
+    memcpy(buf.ptr, data, t.data_size);
+    
+    return arr;
+}
+
+PYBIND11_MODULE(_core, m) {
+    m.doc() = "NeuroFlow C++ Core - Lightweight Brain-Inspired Neural Network";
+    
+    // 版本
+    m.attr("__version__") = "1.0.0";
+    
+    // QuantType枚举
+    py::enum_<QuantType>(m, "QuantType")
+        .value("FP32", QuantType::FP32)
+        .value("FP16", QuantType::FP16)
+        .value("INT8", QuantType::INT8)
+        .value("INT4", QuantType::INT4)
+        .value("FP8_E4M3", QuantType::FP8_E4M3)
+        .value("FP8_E5M2", QuantType::FP8_E5M2)
+        .export_values();
+    
+    // MemoryLayout枚举
+    py::enum_<MemoryLayout>(m, "MemoryLayout")
+        .value("ROW_MAJOR", MemoryLayout::ROW_MAJOR)
+        .value("COL_MAJOR", MemoryLayout::COL_MAJOR)
+        .export_values();
+    
+    // Tensor类
+    py::class_<Tensor>(m, "Tensor")
+        .def(py::init<>())
+        .def(py::init<std::vector<size_t>, QuantType>(), 
+             py::arg("shape"), py::arg("dtype") = QuantType::FP32)
+        .def_property_readonly("shape", [](const Tensor& t) { return t.shape; })
+        .def_property_readonly("dtype", [](const Tensor& t) { return t.dtype; })
+        .def_property_readonly("numel", &Tensor::numel)
+        .def_property_readonly("data_size", [](const Tensor& t) { return t.data_size; })
+        .def("clone", &Tensor::clone)
+        .def("reshape", &Tensor::reshape)
+        .def("as_numpy", &tensor_to_numpy)
+        .def("from_numpy", [](Tensor& t, py::array_t<float> arr) {
+            py::buffer_info buf = arr.request();
+            float* data = t.as_fp32();
+            memcpy(data, buf.ptr, t.data_size);
+        })
+        .def_static("from_numpy_array", &numpy_to_tensor);
+    
+    // TensorOps
+    py::class_<TensorOps>(m, "TensorOps")
+        .def_static("gemm", &TensorOps::gemm,
+            py::arg("A"), py::arg("B"), py::arg("C"),
+            py::arg("transA") = false, py::arg("transB") = false,
+            py::arg("alpha") = 1.0f, py::arg("beta") = 0.0f)
+        .def_static("layer_norm", &TensorOps::layer_norm,
+            py::arg("x"), py::arg("weight"), py::arg("bias"),
+            py::arg("eps") = 1e-5f)
+        .def_static("gelu", &TensorOps::gelu)
+        .def_static("softmax", &TensorOps::softmax,
+            py::arg("x"), py::arg("axis") = -1)
+        .def_static("dropout", &TensorOps::dropout,
+            py::arg("x"), py::arg("rate"), py::arg("training") = true)
+        .def_static("add", &TensorOps::add)
+        .def_static("mul", &TensorOps::mul)
+        .def_static("concat", &TensorOps::concat,
+            py::arg("tensors"), py::arg("axis") = 0)
+        .def_static("quantize_int8", &TensorOps::quantize_int8)
+        .def_static("dequantize_int8", &TensorOps::dequantize_int8);
+    
+    // Linear层
+    py::class_<Linear>(m, "Linear")
+        .def(py::init<size_t, size_t, bool, bool>(),
+            py::arg("in_features"), py::arg("out_features"),
+            py::arg("use_bias") = true, py::arg("quant") = false)
+        .def("forward", &Linear::forward)
+        .def("quantize", &Linear::quantize)
+        .def_property_readonly("weight", [](Linear& l) { return l.weight; })
+        .def_property_readonly("bias", [](Linear& l) { return l.bias; })
+        .def_property_readonly("quantized", [](Linear& l) { return l.quantized; });
+    
+    // LayerNorm
+    py::class_<LayerNorm>(m, "LayerNorm")
+        .def(py::init<size_t, float>(), py::arg("dim"), py::arg("eps") = 1e-5f)
+        .def("forward", &LayerNorm::forward)
+        .def_property_readonly("weight", [](LayerNorm& l) { return l.weight; })
+        .def_property_readonly("bias", [](LayerNorm& l) { return l.bias; });
+    
+    // ExecutiveControlNetwork
+    py::class_<ExecutiveControlNetwork::Output>(m, "ECNOutput")
+        .def_property_readonly("decision", [](ExecutiveControlNetwork::Output& o) { return o.decision; })
+        .def_property_readonly("value", [](ExecutiveControlNetwork::Output& o) { return o.value; })
+        .def_property_readonly("hidden_states", [](ExecutiveControlNetwork::Output& o) { return o.hidden_states; });
+    
+    py::class_<ExecutiveControlNetwork>(m, "ExecutiveControlNetwork")
+        .def(py::init<size_t, size_t, size_t, size_t>(),
+            py::arg("input_dim"), py::arg("hidden_dim"), py::arg("output_dim"),
+            py::arg("num_layers") = 2)
+        .def("forward", &ExecutiveControlNetwork::forward)
+        .def("set_training", &ExecutiveControlNetwork::set_training)
+        .def("quantize", &ExecutiveControlNetwork::quantize);
+    
+    // DefaultModeNetwork
+    py::class_<DefaultModeNetwork::Output>(m, "DMNOutput")
+        .def_property_readonly("vision", [](DefaultModeNetwork::Output& o) { return o.vision; })
+        .def_property_readonly("associations", [](DefaultModeNetwork::Output& o) { return o.associations; })
+        .def_property_readonly("latent", [](DefaultModeNetwork::Output& o) { return o.latent; });
+    
+    py::class_<DefaultModeNetwork>(m, "DefaultModeNetwork")
+        .def(py::init<size_t, size_t, size_t>(),
+            py::arg("memory_dim"), py::arg("latent_dim"), py::arg("num_associations") = 8)
+        .def("forward", &DefaultModeNetwork::forward)
+        .def("quantize", &DefaultModeNetwork::quantize);
+    
+    // SalienceNetwork
+    py::class_<SalienceNetwork::Output>(m, "SNOutput")
+        .def_property_readonly("saliency", [](SalienceNetwork::Output& o) { return o.saliency; })
+        .def_property_readonly("gates", [](SalienceNetwork::Output& o) { return o.gates; })
+        .def_property_readonly("anomaly", [](SalienceNetwork::Output& o) { return o.anomaly; });
+    
+    py::class_<SalienceNetwork>(m, "SalienceNetwork")
+        .def(py::init<size_t, size_t>(), py::arg("input_dim"), py::arg("hidden_dim"))
+        .def("forward", [](SalienceNetwork& sn, const Tensor& x) { return sn.forward(x); },
+            py::arg("x"))
+        .def("forward_with_baseline", [](SalienceNetwork& sn, const Tensor& x, const Tensor& baseline) {
+            return sn.forward(x, &baseline);
+        }, py::arg("x"), py::arg("baseline"))
+        .def("quantize", &SalienceNetwork::quantize);
+    
+    // MemoryConsolidationModule
+    py::class_<MemoryConsolidationModule::RetrievalResult>(m, "MemoryResult")
+        .def_property_readonly("retrieved", [](MemoryConsolidationModule::RetrievalResult& r) { return r.retrieved; })
+        .def_property_readonly("attention", [](MemoryConsolidationModule::RetrievalResult& r) { return r.attention; });
+    
+    py::class_<MemoryConsolidationModule>(m, "MemoryConsolidationModule")
+        .def(py::init<size_t, size_t, size_t, float>(),
+            py::arg("input_dim"), py::arg("memory_slots") = 64,
+            py::arg("memory_dim") = 128, py::arg("ltp_rate") = 0.01f)
+        .def("encode", &MemoryConsolidationModule::encode)
+        .def("retrieve", &MemoryConsolidationModule::retrieve)
+        .def("consolidate", &MemoryConsolidationModule::consolidate)
+        .def("forward", &MemoryConsolidationModule::forward);
+    
+    // LatentKVCache (MLA)
+    py::class_<LatentKVCache>(m, "LatentKVCache")
+        .def(py::init<size_t, size_t, size_t, size_t>(),
+            py::arg("model_dim"), py::arg("heads"), py::arg("latent_dim"),
+            py::arg("max_len") = 4096)
+        .def("forward", &LatentKVCache::forward,
+            py::arg("x"), py::arg("use_cache") = true)
+        .def("clear_cache", &LatentKVCache::clear_cache)
+        .def("cache_size_bytes", &LatentKVCache::cache_size_bytes)
+        .def("memory_saving_ratio", &LatentKVCache::memory_saving_ratio);
+    
+    // NeuroFlowModel::Config
+    py::class_<NeuroFlowModel::Config>(m, "ModelConfig")
+        .def(py::init<>())
+        .def_readwrite("input_dim", &NeuroFlowModel::Config::input_dim)
+        .def_readwrite("hidden_dim", &NeuroFlowModel::Config::hidden_dim)
+        .def_readwrite("output_dim", &NeuroFlowModel::Config::output_dim)
+        .def_readwrite("memory_dim", &NeuroFlowModel::Config::memory_dim)
+        .def_readwrite("memory_slots", &NeuroFlowModel::Config::memory_slots)
+        .def_readwrite("num_layers", &NeuroFlowModel::Config::num_layers)
+        .def_readwrite("num_associations", &NeuroFlowModel::Config::num_associations)
+        .def_readwrite("use_quantization", &NeuroFlowModel::Config::use_quantization)
+        .def_readwrite("use_mla", &NeuroFlowModel::Config::use_mla)
+        .def_readwrite("mla_latent_dim", &NeuroFlowModel::Config::mla_latent_dim);
+    
+    // NeuroFlowModel::Output
+    py::class_<NeuroFlowModel::Output>(m, "ModelOutput")
+        .def_property_readonly("output", [](NeuroFlowModel::Output& o) { return tensor_to_numpy(o.output); })
+        .def_property_readonly("decision", [](NeuroFlowModel::Output& o) { return tensor_to_numpy(o.decision); })
+        .def_property_readonly("value", [](NeuroFlowModel::Output& o) { return tensor_to_numpy(o.value); })
+        .def_property_readonly("saliency", [](NeuroFlowModel::Output& o) { return tensor_to_numpy(o.saliency); })
+        .def_property_readonly("ecn_gate", [](NeuroFlowModel::Output& o) { return tensor_to_numpy(o.ecn_gate); })
+        .def_property_readonly("dmn_gate", [](NeuroFlowModel::Output& o) { return tensor_to_numpy(o.dmn_gate); })
+        .def_property_readonly("anomaly", [](NeuroFlowModel::Output& o) { return tensor_to_numpy(o.anomaly); })
+        .def_property_readonly("mem_attention", [](NeuroFlowModel::Output& o) { return tensor_to_numpy(o.mem_attention); })
+        .def_property_readonly("retrieved_mem", [](NeuroFlowModel::Output& o) { return tensor_to_numpy(o.retrieved_mem); })
+        .def_property_readonly("manifold", [](NeuroFlowModel::Output& o) { 
+            if (o.manifold.data) return tensor_to_numpy(o.manifold);
+            return py::array_t<float>();
+        });
+    
+    // NeuroFlowModel::Stats
+    py::class_<NeuroFlowModel::Stats>(m, "ModelStats")
+        .def_readonly("total_params", &NeuroFlowModel::Stats::total_params)
+        .def_readonly("memory_bytes", &NeuroFlowModel::Stats::memory_bytes)
+        .def_readonly("quantization_ratio", &NeuroFlowModel::Stats::quantization_ratio);
+    
+    // NeuroFlowModel主类
+    py::class_<NeuroFlowModel>(m, "NeuroFlowModel")
+        .def(py::init<>())
+        .def(py::init<NeuroFlowModel::Config>(), py::arg("config"))
+        .def("forward", [](NeuroFlowModel& m, py::array_t<float> x, 
+                          py::object memory_input, bool consolidate, bool return_manifold) {
+            Tensor input = numpy_to_tensor(x);
+            const Tensor* mem_ptr = nullptr;
+            Tensor mem;
+            
+            if (!memory_input.is_none()) {
+                mem = numpy_to_tensor(memory_input.cast<py::array_t<float>>());
+                mem_ptr = &mem;
+            }
+            
+            return m.forward(input, mem_ptr, consolidate, return_manifold);
+        }, py::arg("x"), py::arg("memory_input") = py::none(),
+           py::arg("consolidate") = false, py::arg("return_manifold") = false)
+        .def("get_manifold_trajectory", [](NeuroFlowModel& m, py::array_t<float> x, size_t steps) {
+            Tensor input = numpy_to_tensor(x);
+            auto trajectory = m.get_manifold_trajectory(input, steps);
+            
+            py::list result;
+            for (auto& t : trajectory) {
+                result.append(tensor_to_numpy(t));
+            }
+            return result;
+        }, py::arg("x"), py::arg("steps") = 10)
+        .def("set_training", &NeuroFlowModel::set_training)
+        .def("quantize", &NeuroFlowModel::quantize)
+        .def("get_stats", &NeuroFlowModel::get_stats)
+        .def("save", &NeuroFlowModel::save)
+        .def("load", &NeuroFlowModel::load)
+        .def("forward_text", [](NeuroFlowModel& m, py::array_t<float> x) {
+            Tensor input = numpy_to_tensor(x);
+            return m.forward(input);
+        })
+        .def_property_readonly("config", [](NeuroFlowModel& m) { return m.config; });
+    
+    // NeuroFlowLite
+    py::class_<NeuroFlowLite, NeuroFlowModel>(m, "NeuroFlowLite")
+        .def(py::init<size_t>(), py::arg("input_dim") = 512);
+    
+    // 便捷函数
+    m.def("create_tensor", [](py::array_t<float> arr) {
+        return numpy_to_tensor(arr);
+    }, "Create Tensor from numpy array");
+    
+    m.def("benchmark", []() {
+        NeuroFlowModel::Config cfg;
+        cfg.input_dim = 512;
+        cfg.hidden_dim = 256;
+        cfg.output_dim = 10;
+        
+        NeuroFlowModel original(cfg);
+        
+        NeuroFlowModel::Config lite_cfg;
+        lite_cfg.input_dim = 512;
+        lite_cfg.hidden_dim = 128;
+        lite_cfg.output_dim = 10;
+        lite_cfg.memory_dim = 64;
+        lite_cfg.memory_slots = 32;
+        lite_cfg.num_layers = 1;
+        lite_cfg.use_quantization = true;
+        
+        NeuroFlowModel lite(lite_cfg);
+        
+        auto orig_stats = original.get_stats();
+        auto lite_stats = lite.get_stats();
+        
+        py::dict result;
+        result["original_params"] = orig_stats.total_params;
+        result["original_memory_mb"] = orig_stats.memory_bytes / 1024.0 / 1024.0;
+        result["lite_params"] = lite_stats.total_params;
+        result["lite_memory_mb"] = lite_stats.memory_bytes / 1024.0 / 1024.0;
+        result["size_reduction"] = 1.0 - static_cast<float>(lite_stats.total_params) / orig_stats.total_params;
+        
+        return result;
+    }, "Benchmark comparison between original and lite models");
+
+    // ================================================================
+    // MultiModal Bindings
+    // ================================================================
+    
+    // VisionEncoder
+    py::class_<VisionEncoder>(m, "VisionEncoder")
+        .def(py::init<size_t, size_t, size_t, size_t, size_t>(),
+            py::arg("image_size") = 224, py::arg("patch_size") = 16,
+            py::arg("embed_dim") = 256, py::arg("num_heads") = 8,
+            py::arg("num_layers") = 4)
+        .def("forward", &VisionEncoder::forward);
+    
+    // CrossModalFusion
+    py::class_<CrossModalFusion::Output>(m, "FusionOutput")
+        .def_readonly("fused", &CrossModalFusion::Output::fused)
+        .def_readonly("text_feat", &CrossModalFusion::Output::text_feat)
+        .def_readonly("image_feat", &CrossModalFusion::Output::image_feat)
+        .def_readonly("similarity", &CrossModalFusion::Output::similarity);
+    
+    py::class_<CrossModalFusion>(m, "CrossModalFusion")
+        .def(py::init<size_t, size_t, size_t>(),
+            py::arg("text_dim") = 512, py::arg("vision_dim") = 256,
+            py::arg("fusion_dim") = 256)
+        .def("forward", &CrossModalFusion::forward);
+    
+    // NeuroFlowMultiModal::Config
+    py::class_<NeuroFlowMultiModal::Config>(m, "MultiModalConfig")
+        .def(py::init<>())
+        .def_readwrite("text_dim", &NeuroFlowMultiModal::Config::text_dim)
+        .def_readwrite("image_size", &NeuroFlowMultiModal::Config::image_size)
+        .def_readwrite("patch_size", &NeuroFlowMultiModal::Config::patch_size)
+        .def_readwrite("vision_dim", &NeuroFlowMultiModal::Config::vision_dim)
+        .def_readwrite("fusion_dim", &NeuroFlowMultiModal::Config::fusion_dim)
+        .def_readwrite("hidden_dim", &NeuroFlowMultiModal::Config::hidden_dim)
+        .def_readwrite("output_dim", &NeuroFlowMultiModal::Config::output_dim)
+        .def_readwrite("memory_dim", &NeuroFlowMultiModal::Config::memory_dim)
+        .def_readwrite("memory_slots", &NeuroFlowMultiModal::Config::memory_slots)
+        .def_readwrite("num_layers", &NeuroFlowMultiModal::Config::num_layers)
+        .def_readwrite("num_associations", &NeuroFlowMultiModal::Config::num_associations)
+        .def_readwrite("vision_layers", &NeuroFlowMultiModal::Config::vision_layers)
+        .def_readwrite("vision_heads", &NeuroFlowMultiModal::Config::vision_heads)
+        .def_readwrite("use_quantization", &NeuroFlowMultiModal::Config::use_quantization)
+        .def_readwrite("use_mla", &NeuroFlowMultiModal::Config::use_mla)
+        .def_readwrite("mla_latent_dim", &NeuroFlowMultiModal::Config::mla_latent_dim);
+    
+    // NeuroFlowMultiModal::Output
+    py::class_<NeuroFlowMultiModal::Output>(m, "MultiModalOutput")
+        .def_property_readonly("output", [](NeuroFlowMultiModal::Output& o) { return tensor_to_numpy(o.output); })
+        .def_property_readonly("decision", [](NeuroFlowMultiModal::Output& o) { return tensor_to_numpy(o.decision); })
+        .def_property_readonly("value", [](NeuroFlowMultiModal::Output& o) { return tensor_to_numpy(o.value); })
+        .def_property_readonly("saliency", [](NeuroFlowMultiModal::Output& o) { return tensor_to_numpy(o.saliency); })
+        .def_property_readonly("text_image_sim", [](NeuroFlowMultiModal::Output& o) { return tensor_to_numpy(o.text_image_sim); })
+        .def_property_readonly("gates", [](NeuroFlowMultiModal::Output& o) { return tensor_to_numpy(o.gates); })
+        .def_property_readonly("anomaly", [](NeuroFlowMultiModal::Output& o) { return tensor_to_numpy(o.anomaly); })
+        .def_property_readonly("retrieved_mem", [](NeuroFlowMultiModal::Output& o) { return tensor_to_numpy(o.retrieved_mem); })
+        .def_property_readonly("vision_feat", [](NeuroFlowMultiModal::Output& o) { return tensor_to_numpy(o.vision_feat); })
+        .def_property_readonly("text_feat", [](NeuroFlowMultiModal::Output& o) { return tensor_to_numpy(o.text_feat); })
+        .def_property_readonly("fused_feat", [](NeuroFlowMultiModal::Output& o) { return tensor_to_numpy(o.fused_feat); });
+    
+    // NeuroFlowMultiModal
+    py::class_<NeuroFlowMultiModal>(m, "NeuroFlowMultiModal")
+        .def(py::init<NeuroFlowMultiModal::Config>(), py::arg("config"))
+        .def("forward_multimodal", [](NeuroFlowMultiModal& mm, py::array_t<float> text, py::array_t<float> image, bool consolidate, bool return_manifold) {
+            return mm.forward_multimodal(numpy_to_tensor(text), numpy_to_tensor(image), consolidate, return_manifold);
+        }, py::arg("text"), py::arg("image"), py::arg("consolidate") = false, py::arg("return_manifold") = false)
+        .def("forward_text", [](NeuroFlowMultiModal& mm, py::array_t<float> text) {
+            return mm.forward_text(numpy_to_tensor(text));
+        }, py::arg("text"))
+        .def("forward_image", [](NeuroFlowMultiModal& mm, py::array_t<float> image) {
+            return mm.forward_image_only(numpy_to_tensor(image));
+        }, py::arg("image"))
+        .def("set_training", &NeuroFlowMultiModal::set_training)
+        .def("quantize", &NeuroFlowMultiModal::quantize)
+        .def("get_stats", &NeuroFlowMultiModal::get_stats)
+        .def("save", &NeuroFlowMultiModal::save)
+        .def("load", &NeuroFlowMultiModal::load)
+        .def_readonly("config", &NeuroFlowMultiModal::config);
+    
+    // 便捷：创建默认多模态模型
+    m.def("create_multimodal", [](size_t text_dim, size_t image_size, size_t output_dim, bool quantize) {
+        NeuroFlowMultiModal::Config cfg;
+        cfg.text_dim = text_dim;
+        cfg.image_size = image_size;
+        cfg.output_dim = output_dim;
+        cfg.use_quantization = quantize;
+        return std::make_unique<NeuroFlowMultiModal>(cfg);
+    }, py::arg("text_dim") = 512, py::arg("image_size") = 224, py::arg("output_dim") = 10, py::arg("quantize") = false);
+}

build.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+# NeuroFlow C++ 编译脚本 — 自动暂停/恢复 cron 防止冲突
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+BUILD_DIR="$SCRIPT_DIR/build"
+DEST="/mnt/d/neuroflow-model/neuroflow/_core.cpython-311-x86_64-linux-gnu.so"
+VENV_PYTHON="$HOME/.hermes/hermes-agent/venv/bin/python3"
+PYBIND11_DIR="$($VENV_PYTHON -c 'import pybind11; print(pybind11.get_cmake_dir())')"
+CRON_LEARN="bd007dc2fe14"
+CRON_REPORT="5d4dac90ff0d"
+
+echo "=== NeuroFlow C++ Build ==="
+
+# 1. 暂停 cron 任务
+echo "[1/5] Pausing cron jobs..."
+hermes cron pause "$CRON_LEARN" 2>/dev/null || echo "  ($CRON_LEARN already paused)"
+hermes cron pause "$CRON_REPORT" 2>/dev/null || echo "  ($CRON_REPORT already paused)"
+sleep 2  # 等正在运行的任务结束
+
+# 2. 编译
+echo "[2/5] Building with ninja..."
+mkdir -p "$BUILD_DIR"
+cd "$BUILD_DIR"
+cmake .. -G Ninja \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DNEUROFLOW_BUILD_TESTS=OFF \
+  -DNEUROFLOW_BUILD_PYTHON=ON \
+  -DPython3_EXECUTABLE="$VENV_PYTHON" \
+  -Dpybind11_DIR="$PYBIND11_DIR"
+ninja -j$(nproc)
+
+# 3. 部署 so 到 neuroflow 包目录
+echo "[3/5] Deploying .so to neuroflow package..."
+cp "$BUILD_DIR/_core.cpython-311-x86_64-linux-gnu.so" "$DEST"
+
+# 4. 验证
+echo "[4/5] Verifying..."
+"$VENV_PYTHON" -c "
+import sys; sys.path.insert(0, '/mnt/d/neuroflow-model')
+from neuroflow._core import create_multimodal
+m = create_multimodal()
+import numpy as np
+x = np.random.randn(1, 512).astype(np.float32)
+out = m.forward_text(x)
+print(f'  forward_text OK, output shape: {out.decision.shape}')
+m.save('/tmp/nf_build_test.bin')
+m2 = create_multimodal()
+m2.load('/tmp/nf_build_test.bin')
+print('  save/load roundtrip OK')
+print('  ✅ Verification passed')
+"
+
+# 5. 恢复 cron
+echo "[5/5] Resuming cron jobs..."
+hermes cron resume "$CRON_LEARN" 2>/dev/null || echo "  ($CRON_LEARN already running)"
+hermes cron resume "$CRON_REPORT" 2>/dev/null || echo "  ($CRON_REPORT already running)"
+
+echo "=== Build complete ==="

configs/config.json

@@ -0,0 +1,47 @@
+{
+ "input_dim": 512,
+ "hidden_dim": 2048,
+ "output_dim": 2048,
+ "memory_dim": 512,
+ "memory_slots": 64,
+ "num_layers": 12,
+ "num_associations": 8,
+ "use_quantization": false,
+ "use_mla": false,
+ "mla_latent_dim": 32,
+ "use_causal_lm": true,
+ "vocab_size": 128000,
+ "max_seq_len": 512,
+ "causal_window_size": 64,
+ "sae_k": 64,
+ "ntm_memory_slots": 16,
+ "d_model": 512,
+ "mla_n_heads": 8,
+ "mla_max_cache_len": 4096,
+ "lm_num_attn_layers": 2,
+ "lm_pooling": "mean",
+ "_comment": {
+  "input_dim": "输入维度(d_model)",
+  "hidden_dim": "隐藏层维度(FFN)",
+   "output_dim": "三脑内部输出维度(=hidden_dim, 非vocab_size)",
+   "memory_dim": "记忆维度(d_mem)",
+  "memory_slots": "记忆槽数(MEM_SLOTS)",
+  "num_layers": "网络层数(ECN层)",
+  "num_associations": "DMN关联头数",
+  "use_quantization": "启用量化",
+  "use_mla": "启用MLA",
+  "mla_latent_dim": "MLA潜在维度",
+  "use_causal_lm": "因果LM — next-token预测",
+  "vocab_size": "128K多语言多领域词表(中/英/代码/数字/标点)",
+  "max_seq_len": "最大序列长度",
+  "causal_window_size": "因果窗口大小",
+  "sae_k": "SAE稀疏度",
+  "ntm_memory_slots": "NTM记忆槽数",
+  "d_model": "模型维度(512)",
+  "mla_n_heads": "MLA头数",
+   "mla_max_cache_len": "MLA最大缓存长度",
+   "lm_num_attn_layers": "CausalLM注意力层数(2-4)",
+   "lm_pooling": "池化方式(mean/last)"
+ },
+  "_fix_log": "2026-06-13: 128K BPE tokenizer (train_128k_tokenizer.py)"
+}

configs/config_distill.json

@@ -0,0 +1,23 @@
+{
+ "input_dim": 512,
+ "hidden_dim": 2048,
+ "output_dim": 2048,
+ "memory_dim": 512,
+ "memory_slots": 64,
+ "num_layers": 12,
+ "num_associations": 8,
+ "use_quantization": false,
+ "use_mla": false,
+ "mla_latent_dim": 32,
+ "use_causal_lm": true,
+ "vocab_size": 5000,
+ "max_seq_len": 128,
+ "causal_window_size": 64,
+ "sae_k": 64,
+ "ntm_memory_slots": 16,
+ "d_model": 512,
+ "mla_n_heads": 8,
+ "mla_max_cache_len": 4096,
+ "lm_num_attn_layers": 2,
+ "lm_pooling": "mean"
+}

configs/config_distill_128k.json

@@ -0,0 +1,29 @@
+{
+  "input_dim": 128,
+  "hidden_dim": 512,
+  "output_dim": 512,
+  "memory_dim": 128,
+  "memory_slots": 16,
+  "num_layers": 4,
+  "num_associations": 4,
+  "use_quantization": false,
+  "use_mla": false,
+  "mla_latent_dim": 16,
+  "use_causal_lm": true,
+  "vocab_size": 128000,
+  "max_seq_len": 128,
+  "causal_window_size": 16,
+  "sae_k": 16,
+  "ntm_memory_slots": 4,
+  "d_model": 128,
+  "mla_n_heads": 4,
+  "mla_max_cache_len": 1024,
+  "lm_num_attn_layers": 2,
+  "lm_pooling": "mean",
+  "data_path": "data/distill_train.txt",
+  "batch_size": 1,
+  "learning_rate": 0.0001,
+  "num_epochs": 100,
+  "save_every": 1,
+  "output_dir": "distill_output_128k"
+}

configs/generation_config.json

@@ -0,0 +1,22 @@
+{
+  "max_new_tokens": 50,
+  "temperature": 1.0,
+  "top_k": 40,
+  "top_p": 0.9,
+  "repetition_penalty": 1.0,
+  "punct_penalty": 0.0,
+  "random_seed": 0,
+  "strategy": "top_k",
+  "eos_id": 3,
+  "_comment": {
+    "max_new_tokens": "最大生成token数",
+    "temperature": "采样温度(0.0,2.0]",
+    "top_k": "Top-K采样K值",
+    "top_p": "Top-P采样P值[0.0,1.0]",
+    "repetition_penalty": "重复惩罚[1.0,2.0]",
+    "punct_penalty": "标点惩罚",
+    "random_seed": "随机种子(0=随机)",
+    "strategy": "采样策略(greedy/top_k/top_p/top_k_top_p)",
+    "eos_id": "EOS token ID"
+  }
+}

configs/huggingface/merges.txt

@@ -0,0 +1,8 @@
+#version: 0.2
+out out
+out who
+out get
+so me
+10 00
+0 0
+0 1

configs/huggingface/vocab.json

@@ -0,0 +1,4999 @@
+{
+  "<pad>": 0,
+  "<s>": 1,
+  "</s>": 2,
+  "<unk>": 3,
+  "一": 4,
+  "丁": 5,
+  "丂": 6,
+  "七": 7,
+  "丄": 8,
+  "丅": 9,
+  "丆": 10,
+  "万": 11,
+  "丈": 12,
+  "三": 13,
+  "上": 14,
+  "下": 15,
+  "丌": 16,
+  "不": 17,
+  "与": 18,
+  "丏": 19,
+  "丐": 20,
+  "丑": 21,
+  "丒": 22,
+  "专": 23,
+  "且": 24,
+  "丕": 25,
+  "世": 26,
+  "丗": 27,
+  "丘": 28,
+  "丙": 29,
+  "业": 30,
+  "丛": 31,
+  "东": 32,
+  "丝": 33,
+  "丞": 34,
+  "丟": 35,
+  "丠": 36,
+  "両": 37,
+  "丢": 38,
+  "丣": 39,
+  "两": 40,
+  "严": 41,
+  "並": 42,
+  "丧": 43,
+  "丨": 44,
+  "丩": 45,
+  "个": 46,
+  "丫": 47,
+  "丬": 48,
+  "中": 49,
+  "丮": 50,
+  "丯": 51,
+  "丰": 52,
+  "丱": 53,
+  "串": 54,
+  "丳": 55,
+  "临": 56,
+  "丵": 57,
+  "丶": 58,
+  "丷": 59,
+  "丸": 60,
+  "丹": 61,
+  "为": 62,
+  "主": 63,
+  "丼": 64,
+  "丽": 65,
+  "举": 66,
+  "丿": 67,
+  "乀": 68,
+  "乁": 69,
+  "乂": 70,
+  "乃": 71,
+  "乄": 72,
+  "久": 73,
+  "乆": 74,
+  "乇": 75,
+  "么": 76,
+  "义": 77,
+  "乊": 78,
+  "之": 79,
+  "乌": 80,
+  "乍": 81,
+  "乎": 82,
+  "乏": 83,
+  "乐": 84,
+  "乑": 85,
+  "乒": 86,
+  "乓": 87,
+  "乔": 88,
+  "乕": 89,
+  "乖": 90,
+  "乗": 91,
+  "乘": 92,
+  "乙": 93,
+  "乚": 94,
+  "乛": 95,
+  "乜": 96,
+  "九": 97,
+  "乞": 98,
+  "也": 99,
+  "习": 100,
+  "乡": 101,
+  "乢": 102,
+  "乣": 103,
+  "乤": 104,
+  "乥": 105,
+  "书": 106,
+  "乧": 107,
+  "乨": 108,
+  "乩": 109,
+  "乪": 110,
+  "乫": 111,
+  "乬": 112,
+  "乭": 113,
+  "乮": 114,
+  "乯": 115,
+  "买": 116,
+  "乱": 117,
+  "乲": 118,
+  "乳": 119,
+  "乴": 120,
+  "乵": 121,
+  "乶": 122,
+  "乷": 123,
+  "乸": 124,
+  "乹": 125,
+  "乺": 126,
+  "乻": 127,
+  "乼": 128,
+  "乽": 129,
+  "乾": 130,
+  "乿": 131,
+  "亀": 132,
+  "亁": 133,
+  "亂": 134,
+  "亃": 135,
+  "亄": 136,
+  "亅": 137,
+  "了": 138,
+  "亇": 139,
+  "予": 140,
+  "争": 141,
+  "亊": 142,
+  "事": 143,
+  "二": 144,
+  "亍": 145,
+  "于": 146,
+  "亏": 147,
+  "亐": 148,
+  "云": 149,
+  "互": 150,
+  "亓": 151,
+  "五": 152,
+  "井": 153,
+  "亖": 154,
+  "亗": 155,
+  "亘": 156,
+  "亙": 157,
+  "亚": 158,
+  "些": 159,
+  "亜": 160,
+  "亝": 161,
+  "亞": 162,
+  "亟": 163,
+  "亠": 164,
+  "亡": 165,
+  "亢": 166,
+  "亣": 167,
+  "交": 168,
+  "亥": 169,
+  "亦": 170,
+  "产": 171,
+  "亨": 172,
+  "亩": 173,
+  "亪": 174,
+  "享": 175,
+  "京": 176,
+  "亭": 177,
+  "亮": 178,
+  "亯": 179,
+  "亰": 180,
+  "亱": 181,
+  "亲": 182,
+  "亳": 183,
+  "亴": 184,
+  "亵": 185,
+  "亶": 186,
+  "亷": 187,
+  "亸": 188,
+  "亹": 189,
+  "人": 190,
+  "亻": 191,
+  "亼": 192,
+  "亽": 193,
+  "亾": 194,
+  "亿": 195,
+  "什": 196,
+  "仁": 197,
+  "仂": 198,
+  "仃": 199,
+  "仄": 200,
+  "仅": 201,
+  "仆": 202,
+  "仇": 203,
+  "仈": 204,
+  "仉": 205,
+  "今": 206,
+  "介": 207,
+  "仌": 208,
+  "仍": 209,
+  "从": 210,
+  "仏": 211,
+  "仐": 212,
+  "仑": 213,
+  "仒": 214,
+  "仓": 215,
+  "仔": 216,
+  "仕": 217,
+  "他": 218,
+  "仗": 219,
+  "付": 220,
+  "仙": 221,
+  "仚": 222,
+  "仛": 223,
+  "仜": 224,
+  "仝": 225,
+  "仞": 226,
+  "仟": 227,
+  "仠": 228,
+  "仡": 229,
+  "仢": 230,
+  "代": 231,
+  "令": 232,
+  "以": 233,
+  "仦": 234,
+  "仧": 235,
+  "仨": 236,
+  "仩": 237,
+  "仪": 238,
+  "仫": 239,
+  "们": 240,
+  "仭": 241,
+  "仮": 242,
+  "仯": 243,
+  "仰": 244,
+  "仱": 245,
+  "仲": 246,
+  "仳": 247,
+  "仴": 248,
+  "仵": 249,
+  "件": 250,
+  "价": 251,
+  "仸": 252,
+  "仹": 253,
+  "仺": 254,
+  "任": 255,
+  "仼": 256,
+  "份": 257,
+  "仾": 258,
+  "仿": 259,
+  "伀": 260,
+  "企": 261,
+  "伂": 262,
+  "伃": 263,
+  "伄": 264,
+  "伅": 265,
+  "伆": 266,
+  "伇": 267,
+  "伈": 268,
+  "伉": 269,
+  "伊": 270,
+  "伋": 271,
+  "伌": 272,
+  "伍": 273,
+  "伎": 274,
+  "伏": 275,
+  "伐": 276,
+  "休": 277,
+  "伒": 278,
+  "伓": 279,
+  "伔": 280,
+  "伕": 281,
+  "伖": 282,
+  "众": 283,
+  "优": 284,
+  "伙": 285,
+  "会": 286,
+  "伛": 287,
+  "伜": 288,
+  "伝": 289,
+  "伞": 290,
+  "伟": 291,
+  "传": 292,
+  "伡": 293,
+  "伢": 294,
+  "伣": 295,
+  "伤": 296,
+  "伥": 297,
+  "伦": 298,
+  "伧": 299,
+  "伨": 300,
+  "伩": 301,
+  "伪": 302,
+  "伫": 303,
+  "伬": 304,
+  "伭": 305,
+  "伮": 306,
+  "伯": 307,
+  "估": 308,
+  "伱": 309,
+  "伲": 310,
+  "伳": 311,
+  "伴": 312,
+  "伵": 313,
+  "伶": 314,
+  "伷": 315,
+  "伸": 316,
+  "伹": 317,
+  "伺": 318,
+  "伻": 319,
+  "似": 320,
+  "伽": 321,
+  "伾": 322,
+  "伿": 323,
+  "佀": 324,
+  "佁": 325,
+  "佂": 326,
+  "佃": 327,
+  "佄": 328,
+  "佅": 329,
+  "但": 330,
+  "佇": 331,
+  "佈": 332,
+  "佉": 333,
+  "佊": 334,
+  "佋": 335,
+  "佌": 336,
+  "位": 337,
+  "低": 338,
+  "住": 339,
+  "佐": 340,
+  "佑": 341,
+  "佒": 342,
+  "体": 343,
+  "佔": 344,
+  "何": 345,
+  "佖": 346,
+  "佗": 347,
+  "佘": 348,
+  "余": 349,
+  "佚": 350,
+  "佛": 351,
+  "作": 352,
+  "佝": 353,
+  "佞": 354,
+  "佟": 355,
+  "你": 356,
+  "佡": 357,
+  "佢": 358,
+  "佣": 359,
+  "佤": 360,
+  "佥": 361,
+  "佦": 362,
+  "佧": 363,
+  "佨": 364,
+  "佩": 365,
+  "佪": 366,
+  "佫": 367,
+  "佬": 368,
+  "佭": 369,
+  "佮": 370,
+  "佯": 371,
+  "佰": 372,
+  "佱": 373,
+  "佲": 374,
+  "佳": 375,
+  "佴": 376,
+  "併": 377,
+  "佶": 378,
+  "佷": 379,
+  "佸": 380,
+  "佹": 381,
+  "佺": 382,
+  "佻": 383,
+  "佼": 384,
+  "佽": 385,
+  "佾": 386,
+  "使": 387,
+  "侀": 388,
+  "侁": 389,
+  "侂": 390,
+  "侃": 391,
+  "侄": 392,
+  "侅": 393,
+  "來": 394,
+  "侇": 395,
+  "侈": 396,
+  "侉": 397,
+  "侊": 398,
+  "例": 399,
+  "侌": 400,
+  "侍": 401,
+  "侎": 402,
+  "侏": 403,
+  "侐": 404,
+  "侑": 405,
+  "侒": 406,
+  "侓": 407,
+  "侔": 408,
+  "侕": 409,
+  "侖": 410,
+  "侗": 411,
+  "侘": 412,
+  "侙": 413,
+  "侚": 414,
+  "供": 415,
+  "侜": 416,
+  "依": 417,
+  "侞": 418,
+  "侟": 419,
+  "侠": 420,
+  "価": 421,
+  "侢": 422,
+  "侣": 423,
+  "侤": 424,
+  "侥": 425,
+  "侦": 426,
+  "侧": 427,
+  "侨": 428,
+  "侩": 429,
+  "侪": 430,
+  "侫": 431,
+  "侬": 432,
+  "侭": 433,
+  "侮": 434,
+  "侯": 435,
+  "侰": 436,
+  "侱": 437,
+  "侲": 438,
+  "侳": 439,
+  "侴": 440,
+  "侵": 441,
+  "侶": 442,
+  "侷": 443,
+  "侸": 444,
+  "侹": 445,
+  "侺": 446,
+  "侻": 447,
+  "侼": 448,
+  "侽": 449,
+  "侾": 450,
+  "便": 451,
+  "俀": 452,
+  "俁": 453,
+  "係": 454,
+  "促": 455,
+  "俄": 456,
+  "俅": 457,
+  "俆": 458,
+  "俇": 459,
+  "俈": 460,
+  "俉": 461,
+  "俊": 462,
+  "俋": 463,
+  "俌": 464,
+  "俍": 465,
+  "俎": 466,
+  "俏": 467,
+  "俐": 468,
+  "俑": 469,
+  "俒": 470,
+  "俓": 471,
+  "俔": 472,
+  "俕": 473,
+  "俖": 474,
+  "俗": 475,
+  "俘": 476,
+  "俙": 477,
+  "俚": 478,
+  "俛": 479,
+  "俜": 480,
+  "保": 481,
+  "俞": 482,
+  "俟": 483,
+  "俠": 484,
+  "信": 485,
+  "俢": 486,
+  "俣": 487,
+  "俤": 488,
+  "俥": 489,
+  "俦": 490,
+  "俧": 491,
+  "俨": 492,
+  "俩": 493,
+  "俪": 494,
+  "俫": 495,
+  "俬": 496,
+  "俭": 497,
+  "修": 498,
+  "俯": 499,
+  "俰": 500,
+  "俱": 501,
+  "俲": 502,
+  "俳": 503,
+  "俴": 504,
+  "俵": 505,
+  "俶": 506,
+  "俷": 507,
+  "俸": 508,
+  "俹": 509,
+  "俺": 510,
+  "俻": 511,
+  "俼": 512,
+  "俽": 513,
+  "俾": 514,
+  "俿": 515,
+  "倀": 516,
+  "倁": 517,
+  "倂": 518,
+  "倃": 519,
+  "倄": 520,
+  "倅": 521,
+  "倆": 522,
+  "倇": 523,
+  "倈": 524,
+  "倉": 525,
+  "倊": 526,
+  "個": 527,
+  "倌": 528,
+  "倍": 529,
+  "倎": 530,
+  "倏": 531,
+  "倐": 532,
+  "們": 533,
+  "倒": 534,
+  "倓": 535,
+  "倔": 536,
+  "倕": 537,
+  "倖": 538,
+  "倗": 539,
+  "倘": 540,
+  "候": 541,
+  "倚": 542,
+  "倛": 543,
+  "倜": 544,
+  "倝": 545,
+  "倞": 546,
+  "借": 547,
+  "倠": 548,
+  "倡": 549,
+  "倢": 550,
+  "倣": 551,
+  "値": 552,
+  "倥": 553,
+  "倦": 554,
+  "倧": 555,
+  "倨": 556,
+  "倩": 557,
+  "倪": 558,
+  "倫": 559,
+  "倬": 560,
+  "倭": 561,
+  "倮": 562,
+  "倯": 563,
+  "倰": 564,
+  "倱": 565,
+  "倲": 566,
+  "倳": 567,
+  "倴": 568,
+  "倵": 569,
+  "倶": 570,
+  "倷": 571,
+  "倸": 572,
+  "倹": 573,
+  "债": 574,
+  "倻": 575,
+  "值": 576,
+  "倽": 577,
+  "倾": 578,
+  "倿": 579,
+  "偀": 580,
+  "偁": 581,
+  "偂": 582,
+  "偃": 583,
+  "偄": 584,
+  "偅": 585,
+  "偆": 586,
+  "假": 587,
+  "偈": 588,
+  "偉": 589,
+  "偊": 590,
+  "偋": 591,
+  "偌": 592,
+  "偍": 593,
+  "偎": 594,
+  "偏": 595,
+  "偐": 596,
+  "偑": 597,
+  "偒": 598,
+  "偓": 599,
+  "偔": 600,
+  "偕": 601,
+  "偖": 602,
+  "偗": 603,
+  "偘": 604,
+  "偙": 605,
+  "做": 606,
+  "偛": 607,
+  "停": 608,
+  "偝": 609,
+  "偞": 610,
+  "偟": 611,
+  "偠": 612,
+  "偡": 613,
+  "偢": 614,
+  "偣": 615,
+  "偤": 616,
+  "健": 617,
+  "偦": 618,
+  "偧": 619,
+  "偨": 620,
+  "偩": 621,
+  "偪": 622,
+  "偫": 623,
+  "偬": 624,
+  "偭": 625,
+  "偮": 626,
+  "偯": 627,
+  "偰": 628,
+  "偱": 629,
+  "偲": 630,
+  "偳": 631,
+  "側": 632,
+  "偵": 633,
+  "偶": 634,
+  "偷": 635,
+  "偸": 636,
+  "偹": 637,
+  "偺": 638,
+  "偻": 639,
+  "偼": 640,
+  "偽": 641,
+  "偾": 642,
+  "偿": 643,
+  "傀": 644,
+  "傁": 645,
+  "傂": 646,
+  "傃": 647,
+  "傄": 648,
+  "傅": 649,
+  "傆": 650,
+  "傇": 651,
+  "傈": 652,
+  "傉": 653,
+  "傊": 654,
+  "傋": 655,
+  "傌": 656,
+  "傍": 657,
+  "傎": 658,
+  "傏": 659,
+  "傐": 660,
+  "傑": 661,
+  "傒": 662,
+  "傓": 663,
+  "傔": 664,
+  "傕": 665,
+  "傖": 666,
+  "傗": 667,
+  "傘": 668,
+  "備": 669,
+  "傚": 670,
+  "傛": 671,
+  "傜": 672,
+  "傝": 673,
+  "傞": 674,
+  "傟": 675,
+  "傠": 676,
+  "傡": 677,
+  "傢": 678,
+  "傣": 679,
+  "傤": 680,
+  "傥": 681,
+  "傦": 682,
+  "傧": 683,
+  "储": 684,
+  "傩": 685,
+  "傪": 686,
+  "傫": 687,
+  "催": 688,
+  "傭": 689,
+  "傮": 690,
+  "傯": 691,
+  "傰": 692,
+  "傱": 693,
+  "傲": 694,
+  "傳": 695,
+  "傴": 696,
+  "債": 697,
+  "傶": 698,
+  "傷": 699,
+  "傸": 700,
+  "傹": 701,
+  "傺": 702,
+  "傻": 703,
+  "傼": 704,
+  "傽": 705,
+  "傾": 706,
+  "傿": 707,
+  "僀": 708,
+  "僁": 709,
+  "僂": 710,
+  "僃": 711,
+  "僄": 712,
+  "僅": 713,
+  "僆": 714,
+  "僇": 715,
+  "僈": 716,
+  "僉": 717,
+  "僊": 718,
+  "僋": 719,
+  "僌": 720,
+  "働": 721,
+  "僎": 722,
+  "像": 723,
+  "僐": 724,
+  "僑": 725,
+  "僒": 726,
+  "僓": 727,
+  "僔": 728,
+  "僕": 729,
+  "僖": 730,
+  "僗": 731,
+  "僘": 732,
+  "僙": 733,
+  "僚": 734,
+  "僛": 735,
+  "僜": 736,
+  "僝": 737,
+  "僞": 738,
+  "僟": 739,
+  "僠": 740,
+  "僡": 741,
+  "僢": 742,
+  "僣": 743,
+  "僤": 744,
+  "僥": 745,
+  "僦": 746,
+  "僧": 747,
+  "僨": 748,
+  "僩": 749,
+  "僪": 750,
+  "僫": 751,
+  "僬": 752,
+  "僭": 753,
+  "僮": 754,
+  "僯": 755,
+  "僰": 756,
+  "僱": 757,
+  "僲": 758,
+  "僳": 759,
+  "僴": 760,
+  "僵": 761,
+  "僶": 762,
+  "僷": 763,
+  "僸": 764,
+  "價": 765,
+  "僺": 766,
+  "僻": 767,
+  "僼": 768,
+  "僽": 769,
+  "僾": 770,
+  "僿": 771,
+  "儀": 772,
+  "儁": 773,
+  "儂": 774,
+  "儃": 775,
+  "億": 776,
+  "儅": 777,
+  "儆": 778,
+  "儇": 779,
+  "儈": 780,
+  "儉": 781,
+  "儊": 782,
+  "儋": 783,
+  "儌": 784,
+  "儍": 785,
+  "儎": 786,
+  "儏": 787,
+  "儐": 788,
+  "儑": 789,
+  "儒": 790,
+  "儓": 791,
+  "儔": 792,
+  "儕": 793,
+  "儖": 794,
+  "儗": 795,
+  "儘": 796,
+  "儙": 797,
+  "儚": 798,
+  "儛": 799,
+  "儜": 800,
+  "儝": 801,
+  "儞": 802,
+  "償": 803,
+  "儠": 804,
+  "儡": 805,
+  "儢": 806,
+  "儣": 807,
+  "儤": 808,
+  "儥": 809,
+  "儦": 810,
+  "儧": 811,
+  "儨": 812,
+  "儩": 813,
+  "優": 814,
+  "儫": 815,
+  "儬": 816,
+  "儭": 817,
+  "儮": 818,
+  "儯": 819,
+  "儰": 820,
+  "儱": 821,
+  "儲": 822,
+  "儳": 823,
+  "儴": 824,
+  "儵": 825,
+  "儶": 826,
+  "儷": 827,
+  "儸": 828,
+  "儹": 829,
+  "儺": 830,
+  "儻": 831,
+  "儼": 832,
+  "儽": 833,
+  "儾": 834,
+  "儿": 835,
+  "兀": 836,
+  "允": 837,
+  "兂": 838,
+  "元": 839,
+  "兄": 840,
+  "充": 841,
+  "兆": 842,
+  "兇": 843,
+  "先": 844,
+  "光": 845,
+  "兊": 846,
+  "克": 847,
+  "兌": 848,
+  "免": 849,
+  "兎": 850,
+  "兏": 851,
+  "児": 852,
+  "兑": 853,
+  "兒": 854,
+  "兓": 855,
+  "兔": 856,
+  "兕": 857,
+  "兖": 858,
+  "兗": 859,
+  "兘": 860,
+  "兙": 861,
+  "党": 862,
+  "兛": 863,
+  "兜": 864,
+  "兝": 865,
+  "兞": 866,
+  "兟": 867,
+  "兠": 868,
+  "兡": 869,
+  "兢": 870,
+  "兣": 871,
+  "兤": 872,
+  "入": 873,
+  "兦": 874,
+  "內": 875,
+  "全": 876,
+  "兩": 877,
+  "兪": 878,
+  "八": 879,
+  "公": 880,
+  "六": 881,
+  "兮": 882,
+  "兯": 883,
+  "兰": 884,
+  "共": 885,
+  "兲": 886,
+  "关": 887,
+  "兴": 888,
+  "兵": 889,
+  "其": 890,
+  "具": 891,
+  "典": 892,
+  "兹": 893,
+  "兺": 894,
+  "养": 895,
+  "兼": 896,
+  "兽": 897,
+  "兾": 898,
+  "兿": 899,
+  "冀": 900,
+  "冁": 901,
+  "冂": 902,
+  "冃": 903,
+  "冄": 904,
+  "内": 905,
+  "円": 906,
+  "冇": 907,
+  "冈": 908,
+  "冉": 909,
+  "冊": 910,
+  "冋": 911,
+  "册": 912,
+  "再": 913,
+  "冎": 914,
+  "冏": 915,
+  "冐": 916,
+  "冑": 917,
+  "冒": 918,
+  "冓": 919,
+  "冔": 920,
+  "冕": 921,
+  "冖": 922,
+  "冗": 923,
+  "冘": 924,
+  "写": 925,
+  "冚": 926,
+  "军": 927,
+  "农": 928,
+  "冝": 929,
+  "冞": 930,
+  "冟": 931,
+  "冠": 932,
+  "冡": 933,
+  "冢": 934,
+  "冣": 935,
+  "冤": 936,
+  "冥": 937,
+  "冦": 938,
+  "冧": 939,
+  "冨": 940,
+  "冩": 941,
+  "冪": 942,
+  "冫": 943,
+  "冬": 944,
+  "冭": 945,
+  "冮": 946,
+  "冯": 947,
+  "冰": 948,
+  "冱": 949,
+  "冲": 950,
+  "决": 951,
+  "冴": 952,
+  "况": 953,
+  "冶": 954,
+  "冷": 955,
+  "冸": 956,
+  "冹": 957,
+  "冺": 958,
+  "冻": 959,
+  "冼": 960,
+  "冽": 961,
+  "冾": 962,
+  "冿": 963,
+  "净": 964,
+  "凁": 965,
+  "凂": 966,
+  "凃": 967,
+  "凄": 968,
+  "凅": 969,
+  "准": 970,
+  "凇": 971,
+  "凈": 972,
+  "凉": 973,
+  "凊": 974,
+  "凋": 975,
+  "凌": 976,
+  "凍": 977,
+  "凎": 978,
+  "减": 979,
+  "凐": 980,
+  "凑": 981,
+  "凒": 982,
+  "凓": 983,
+  "凔": 984,
+  "凕": 985,
+  "凖": 986,
+  "凗": 987,
+  "凘": 988,
+  "凙": 989,
+  "凚": 990,
+  "凛": 991,
+  "凜": 992,
+  "凝": 993,
+  "凞": 994,
+  "凟": 995,
+  "几": 996,
+  "凡": 997,
+  "凢": 998,
+  "凣": 999,
+  "凤": 1000,
+  "凥": 1001,
+  "処": 1002,
+  "凧": 1003,
+  "凨": 1004,
+  "凩": 1005,
+  "凪": 1006,
+  "凫": 1007,
+  "凬": 1008,
+  "凭": 1009,
+  "凮": 1010,
+  "凯": 1011,
+  "凰": 1012,
+  "凱": 1013,
+  "凲": 1014,
+  "凳": 1015,
+  "凴": 1016,
+  "凵": 1017,
+  "凶": 1018,
+  "凷": 1019,
+  "凸": 1020,
+  "凹": 1021,
+  "出": 1022,
+  "击": 1023,
+  "凼": 1024,
+  "函": 1025,
+  "凾": 1026,
+  "凿": 1027,
+  "刀": 1028,
+  "刁": 1029,
+  "刂": 1030,
+  "刃": 1031,
+  "刄": 1032,
+  "刅": 1033,
+  "分": 1034,
+  "切": 1035,
+  "刈": 1036,
+  "刉": 1037,
+  "刊": 1038,
+  "刋": 1039,
+  "刌": 1040,
+  "刍": 1041,
+  "刎": 1042,
+  "刏": 1043,
+  "刐": 1044,
+  "刑": 1045,
+  "划": 1046,
+  "刓": 1047,
+  "刔": 1048,
+  "刕": 1049,
+  "刖": 1050,
+  "列": 1051,
+  "刘": 1052,
+  "则": 1053,
+  "刚": 1054,
+  "创": 1055,
+  "刜": 1056,
+  "初": 1057,
+  "刞": 1058,
+  "刟": 1059,
+  "删": 1060,
+  "刡": 1061,
+  "刢": 1062,
+  "刣": 1063,
+  "判": 1064,
+  "別": 1065,
+  "刦": 1066,
+  "刧": 1067,
+  "刨": 1068,
+  "利": 1069,
+  "刪": 1070,
+  "别": 1071,
+  "刬": 1072,
+  "刭": 1073,
+  "刮": 1074,
+  "刯": 1075,
+  "到": 1076,
+  "刱": 1077,
+  "刲": 1078,
+  "刳": 1079,
+  "刴": 1080,
+  "刵": 1081,
+  "制": 1082,
+  "刷": 1083,
+  "券": 1084,
+  "刹": 1085,
+  "刺": 1086,
+  "刻": 1087,
+  "刼": 1088,
+  "刽": 1089,
+  "刾": 1090,
+  "刿": 1091,
+  "剀": 1092,
+  "剁": 1093,
+  "剂": 1094,
+  "剃": 1095,
+  "剄": 1096,
+  "剅": 1097,
+  "剆": 1098,
+  "則": 1099,
+  "剈": 1100,
+  "剉": 1101,
+  "削": 1102,
+  "剋": 1103,
+  "剌": 1104,
+  "前": 1105,
+  "剎": 1106,
+  "剏": 1107,
+  "剐": 1108,
+  "剑": 1109,
+  "剒": 1110,
+  "剓": 1111,
+  "剔": 1112,
+  "剕": 1113,
+  "剖": 1114,
+  "剗": 1115,
+  "剘": 1116,
+  "剙": 1117,
+  "剚": 1118,
+  "剛": 1119,
+  "剜": 1120,
+  "剝": 1121,
+  "剞": 1122,
+  "剟": 1123,
+  "剠": 1124,
+  "剡": 1125,
+  "剢": 1126,
+  "剣": 1127,
+  "剤": 1128,
+  "剥": 1129,
+  "剦": 1130,
+  "剧": 1131,
+  "剨": 1132,
+  "剩": 1133,
+  "剪": 1134,
+  "剫": 1135,
+  "剬": 1136,
+  "剭": 1137,
+  "剮": 1138,
+  "副": 1139,
+  "剰": 1140,
+  "剱": 1141,
+  "割": 1142,
+  "剳": 1143,
+  "剴": 1144,
+  "創": 1145,
+  "剶": 1146,
+  "剷": 1147,
+  "剸": 1148,
+  "剹": 1149,
+  "剺": 1150,
+  "剻": 1151,
+  "剼": 1152,
+  "剽": 1153,
+  "剾": 1154,
+  "剿": 1155,
+  "劀": 1156,
+  "劁": 1157,
+  "劂": 1158,
+  "劃": 1159,
+  "劄": 1160,
+  "劅": 1161,
+  "劆": 1162,
+  "劇": 1163,
+  "劈": 1164,
+  "劉": 1165,
+  "劊": 1166,
+  "劋": 1167,
+  "劌": 1168,
+  "劍": 1169,
+  "劎": 1170,
+  "劏": 1171,
+  "劐": 1172,
+  "劑": 1173,
+  "劒": 1174,
+  "劓": 1175,
+  "劔": 1176,
+  "劕": 1177,
+  "劖": 1178,
+  "劗": 1179,
+  "劘": 1180,
+  "劙": 1181,
+  "劚": 1182,
+  "力": 1183,
+  "劜": 1184,
+  "劝": 1185,
+  "办": 1186,
+  "功": 1187,
+  "加": 1188,
+  "务": 1189,
+  "劢": 1190,
+  "劣": 1191,
+  "劤": 1192,
+  "劥": 1193,
+  "劦": 1194,
+  "劧": 1195,
+  "动": 1196,
+  "助": 1197,
+  "努": 1198,
+  "劫": 1199,
+  "劬": 1200,
+  "劭": 1201,
+  "劮": 1202,
+  "劯": 1203,
+  "劰": 1204,
+  "励": 1205,
+  "劲": 1206,
+  "劳": 1207,
+  "労": 1208,
+  "劵": 1209,
+  "劶": 1210,
+  "劷": 1211,
+  "劸": 1212,
+  "効": 1213,
+  "劺": 1214,
+  "劻": 1215,
+  "劼": 1216,
+  "劽": 1217,
+  "劾": 1218,
+  "势": 1219,
+  "勀": 1220,
+  "勁": 1221,
+  "勂": 1222,
+  "勃": 1223,
+  "勄": 1224,
+  "勅": 1225,
+  "勆": 1226,
+  "勇": 1227,
+  "勈": 1228,
+  "勉": 1229,
+  "勊": 1230,
+  "勋": 1231,
+  "勌": 1232,
+  "勍": 1233,
+  "勎": 1234,
+  "勏": 1235,
+  "勐": 1236,
+  "勑": 1237,
+  "勒": 1238,
+  "勓": 1239,
+  "勔": 1240,
+  "動": 1241,
+  "勖": 1242,
+  "勗": 1243,
+  "勘": 1244,
+  "務": 1245,
+  "勚": 1246,
+  "勛": 1247,
+  "勜": 1248,
+  "勝": 1249,
+  "勞": 1250,
+  "募": 1251,
+  "勠": 1252,
+  "勡": 1253,
+  "勢": 1254,
+  "勣": 1255,
+  "勤": 1256,
+  "勥": 1257,
+  "勦": 1258,
+  "勧": 1259,
+  "勨": 1260,
+  "勩": 1261,
+  "勪": 1262,
+  "勫": 1263,
+  "勬": 1264,
+  "勭": 1265,
+  "勮": 1266,
+  "勯": 1267,
+  "��": 1268,
+  "勱": 1269,
+  "勲": 1270,
+  "勳": 1271,
+  "勴": 1272,
+  "勵": 1273,
+  "勶": 1274,
+  "勷": 1275,
+  "勸": 1276,
+  "勹": 1277,
+  "勺": 1278,
+  "勻": 1279,
+  "勼": 1280,
+  "勽": 1281,
+  "勾": 1282,
+  "勿": 1283,
+  "匀": 1284,
+  "匁": 1285,
+  "匂": 1286,
+  "匃": 1287,
+  "匄": 1288,
+  "包": 1289,
+  "匆": 1290,
+  "匇": 1291,
+  "匈": 1292,
+  "匉": 1293,
+  "匊": 1294,
+  "匋": 1295,
+  "匌": 1296,
+  "匍": 1297,
+  "匎": 1298,
+  "匏": 1299,
+  "匐": 1300,
+  "匑": 1301,
+  "匒": 1302,
+  "匓": 1303,
+  "匔": 1304,
+  "匕": 1305,
+  "化": 1306,
+  "北": 1307,
+  "匘": 1308,
+  "匙": 1309,
+  "匚": 1310,
+  "匛": 1311,
+  "匜": 1312,
+  "匝": 1313,
+  "匞": 1314,
+  "匟": 1315,
+  "匠": 1316,
+  "匡": 1317,
+  "匢": 1318,
+  "匣": 1319,
+  "匤": 1320,
+  "匥": 1321,
+  "匦": 1322,
+  "匧": 1323,
+  "匨": 1324,
+  "匩": 1325,
+  "匪": 1326,
+  "匫": 1327,
+  "匬": 1328,
+  "匭": 1329,
+  "匮": 1330,
+  "匯": 1331,
+  "匰": 1332,
+  "匱": 1333,
+  "匲": 1334,
+  "匳": 1335,
+  "匴": 1336,
+  "匵": 1337,
+  "匶": 1338,
+  "匷": 1339,
+  "匸": 1340,
+  "匹": 1341,
+  "区": 1342,
+  "医": 1343,
+  "匼": 1344,
+  "匽": 1345,
+  "匾": 1346,
+  "匿": 1347,
+  "區": 1348,
+  "十": 1349,
+  "卂": 1350,
+  "千": 1351,
+  "卄": 1352,
+  "卅": 1353,
+  "卆": 1354,
+  "升": 1355,
+  "午": 1356,
+  "卉": 1357,
+  "半": 1358,
+  "卋": 1359,
+  "卌": 1360,
+  "卍": 1361,
+  "华": 1362,
+  "协": 1363,
+  "卐": 1364,
+  "卑": 1365,
+  "卒": 1366,
+  "卓": 1367,
+  "協": 1368,
+  "单": 1369,
+  "卖": 1370,
+  "南": 1371,
+  "単": 1372,
+  "卙": 1373,
+  "博": 1374,
+  "卛": 1375,
+  "卜": 1376,
+  "卝": 1377,
+  "卞": 1378,
+  "卟": 1379,
+  "占": 1380,
+  "卡": 1381,
+  "卢": 1382,
+  "卣": 1383,
+  "卤": 1384,
+  "卥": 1385,
+  "卦": 1386,
+  "卧": 1387,
+  "卨": 1388,
+  "卩": 1389,
+  "卪": 1390,
+  "卫": 1391,
+  "卬": 1392,
+  "卭": 1393,
+  "卮": 1394,
+  "卯": 1395,
+  "印": 1396,
+  "危": 1397,
+  "卲": 1398,
+  "即": 1399,
+  "却": 1400,
+  "卵": 1401,
+  "卶": 1402,
+  "卷": 1403,
+  "卸": 1404,
+  "卹": 1405,
+  "卺": 1406,
+  "卻": 1407,
+  "卼": 1408,
+  "卽": 1409,
+  "卾": 1410,
+  "卿": 1411,
+  "厀": 1412,
+  "厁": 1413,
+  "厂": 1414,
+  "厃": 1415,
+  "厄": 1416,
+  "厅": 1417,
+  "历": 1418,
+  "厇": 1419,
+  "厈": 1420,
+  "厉": 1421,
+  "厊": 1422,
+  "压": 1423,
+  "厌": 1424,
+  "厍": 1425,
+  "厎": 1426,
+  "厏": 1427,
+  "厐": 1428,
+  "厑": 1429,
+  "厒": 1430,
+  "厓": 1431,
+  "厔": 1432,
+  "厕": 1433,
+  "厖": 1434,
+  "厗": 1435,
+  "厘": 1436,
+  "厙": 1437,
+  "厚": 1438,
+  "厛": 1439,
+  "厜": 1440,
+  "厝": 1441,
+  "厞": 1442,
+  "原": 1443,
+  "厠": 1444,
+  "厡": 1445,
+  "厢": 1446,
+  "厣": 1447,
+  "厤": 1448,
+  "厥": 1449,
+  "厦": 1450,
+  "厧": 1451,
+  "厨": 1452,
+  "厩": 1453,
+  "厪": 1454,
+  "厫": 1455,
+  "厬": 1456,
+  "厭": 1457,
+  "厮": 1458,
+  "厯": 1459,
+  "厰": 1460,
+  "厱": 1461,
+  "厲": 1462,
+  "厳": 1463,
+  "厴": 1464,
+  "厵": 1465,
+  "厶": 1466,
+  "厷": 1467,
+  "厸": 1468,
+  "厹": 1469,
+  "厺": 1470,
+  "去": 1471,
+  "厼": 1472,
+  "厽": 1473,
+  "厾": 1474,
+  "县": 1475,
+  "叀": 1476,
+  "叁": 1477,
+  "参": 1478,
+  "參": 1479,
+  "叄": 1480,
+  "叅": 1481,
+  "叆": 1482,
+  "叇": 1483,
+  "又": 1484,
+  "叉": 1485,
+  "及": 1486,
+  "友": 1487,
+  "双": 1488,
+  "反": 1489,
+  "収": 1490,
+  "叏": 1491,
+  "叐": 1492,
+  "发": 1493,
+  "叒": 1494,
+  "叓": 1495,
+  "叔": 1496,
+  "叕": 1497,
+  "取": 1498,
+  "受": 1499,
+  "变": 1500,
+  "叙": 1501,
+  "叚": 1502,
+  "叛": 1503,
+  "叜": 1504,
+  "叝": 1505,
+  "叞": 1506,
+  "叟": 1507,
+  "叠": 1508,
+  "叡": 1509,
+  "叢": 1510,
+  "口": 1511,
+  "古": 1512,
+  "句": 1513,
+  "另": 1514,
+  "叧": 1515,
+  "叨": 1516,
+  "叩": 1517,
+  "只": 1518,
+  "叫": 1519,
+  "召": 1520,
+  "叭": 1521,
+  "叮": 1522,
+  "可": 1523,
+  "台": 1524,
+  "叱": 1525,
+  "史": 1526,
+  "右": 1527,
+  "叴": 1528,
+  "叵": 1529,
+  "叶": 1530,
+  "号": 1531,
+  "司": 1532,
+  "叹": 1533,
+  "叺": 1534,
+  "叻": 1535,
+  "叼": 1536,
+  "叽": 1537,
+  "叾": 1538,
+  "叿": 1539,
+  "吀": 1540,
+  "吁": 1541,
+  "吂": 1542,
+  "吃": 1543,
+  "各": 1544,
+  "吅": 1545,
+  "吆": 1546,
+  "吇": 1547,
+  "合": 1548,
+  "吉": 1549,
+  "吊": 1550,
+  "吋": 1551,
+  "同": 1552,
+  "名": 1553,
+  "后": 1554,
+  "吏": 1555,
+  "吐": 1556,
+  "向": 1557,
+  "吒": 1558,
+  "吓": 1559,
+  "吔": 1560,
+  "吕": 1561,
+  "吖": 1562,
+  "吗": 1563,
+  "吘": 1564,
+  "吙": 1565,
+  "吚": 1566,
+  "君": 1567,
+  "吜": 1568,
+  "吝": 1569,
+  "吞": 1570,
+  "吟": 1571,
+  "吠": 1572,
+  "吡": 1573,
+  "吢": 1574,
+  "吣": 1575,
+  "吤": 1576,
+  "吥": 1577,
+  "否": 1578,
+  "吧": 1579,
+  "吨": 1580,
+  "吩": 1581,
+  "吪": 1582,
+  "含": 1583,
+  "听": 1584,
+  "吭": 1585,
+  "吮": 1586,
+  "启": 1587,
+  "吰": 1588,
+  "吱": 1589,
+  "吲": 1590,
+  "吳": 1591,
+  "吴": 1592,
+  "吵": 1593,
+  "吶": 1594,
+  "吷": 1595,
+  "吸": 1596,
+  "吹": 1597,
+  "吺": 1598,
+  "吻": 1599,
+  "吼": 1600,
+  "吽": 1601,
+  "吾": 1602,
+  "吿": 1603,
+  "呀": 1604,
+  "呁": 1605,
+  "呂": 1606,
+  "呃": 1607,
+  "呄": 1608,
+  "呅": 1609,
+  "呆": 1610,
+  "呇": 1611,
+  "呈": 1612,
+  "呉": 1613,
+  "告": 1614,
+  "呋": 1615,
+  "呌": 1616,
+  "呍": 1617,
+  "呎": 1618,
+  "呏": 1619,
+  "呐": 1620,
+  "呑": 1621,
+  "呒": 1622,
+  "呓": 1623,
+  "呔": 1624,
+  "呕": 1625,
+  "呖": 1626,
+  "呗": 1627,
+  "员": 1628,
+  "呙": 1629,
+  "呚": 1630,
+  "呛": 1631,
+  "呜": 1632,
+  "呝": 1633,
+  "呞": 1634,
+  "呟": 1635,
+  "呠": 1636,
+  "呡": 1637,
+  "呢": 1638,
+  "呣": 1639,
+  "呤": 1640,
+  "呥": 1641,
+  "呦": 1642,
+  "呧": 1643,
+  "周": 1644,
+  "呩": 1645,
+  "呪": 1646,
+  "呫": 1647,
+  "呬": 1648,
+  "呭": 1649,
+  "呮": 1650,
+  "呯": 1651,
+  "呰": 1652,
+  "呱": 1653,
+  "呲": 1654,
+  "味": 1655,
+  "呴": 1656,
+  "呵": 1657,
+  "呶": 1658,
+  "呷": 1659,
+  "呸": 1660,
+  "呹": 1661,
+  "呺": 1662,
+  "呻": 1663,
+  "呼": 1664,
+  "命": 1665,
+  "呾": 1666,
+  "呿": 1667,
+  "咀": 1668,
+  "咁": 1669,
+  "咂": 1670,
+  "咃": 1671,
+  "咄": 1672,
+  "咅": 1673,
+  "咆": 1674,
+  "咇": 1675,
+  "咈": 1676,
+  "咉": 1677,
+  "咊": 1678,
+  "咋": 1679,
+  "和": 1680,
+  "咍": 1681,
+  "咎": 1682,
+  "咏": 1683,
+  "咐": 1684,
+  "咑": 1685,
+  "咒": 1686,
+  "咓": 1687,
+  "咔": 1688,
+  "咕": 1689,
+  "咖": 1690,
+  "咗": 1691,
+  "咘": 1692,
+  "咙": 1693,
+  "咚": 1694,
+  "咛": 1695,
+  "咜": 1696,
+  "咝": 1697,
+  "咞": 1698,
+  "咟": 1699,
+  "咠": 1700,
+  "咡": 1701,
+  "咢": 1702,
+  "咣": 1703,
+  "咤": 1704,
+  "咥": 1705,
+  "咦": 1706,
+  "咧": 1707,
+  "咨": 1708,
+  "咩": 1709,
+  "咪": 1710,
+  "咫": 1711,
+  "咬": 1712,
+  "咭": 1713,
+  "咮": 1714,
+  "咯": 1715,
+  "咰": 1716,
+  "咱": 1717,
+  "咲": 1718,
+  "咳": 1719,
+  "咴": 1720,
+  "咵": 1721,
+  "咶": 1722,
+  "咷": 1723,
+  "咸": 1724,
+  "咹": 1725,
+  "咺": 1726,
+  "咻": 1727,
+  "咼": 1728,
+  "咽": 1729,
+  "咾": 1730,
+  "咿": 1731,
+  "哀": 1732,
+  "品": 1733,
+  "哂": 1734,
+  "哃": 1735,
+  "哄": 1736,
+  "哅": 1737,
+  "哆": 1738,
+  "哇": 1739,
+  "哈": 1740,
+  "哉": 1741,
+  "哊": 1742,
+  "哋": 1743,
+  "哌": 1744,
+  "响": 1745,
+  "哎": 1746,
+  "哏": 1747,
+  "哐": 1748,
+  "哑": 1749,
+  "哒": 1750,
+  "哓": 1751,
+  "哔": 1752,
+  "哕": 1753,
+  "哖": 1754,
+  "哗": 1755,
+  "哘": 1756,
+  "哙": 1757,
+  "哚": 1758,
+  "哛": 1759,
+  "哜": 1760,
+  "哝": 1761,
+  "哞": 1762,
+  "哟": 1763,
+  "哠": 1764,
+  "員": 1765,
+  "哢": 1766,
+  "哣": 1767,
+  "哤": 1768,
+  "哥": 1769,
+  "哦": 1770,
+  "哧": 1771,
+  "哨": 1772,
+  "哩": 1773,
+  "哪": 1774,
+  "哫": 1775,
+  "哬": 1776,
+  "哭": 1777,
+  "哮": 1778,
+  "哯": 1779,
+  "哰": 1780,
+  "哱": 1781,
+  "哲": 1782,
+  "哳": 1783,
+  "哴": 1784,
+  "哵": 1785,
+  "哶": 1786,
+  "哷": 1787,
+  "哸": 1788,
+  "哹": 1789,
+  "哺": 1790,
+  "哻": 1791,
+  "哼": 1792,
+  "哽": 1793,
+  "哾": 1794,
+  "哿": 1795,
+  "唀": 1796,
+  "唁": 1797,
+  "唂": 1798,
+  "唃": 1799,
+  "唄": 1800,
+  "唅": 1801,
+  "唆": 1802,
+  "唇": 1803,
+  "唈": 1804,
+  "唉": 1805,
+  "唊": 1806,
+  "唋": 1807,
+  "唌": 1808,
+  "唍": 1809,
+  "唎": 1810,
+  "唏": 1811,
+  "唐": 1812,
+  "唑": 1813,
+  "唒": 1814,
+  "唓": 1815,
+  "唔": 1816,
+  "唕": 1817,
+  "唖": 1818,
+  "唗": 1819,
+  "唘": 1820,
+  "唙": 1821,
+  "唚": 1822,
+  "唛": 1823,
+  "唜": 1824,
+  "唝": 1825,
+  "唞": 1826,
+  "唟": 1827,
+  "唠": 1828,
+  "唡": 1829,
+  "唢": 1830,
+  "唣": 1831,
+  "唤": 1832,
+  "唥": 1833,
+  "唦": 1834,
+  "唧": 1835,
+  "唨": 1836,
+  "唩": 1837,
+  "唪": 1838,
+  "唫": 1839,
+  "唬": 1840,
+  "唭": 1841,
+  "售": 1842,
+  "唯": 1843,
+  "唰": 1844,
+  "唱": 1845,
+  "唲": 1846,
+  "唳": 1847,
+  "唴": 1848,
+  "唵": 1849,
+  "唶": 1850,
+  "唷": 1851,
+  "唸": 1852,
+  "唹": 1853,
+  "唺": 1854,
+  "唻": 1855,
+  "唼": 1856,
+  "唽": 1857,
+  "唾": 1858,
+  "唿": 1859,
+  "啀": 1860,
+  "啁": 1861,
+  "啂": 1862,
+  "啃": 1863,
+  "啄": 1864,
+  "啅": 1865,
+  "商": 1866,
+  "啇": 1867,
+  "啈": 1868,
+  "啉": 1869,
+  "啊": 1870,
+  "啋": 1871,
+  "啌": 1872,
+  "啍": 1873,
+  "啎": 1874,
+  "問": 1875,
+  "啐": 1876,
+  "啑": 1877,
+  "啒": 1878,
+  "啓": 1879,
+  "啔": 1880,
+  "啕": 1881,
+  "啖": 1882,
+  "啗": 1883,
+  "啘": 1884,
+  "啙": 1885,
+  "啚": 1886,
+  "啛": 1887,
+  "啜": 1888,
+  "啝": 1889,
+  "啞": 1890,
+  "啟": 1891,
+  "啠": 1892,
+  "啡": 1893,
+  "啢": 1894,
+  "啣": 1895,
+  "啤": 1896,
+  "啥": 1897,
+  "啦": 1898,
+  "啧": 1899,
+  "啨": 1900,
+  "啩": 1901,
+  "啪": 1902,
+  "啫": 1903,
+  "啬": 1904,
+  "啭": 1905,
+  "啮": 1906,
+  "啯": 1907,
+  "啰": 1908,
+  "啱": 1909,
+  "啲": 1910,
+  "啳": 1911,
+  "啴": 1912,
+  "啵": 1913,
+  "啶": 1914,
+  "啷": 1915,
+  "啸": 1916,
+  "啹": 1917,
+  "啺": 1918,
+  "啻": 1919,
+  "啼": 1920,
+  "啽": 1921,
+  "啾": 1922,
+  "啿": 1923,
+  "喀": 1924,
+  "喁": 1925,
+  "喂": 1926,
+  "喃": 1927,
+  "善": 1928,
+  "喅": 1929,
+  "喆": 1930,
+  "喇": 1931,
+  "喈": 1932,
+  "喉": 1933,
+  "喊": 1934,
+  "喋": 1935,
+  "喌": 1936,
+  "喍": 1937,
+  "喎": 1938,
+  "喏": 1939,
+  "喐": 1940,
+  "喑": 1941,
+  "喒": 1942,
+  "喓": 1943,
+  "喔": 1944,
+  "喕": 1945,
+  "喖": 1946,
+  "喗": 1947,
+  "喘": 1948,
+  "喙": 1949,
+  "喚": 1950,
+  "喛": 1951,
+  "喜": 1952,
+  "喝": 1953,
+  "喞": 1954,
+  "喟": 1955,
+  "喠": 1956,
+  "喡": 1957,
+  "喢": 1958,
+  "喣": 1959,
+  "喤": 1960,
+  "喥": 1961,
+  "喦": 1962,
+  "喧": 1963,
+  "喨": 1964,
+  "喩": 1965,
+  "喪": 1966,
+  "喫": 1967,
+  "喬": 1968,
+  "喭": 1969,
+  "單": 1970,
+  "喯": 1971,
+  "喰": 1972,
+  "喱": 1973,
+  "喲": 1974,
+  "喳": 1975,
+  "喴": 1976,
+  "喵": 1977,
+  "営": 1978,
+  "喷": 1979,
+  "喸": 1980,
+  "喹": 1981,
+  "喺": 1982,
+  "喻": 1983,
+  "喼": 1984,
+  "喽": 1985,
+  "喾": 1986,
+  "喿": 1987,
+  "嗀": 1988,
+  "嗁": 1989,
+  "嗂": 1990,
+  "嗃": 1991,
+  "嗄": 1992,
+  "嗅": 1993,
+  "嗆": 1994,
+  "嗇": 1995,
+  "嗈": 1996,
+  "嗉": 1997,
+  "嗊": 1998,
+  "嗋": 1999,
+  "嗌": 2000,
+  "嗍": 2001,
+  "嗎": 2002,
+  "嗏": 2003,
+  "嗐": 2004,
+  "嗑": 2005,
+  "嗒": 2006,
+  "嗓": 2007,
+  "嗔": 2008,
+  "嗕": 2009,
+  "嗖": 2010,
+  "嗗": 2011,
+  "嗘": 2012,
+  "嗙": 2013,
+  "嗚": 2014,
+  "嗛": 2015,
+  "嗜": 2016,
+  "嗝": 2017,
+  "嗞": 2018,
+  "嗟": 2019,
+  "嗠": 2020,
+  "嗡": 2021,
+  "嗢": 2022,
+  "嗣": 2023,
+  "嗤": 2024,
+  "嗥": 2025,
+  "嗦": 2026,
+  "嗧": 2027,
+  "嗨": 2028,
+  "嗩": 2029,
+  "嗪": 2030,
+  "嗫": 2031,
+  "嗬": 2032,
+  "嗭": 2033,
+  "嗮": 2034,
+  "嗯": 2035,
+  "嗰": 2036,
+  "嗱": 2037,
+  "嗲": 2038,
+  "嗳": 2039,
+  "嗴": 2040,
+  "嗵": 2041,
+  "嗶": 2042,
+  "嗷": 2043,
+  "嗸": 2044,
+  "嗹": 2045,
+  "嗺": 2046,
+  "嗻": 2047,
+  "嗼": 2048,
+  "嗽": 2049,
+  "嗾": 2050,
+  "嗿": 2051,
+  "嘀": 2052,
+  "嘁": 2053,
+  "嘂": 2054,
+  "嘃": 2055,
+  "嘄": 2056,
+  "嘅": 2057,
+  "嘆": 2058,
+  "嘇": 2059,
+  "嘈": 2060,
+  "嘉": 2061,
+  "嘊": 2062,
+  "嘋": 2063,
+  "嘌": 2064,
+  "嘍": 2065,
+  "嘎": 2066,
+  "嘏": 2067,
+  "嘐": 2068,
+  "嘑": 2069,
+  "嘒": 2070,
+  "嘓": 2071,
+  "嘔": 2072,
+  "嘕": 2073,
+  "嘖": 2074,
+  "嘗": 2075,
+  "嘘": 2076,
+  "嘙": 2077,
+  "嘚": 2078,
+  "嘛": 2079,
+  "嘜": 2080,
+  "嘝": 2081,
+  "嘞": 2082,
+  "嘟": 2083,
+  "嘠": 2084,
+  "嘡": 2085,
+  "嘢": 2086,
+  "嘣": 2087,
+  "嘤": 2088,
+  "嘥": 2089,
+  "嘦": 2090,
+  "嘧": 2091,
+  "嘨": 2092,
+  "嘩": 2093,
+  "嘪": 2094,
+  "嘫": 2095,
+  "嘬": 2096,
+  "嘭": 2097,
+  "嘮": 2098,
+  "嘯": 2099,
+  "嘰": 2100,
+  "嘱": 2101,
+  "嘲": 2102,
+  "嘳": 2103,
+  "嘴": 2104,
+  "嘵": 2105,
+  "嘶": 2106,
+  "嘷": 2107,
+  "嘸": 2108,
+  "嘹": 2109,
+  "嘺": 2110,
+  "嘻": 2111,
+  "嘼": 2112,
+  "嘽": 2113,
+  "嘾": 2114,
+  "嘿": 2115,
+  "噀": 2116,
+  "噁": 2117,
+  "噂": 2118,
+  "噃": 2119,
+  "噄": 2120,
+  "噅": 2121,
+  "噆": 2122,
+  "噇": 2123,
+  "噈": 2124,
+  "噉": 2125,
+  "噊": 2126,
+  "噋": 2127,
+  "噌": 2128,
+  "噍": 2129,
+  "噎": 2130,
+  "噏": 2131,
+  "噐": 2132,
+  "噑": 2133,
+  "噒": 2134,
+  "噓": 2135,
+  "噔": 2136,
+  "噕": 2137,
+  "噖": 2138,
+  "噗": 2139,
+  "噘": 2140,
+  "噙": 2141,
+  "噚": 2142,
+  "噛": 2143,
+  "噜": 2144,
+  "噝": 2145,
+  "噞": 2146,
+  "噟": 2147,
+  "噠": 2148,
+  "噡": 2149,
+  "噢": 2150,
+  "噣": 2151,
+  "噤": 2152,
+  "噥": 2153,
+  "噦": 2154,
+  "噧": 2155,
+  "器": 2156,
+  "噩": 2157,
+  "噪": 2158,
+  "噫": 2159,
+  "噬": 2160,
+  "噭": 2161,
+  "噮": 2162,
+  "噯": 2163,
+  "噰": 2164,
+  "噱": 2165,
+  "噲": 2166,
+  "噳": 2167,
+  "噴": 2168,
+  "噵": 2169,
+  "噶": 2170,
+  "噷": 2171,
+  "噸": 2172,
+  "噹": 2173,
+  "噺": 2174,
+  "噻": 2175,
+  "噼": 2176,
+  "噽": 2177,
+  "噾": 2178,
+  "噿": 2179,
+  "嚀": 2180,
+  "嚁": 2181,
+  "嚂": 2182,
+  "嚃": 2183,
+  "嚄": 2184,
+  "嚅": 2185,
+  "嚆": 2186,
+  "嚇": 2187,
+  "嚈": 2188,
+  "嚉": 2189,
+  "嚊": 2190,
+  "嚋": 2191,
+  "嚌": 2192,
+  "嚍": 2193,
+  "嚎": 2194,
+  "嚏": 2195,
+  "嚐": 2196,
+  "嚑": 2197,
+  "嚒": 2198,
+  "嚓": 2199,
+  "嚔": 2200,
+  "嚕": 2201,
+  "嚖": 2202,
+  "嚗": 2203,
+  "嚘": 2204,
+  "嚙": 2205,
+  "嚚": 2206,
+  "嚛": 2207,
+  "嚜": 2208,
+  "嚝": 2209,
+  "嚞": 2210,
+  "嚟": 2211,
+  "嚠": 2212,
+  "嚡": 2213,
+  "嚢": 2214,
+  "嚣": 2215,
+  "嚤": 2216,
+  "嚥": 2217,
+  "嚦": 2218,
+  "嚧": 2219,
+  "嚨": 2220,
+  "嚩": 2221,
+  "嚪": 2222,
+  "嚫": 2223,
+  "嚬": 2224,
+  "嚭": 2225,
+  "嚮": 2226,
+  "嚯": 2227,
+  "嚰": 2228,
+  "嚱": 2229,
+  "嚲": 2230,
+  "嚳": 2231,
+  "嚴": 2232,
+  "嚵": 2233,
+  "嚶": 2234,
+  "嚷": 2235,
+  "嚸": 2236,
+  "嚹": 2237,
+  "嚺": 2238,
+  "嚻": 2239,
+  "嚼": 2240,
+  "嚽": 2241,
+  "嚾": 2242,
+  "嚿": 2243,
+  "囀": 2244,
+  "囁": 2245,
+  "囂": 2246,
+  "囃": 2247,
+  "囄": 2248,
+  "囅": 2249,
+  "囆": 2250,
+  "囇": 2251,
+  "囈": 2252,
+  "囉": 2253,
+  "囊": 2254,
+  "囋": 2255,
+  "囌": 2256,
+  "囍": 2257,
+  "囎": 2258,
+  "囏": 2259,
+  "囐": 2260,
+  "囑": 2261,
+  "囒": 2262,
+  "囓": 2263,
+  "囔": 2264,
+  "囕": 2265,
+  "囖": 2266,
+  "囗": 2267,
+  "囘": 2268,
+  "囙": 2269,
+  "囚": 2270,
+  "四": 2271,
+  "囜": 2272,
+  "囝": 2273,
+  "回": 2274,
+  "囟": 2275,
+  "因": 2276,
+  "囡": 2277,
+  "团": 2278,
+  "団": 2279,
+  "囤": 2280,
+  "囥": 2281,
+  "囦": 2282,
+  "囧": 2283,
+  "囨": 2284,
+  "囩": 2285,
+  "囪": 2286,
+  "囫": 2287,
+  "囬": 2288,
+  "园": 2289,
+  "囮": 2290,
+  "囯": 2291,
+  "困": 2292,
+  "囱": 2293,
+  "囲": 2294,
+  "図": 2295,
+  "围": 2296,
+  "囵": 2297,
+  "囶": 2298,
+  "囷": 2299,
+  "囸": 2300,
+  "囹": 2301,
+  "固": 2302,
+  "囻": 2303,
+  "囼": 2304,
+  "国": 2305,
+  "图": 2306,
+  "囿": 2307,
+  "圀": 2308,
+  "圁": 2309,
+  "圂": 2310,
+  "圃": 2311,
+  "圄": 2312,
+  "圅": 2313,
+  "圆": 2314,
+  "圇": 2315,
+  "圈": 2316,
+  "圉": 2317,
+  "圊": 2318,
+  "國": 2319,
+  "圌": 2320,
+  "圍": 2321,
+  "圎": 2322,
+  "圏": 2323,
+  "圐": 2324,
+  "圑": 2325,
+  "園": 2326,
+  "圓": 2327,
+  "圔": 2328,
+  "圕": 2329,
+  "圖": 2330,
+  "圗": 2331,
+  "團": 2332,
+  "圙": 2333,
+  "圚": 2334,
+  "圛": 2335,
+  "圜": 2336,
+  "圝": 2337,
+  "圞": 2338,
+  "土": 2339,
+  "圠": 2340,
+  "圡": 2341,
+  "圢": 2342,
+  "圣": 2343,
+  "圤": 2344,
+  "圥": 2345,
+  "圦": 2346,
+  "圧": 2347,
+  "在": 2348,
+  "圩": 2349,
+  "圪": 2350,
+  "圫": 2351,
+  "圬": 2352,
+  "圭": 2353,
+  "圮": 2354,
+  "圯": 2355,
+  "地": 2356,
+  "圱": 2357,
+  "圲": 2358,
+  "圳": 2359,
+  "圴": 2360,
+  "圵": 2361,
+  "圶": 2362,
+  "圷": 2363,
+  "圸": 2364,
+  "圹": 2365,
+  "场": 2366,
+  "圻": 2367,
+  "圼": 2368,
+  "圽": 2369,
+  "圾": 2370,
+  "圿": 2371,
+  "址": 2372,
+  "坁": 2373,
+  "坂": 2374,
+  "坃": 2375,
+  "坄": 2376,
+  "坅": 2377,
+  "坆": 2378,
+  "均": 2379,
+  "坈": 2380,
+  "坉": 2381,
+  "坊": 2382,
+  "坋": 2383,
+  "坌": 2384,
+  "坍": 2385,
+  "坎": 2386,
+  "坏": 2387,
+  "坐": 2388,
+  "坑": 2389,
+  "坒": 2390,
+  "坓": 2391,
+  "坔": 2392,
+  "坕": 2393,
+  "坖": 2394,
+  "块": 2395,
+  "坘": 2396,
+  "坙": 2397,
+  "坚": 2398,
+  "坛": 2399,
+  "坜": 2400,
+  "坝": 2401,
+  "坞": 2402,
+  "坟": 2403,
+  "坠": 2404,
+  "坡": 2405,
+  "坢": 2406,
+  "坣": 2407,
+  "坤": 2408,
+  "坥": 2409,
+  "坦": 2410,
+  "坧": 2411,
+  "坨": 2412,
+  "坩": 2413,
+  "坪": 2414,
+  "坫": 2415,
+  "坬": 2416,
+  "坭": 2417,
+  "坮": 2418,
+  "坯": 2419,
+  "坰": 2420,
+  "坱": 2421,
+  "坲": 2422,
+  "坳": 2423,
+  "坴": 2424,
+  "坵": 2425,
+  "坶": 2426,
+  "坷": 2427,
+  "坸": 2428,
+  "坹": 2429,
+  "坺": 2430,
+  "坻": 2431,
+  "坼": 2432,
+  "坽": 2433,
+  "坾": 2434,
+  "坿": 2435,
+  "垀": 2436,
+  "垁": 2437,
+  "垂": 2438,
+  "垃": 2439,
+  "垄": 2440,
+  "垅": 2441,
+  "垆": 2442,
+  "垇": 2443,
+  "垈": 2444,
+  "垉": 2445,
+  "垊": 2446,
+  "型": 2447,
+  "垌": 2448,
+  "垍": 2449,
+  "垎": 2450,
+  "垏": 2451,
+  "垐": 2452,
+  "垑": 2453,
+  "垒": 2454,
+  "垓": 2455,
+  "垔": 2456,
+  "垕": 2457,
+  "垖": 2458,
+  "垗": 2459,
+  "垘": 2460,
+  "垙": 2461,
+  "垚": 2462,
+  "垛": 2463,
+  "垜": 2464,
+  "垝": 2465,
+  "垞": 2466,
+  "垟": 2467,
+  "垠": 2468,
+  "垡": 2469,
+  "垢": 2470,
+  "垣": 2471,
+  "垤": 2472,
+  "垥": 2473,
+  "垦": 2474,
+  "垧": 2475,
+  "垨": 2476,
+  "垩": 2477,
+  "垪": 2478,
+  "垫": 2479,
+  "垬": 2480,
+  "垭": 2481,
+  "垮": 2482,
+  "垯": 2483,
+  "垰": 2484,
+  "垱": 2485,
+  "垲": 2486,
+  "垳": 2487,
+  "垴": 2488,
+  "垵": 2489,
+  "垶": 2490,
+  "垷": 2491,
+  "垸": 2492,
+  "垹": 2493,
+  "垺": 2494,
+  "垻": 2495,
+  "垼": 2496,
+  "垽": 2497,
+  "垾": 2498,
+  "垿": 2499,
+  "埀": 2500,
+  "埁": 2501,
+  "埂": 2502,
+  "埃": 2503,
+  "埄": 2504,
+  "埅": 2505,
+  "埆": 2506,
+  "埇": 2507,
+  "埈": 2508,
+  "埉": 2509,
+  "埊": 2510,
+  "埋": 2511,
+  "埌": 2512,
+  "埍": 2513,
+  "城": 2514,
+  "埏": 2515,
+  "埐": 2516,
+  "埑": 2517,
+  "埒": 2518,
+  "埓": 2519,
+  "埔": 2520,
+  "埕": 2521,
+  "埖": 2522,
+  "埗": 2523,
+  "埘": 2524,
+  "埙": 2525,
+  "埚": 2526,
+  "埛": 2527,
+  "埜": 2528,
+  "埝": 2529,
+  "埞": 2530,
+  "域": 2531,
+  "埠": 2532,
+  "埡": 2533,
+  "埢": 2534,
+  "埣": 2535,
+  "埤": 2536,
+  "埥": 2537,
+  "埦": 2538,
+  "埧": 2539,
+  "埨": 2540,
+  "埩": 2541,
+  "埪": 2542,
+  "埫": 2543,
+  "埬": 2544,
+  "埭": 2545,
+  "埮": 2546,
+  "埯": 2547,
+  "埰": 2548,
+  "埱": 2549,
+  "埲": 2550,
+  "埳": 2551,
+  "埴": 2552,
+  "埵": 2553,
+  "埶": 2554,
+  "執": 2555,
+  "埸": 2556,
+  "培": 2557,
+  "基": 2558,
+  "埻": 2559,
+  "埼": 2560,
+  "埽": 2561,
+  "埾": 2562,
+  "埿": 2563,
+  "堀": 2564,
+  "堁": 2565,
+  "堂": 2566,
+  "堃": 2567,
+  "堄": 2568,
+  "堅": 2569,
+  "堆": 2570,
+  "堇": 2571,
+  "堈": 2572,
+  "堉": 2573,
+  "堊": 2574,
+  "堋": 2575,
+  "堌": 2576,
+  "堍": 2577,
+  "堎": 2578,
+  "堏": 2579,
+  "堐": 2580,
+  "堑": 2581,
+  "堒": 2582,
+  "堓": 2583,
+  "堔": 2584,
+  "堕": 2585,
+  "堖": 2586,
+  "堗": 2587,
+  "堘": 2588,
+  "堙": 2589,
+  "堚": 2590,
+  "堛": 2591,
+  "堜": 2592,
+  "堝": 2593,
+  "堞": 2594,
+  "堟": 2595,
+  "堠": 2596,
+  "堡": 2597,
+  "堢": 2598,
+  "堣": 2599,
+  "堤": 2600,
+  "堥": 2601,
+  "堦": 2602,
+  "堧": 2603,
+  "堨": 2604,
+  "堩": 2605,
+  "堪": 2606,
+  "堫": 2607,
+  "堬": 2608,
+  "堭": 2609,
+  "堮": 2610,
+  "堯": 2611,
+  "堰": 2612,
+  "報": 2613,
+  "堲": 2614,
+  "堳": 2615,
+  "場": 2616,
+  "堵": 2617,
+  "堶": 2618,
+  "堷": 2619,
+  "堸": 2620,
+  "堹": 2621,
+  "堺": 2622,
+  "堻": 2623,
+  "堼": 2624,
+  "堽": 2625,
+  "堾": 2626,
+  "堿": 2627,
+  "塀": 2628,
+  "塁": 2629,
+  "塂": 2630,
+  "塃": 2631,
+  "塄": 2632,
+  "塅": 2633,
+  "塆": 2634,
+  "塇": 2635,
+  "塈": 2636,
+  "塉": 2637,
+  "塊": 2638,
+  "塋": 2639,
+  "塌": 2640,
+  "塍": 2641,
+  "塎": 2642,
+  "塏": 2643,
+  "塐": 2644,
+  "塑": 2645,
+  "塒": 2646,
+  "塓": 2647,
+  "塔": 2648,
+  "塕": 2649,
+  "塖": 2650,
+  "塗": 2651,
+  "塘": 2652,
+  "塙": 2653,
+  "塚": 2654,
+  "塛": 2655,
+  "塜": 2656,
+  "塝": 2657,
+  "塞": 2658,
+  "塟": 2659,
+  "塠": 2660,
+  "塡": 2661,
+  "塢": 2662,
+  "塣": 2663,
+  "塤": 2664,
+  "塥": 2665,
+  "塦": 2666,
+  "塧": 2667,
+  "塨": 2668,
+  "塩": 2669,
+  "塪": 2670,
+  "填": 2671,
+  "塬": 2672,
+  "塭": 2673,
+  "塮": 2674,
+  "塯": 2675,
+  "塰": 2676,
+  "塱": 2677,
+  "塲": 2678,
+  "塳": 2679,
+  "塴": 2680,
+  "塵": 2681,
+  "塶": 2682,
+  "塷": 2683,
+  "塸": 2684,
+  "塹": 2685,
+  "塺": 2686,
+  "塻": 2687,
+  "塼": 2688,
+  "塽": 2689,
+  "塾": 2690,
+  "塿": 2691,
+  "墀": 2692,
+  "墁": 2693,
+  "墂": 2694,
+  "境": 2695,
+  "墄": 2696,
+  "墅": 2697,
+  "墆": 2698,
+  "墇": 2699,
+  "墈": 2700,
+  "墉": 2701,
+  "墊": 2702,
+  "墋": 2703,
+  "墌": 2704,
+  "墍": 2705,
+  "墎": 2706,
+  "墏": 2707,
+  "墐": 2708,
+  "墑": 2709,
+  "墒": 2710,
+  "墓": 2711,
+  "墔": 2712,
+  "墕": 2713,
+  "墖": 2714,
+  "増": 2715,
+  "墘": 2716,
+  "墙": 2717,
+  "墚": 2718,
+  "墛": 2719,
+  "墜": 2720,
+  "墝": 2721,
+  "增": 2722,
+  "墟": 2723,
+  "墠": 2724,
+  "墡": 2725,
+  "墢": 2726,
+  "墣": 2727,
+  "墤": 2728,
+  "墥": 2729,
+  "墦": 2730,
+  "墧": 2731,
+  "墨": 2732,
+  "墩": 2733,
+  "墪": 2734,
+  "墫": 2735,
+  "墬": 2736,
+  "墭": 2737,
+  "墮": 2738,
+  "墯": 2739,
+  "墰": 2740,
+  "墱": 2741,
+  "墲": 2742,
+  "墳": 2743,
+  "墴": 2744,
+  "墵": 2745,
+  "墶": 2746,
+  "墷": 2747,
+  "墸": 2748,
+  "墹": 2749,
+  "墺": 2750,
+  "墻": 2751,
+  "墼": 2752,
+  "墽": 2753,
+  "墾": 2754,
+  "墿": 2755,
+  "壀": 2756,
+  "壁": 2757,
+  "壂": 2758,
+  "壃": 2759,
+  "壄": 2760,
+  "壅": 2761,
+  "壆": 2762,
+  "壇": 2763,
+  "壈": 2764,
+  "壉": 2765,
+  "壊": 2766,
+  "壋": 2767,
+  "壌": 2768,
+  "壍": 2769,
+  "壎": 2770,
+  "壏": 2771,
+  "壐": 2772,
+  "壑": 2773,
+  "壒": 2774,
+  "壓": 2775,
+  "壔": 2776,
+  "壕": 2777,
+  "壖": 2778,
+  "壗": 2779,
+  "壘": 2780,
+  "壙": 2781,
+  "壚": 2782,
+  "壛": 2783,
+  "壜": 2784,
+  "壝": 2785,
+  "壞": 2786,
+  "壟": 2787,
+  "壠": 2788,
+  "壡": 2789,
+  "壢": 2790,
+  "壣": 2791,
+  "壤": 2792,
+  "壥": 2793,
+  "壦": 2794,
+  "壧": 2795,
+  "壨": 2796,
+  "壩": 2797,
+  "壪": 2798,
+  "士": 2799,
+  "壬": 2800,
+  "壭": 2801,
+  "壮": 2802,
+  "壯": 2803,
+  "声": 2804,
+  "壱": 2805,
+  "売": 2806,
+  "壳": 2807,
+  "壴": 2808,
+  "壵": 2809,
+  "壶": 2810,
+  "壷": 2811,
+  "壸": 2812,
+  "壹": 2813,
+  "壺": 2814,
+  "壻": 2815,
+  "壼": 2816,
+  "壽": 2817,
+  "壾": 2818,
+  "壿": 2819,
+  "夀": 2820,
+  "夁": 2821,
+  "夂": 2822,
+  "夃": 2823,
+  "处": 2824,
+  "夅": 2825,
+  "夆": 2826,
+  "备": 2827,
+  "夈": 2828,
+  "変": 2829,
+  "夊": 2830,
+  "夋": 2831,
+  "夌": 2832,
+  "复": 2833,
+  "夎": 2834,
+  "夏": 2835,
+  "夐": 2836,
+  "夑": 2837,
+  "夒": 2838,
+  "夓": 2839,
+  "夔": 2840,
+  "夕": 2841,
+  "外": 2842,
+  "夗": 2843,
+  "夘": 2844,
+  "夙": 2845,
+  "多": 2846,
+  "夛": 2847,
+  "夜": 2848,
+  "夝": 2849,
+  "夞": 2850,
+  "够": 2851,
+  "夠": 2852,
+  "夡": 2853,
+  "夢": 2854,
+  "夣": 2855,
+  "夤": 2856,
+  "夥": 2857,
+  "夦": 2858,
+  "大": 2859,
+  "夨": 2860,
+  "天": 2861,
+  "太": 2862,
+  "夫": 2863,
+  "夬": 2864,
+  "夭": 2865,
+  "央": 2866,
+  "夯": 2867,
+  "夰": 2868,
+  "失": 2869,
+  "夲": 2870,
+  "夳": 2871,
+  "头": 2872,
+  "夵": 2873,
+  "夶": 2874,
+  "夷": 2875,
+  "夸": 2876,
+  "夹": 2877,
+  "夺": 2878,
+  "夻": 2879,
+  "夼": 2880,
+  "夽": 2881,
+  "夾": 2882,
+  "夿": 2883,
+  "奀": 2884,
+  "奁": 2885,
+  "奂": 2886,
+  "奃": 2887,
+  "奄": 2888,
+  "奅": 2889,
+  "奆": 2890,
+  "奇": 2891,
+  "奈": 2892,
+  "奉": 2893,
+  "奊": 2894,
+  "奋": 2895,
+  "奌": 2896,
+  "奍": 2897,
+  "奎": 2898,
+  "奏": 2899,
+  "奐": 2900,
+  "契": 2901,
+  "奒": 2902,
+  "奓": 2903,
+  "奔": 2904,
+  "奕": 2905,
+  "奖": 2906,
+  "套": 2907,
+  "奘": 2908,
+  "奙": 2909,
+  "奚": 2910,
+  "奛": 2911,
+  "奜": 2912,
+  "奝": 2913,
+  "奞": 2914,
+  "奟": 2915,
+  "奠": 2916,
+  "奡": 2917,
+  "奢": 2918,
+  "奣": 2919,
+  "奤": 2920,
+  "奥": 2921,
+  "奦": 2922,
+  "奧": 2923,
+  "奨": 2924,
+  "奩": 2925,
+  "奪": 2926,
+  "奫": 2927,
+  "奬": 2928,
+  "奭": 2929,
+  "奮": 2930,
+  "奯": 2931,
+  "奰": 2932,
+  "奱": 2933,
+  "奲": 2934,
+  "女": 2935,
+  "奴": 2936,
+  "奵": 2937,
+  "奶": 2938,
+  "奷": 2939,
+  "奸": 2940,
+  "她": 2941,
+  "奺": 2942,
+  "奻": 2943,
+  "奼": 2944,
+  "好": 2945,
+  "奾": 2946,
+  "奿": 2947,
+  "妀": 2948,
+  "妁": 2949,
+  "如": 2950,
+  "妃": 2951,
+  "妄": 2952,
+  "妅": 2953,
+  "妆": 2954,
+  "妇": 2955,
+  "妈": 2956,
+  "妉": 2957,
+  "妊": 2958,
+  "妋": 2959,
+  "妌": 2960,
+  "妍": 2961,
+  "妎": 2962,
+  "妏": 2963,
+  "妐": 2964,
+  "妑": 2965,
+  "妒": 2966,
+  "妓": 2967,
+  "妔": 2968,
+  "妕": 2969,
+  "妖": 2970,
+  "妗": 2971,
+  "妘": 2972,
+  "妙": 2973,
+  "妚": 2974,
+  "妛": 2975,
+  "妜": 2976,
+  "妝": 2977,
+  "妞": 2978,
+  "妟": 2979,
+  "妠": 2980,
+  "妡": 2981,
+  "妢": 2982,
+  "妣": 2983,
+  "妤": 2984,
+  "妥": 2985,
+  "妦": 2986,
+  "妧": 2987,
+  "妨": 2988,
+  "妩": 2989,
+  "妪": 2990,
+  "妫": 2991,
+  "妬": 2992,
+  "妭": 2993,
+  "妮": 2994,
+  "妯": 2995,
+  "妰": 2996,
+  "妱": 2997,
+  "妲": 2998,
+  "妳": 2999,
+  "妴": 3000,
+  "妵": 3001,
+  "妶": 3002,
+  "妷": 3003,
+  "妸": 3004,
+  "妹": 3005,
+  "妺": 3006,
+  "妻": 3007,
+  "妼": 3008,
+  "妽": 3009,
+  "妾": 3010,
+  "妿": 3011,
+  "姀": 3012,
+  "姁": 3013,
+  "姂": 3014,
+  "姃": 3015,
+  "姄": 3016,
+  "姅": 3017,
+  "姆": 3018,
+  "姇": 3019,
+  "姈": 3020,
+  "姉": 3021,
+  "姊": 3022,
+  "始": 3023,
+  "姌": 3024,
+  "姍": 3025,
+  "姎": 3026,
+  "姏": 3027,
+  "姐": 3028,
+  "姑": 3029,
+  "姒": 3030,
+  "姓": 3031,
+  "委": 3032,
+  "姕": 3033,
+  "姖": 3034,
+  "姗": 3035,
+  "姘": 3036,
+  "姙": 3037,
+  "姚": 3038,
+  "姛": 3039,
+  "姜": 3040,
+  "姝": 3041,
+  "姞": 3042,
+  "姟": 3043,
+  "姠": 3044,
+  "姡": 3045,
+  "姢": 3046,
+  "姣": 3047,
+  "姤": 3048,
+  "姥": 3049,
+  "姦": 3050,
+  "姧": 3051,
+  "姨": 3052,
+  "姩": 3053,
+  "姪": 3054,
+  "姫": 3055,
+  "姬": 3056,
+  "姭": 3057,
+  "姮": 3058,
+  "姯": 3059,
+  "姰": 3060,
+  "姱": 3061,
+  "姲": 3062,
+  "姳": 3063,
+  "姴": 3064,
+  "姵": 3065,
+  "姶": 3066,
+  "姷": 3067,
+  "姸": 3068,
+  "姹": 3069,
+  "姺": 3070,
+  "姻": 3071,
+  "姼": 3072,
+  "姽": 3073,
+  "姾": 3074,
+  "姿": 3075,
+  "娀": 3076,
+  "威": 3077,
+  "娂": 3078,
+  "娃": 3079,
+  "娄": 3080,
+  "娅": 3081,
+  "娆": 3082,
+  "娇": 3083,
+  "娈": 3084,
+  "娉": 3085,
+  "娊": 3086,
+  "娋": 3087,
+  "娌": 3088,
+  "娍": 3089,
+  "娎": 3090,
+  "娏": 3091,
+  "娐": 3092,
+  "娑": 3093,
+  "娒": 3094,
+  "娓": 3095,
+  "娔": 3096,
+  "娕": 3097,
+  "娖": 3098,
+  "娗": 3099,
+  "娘": 3100,
+  "娙": 3101,
+  "娚": 3102,
+  "娛": 3103,
+  "娜": 3104,
+  "娝": 3105,
+  "娞": 3106,
+  "娟": 3107,
+  "娠": 3108,
+  "娡": 3109,
+  "娢": 3110,
+  "娣": 3111,
+  "娤": 3112,
+  "娥": 3113,
+  "娦": 3114,
+  "娧": 3115,
+  "娨": 3116,
+  "娩": 3117,
+  "娪": 3118,
+  "娫": 3119,
+  "娬": 3120,
+  "娭": 3121,
+  "娮": 3122,
+  "娯": 3123,
+  "娰": 3124,
+  "娱": 3125,
+  "娲": 3126,
+  "娳": 3127,
+  "娴": 3128,
+  "娵": 3129,
+  "娶": 3130,
+  "娷": 3131,
+  "娸": 3132,
+  "娹": 3133,
+  "娺": 3134,
+  "娻": 3135,
+  "娼": 3136,
+  "娽": 3137,
+  "娾": 3138,
+  "娿": 3139,
+  "婀": 3140,
+  "婁": 3141,
+  "婂": 3142,
+  "婃": 3143,
+  "婄": 3144,
+  "婅": 3145,
+  "婆": 3146,
+  "婇": 3147,
+  "婈": 3148,
+  "婉": 3149,
+  "婊": 3150,
+  "婋": 3151,
+  "婌": 3152,
+  "婍": 3153,
+  "婎": 3154,
+  "婏": 3155,
+  "婐": 3156,
+  "婑": 3157,
+  "婒": 3158,
+  "婓": 3159,
+  "婔": 3160,
+  "婕": 3161,
+  "婖": 3162,
+  "婗": 3163,
+  "婘": 3164,
+  "婙": 3165,
+  "婚": 3166,
+  "婛": 3167,
+  "婜": 3168,
+  "婝": 3169,
+  "婞": 3170,
+  "婟": 3171,
+  "婠": 3172,
+  "婡": 3173,
+  "婢": 3174,
+  "婣": 3175,
+  "婤": 3176,
+  "婥": 3177,
+  "婦": 3178,
+  "婧": 3179,
+  "婨": 3180,
+  "婩": 3181,
+  "婪": 3182,
+  "婫": 3183,
+  "婬": 3184,
+  "婭": 3185,
+  "婮": 3186,
+  "婯": 3187,
+  "婰": 3188,
+  "婱": 3189,
+  "婲": 3190,
+  "婳": 3191,
+  "婴": 3192,
+  "婵": 3193,
+  "婶": 3194,
+  "婷": 3195,
+  "婸": 3196,
+  "婹": 3197,
+  "婺": 3198,
+  "婻": 3199,
+  "婼": 3200,
+  "婽": 3201,
+  "婾": 3202,
+  "婿": 3203,
+  "媀": 3204,
+  "媁": 3205,
+  "媂": 3206,
+  "媃": 3207,
+  "媄": 3208,
+  "媅": 3209,
+  "媆": 3210,
+  "媇": 3211,
+  "媈": 3212,
+  "媉": 3213,
+  "媊": 3214,
+  "媋": 3215,
+  "媌": 3216,
+  "媍": 3217,
+  "媎": 3218,
+  "媏": 3219,
+  "媐": 3220,
+  "媑": 3221,
+  "媒": 3222,
+  "媓": 3223,
+  "媔": 3224,
+  "媕": 3225,
+  "媖": 3226,
+  "媗": 3227,
+  "媘": 3228,
+  "媙": 3229,
+  "媚": 3230,
+  "媛": 3231,
+  "媜": 3232,
+  "媝": 3233,
+  "媞": 3234,
+  "媟": 3235,
+  "媠": 3236,
+  "媡": 3237,
+  "媢": 3238,
+  "媣": 3239,
+  "媤": 3240,
+  "媥": 3241,
+  "媦": 3242,
+  "媧": 3243,
+  "媨": 3244,
+  "媩": 3245,
+  "媪": 3246,
+  "媫": 3247,
+  "媬": 3248,
+  "媭": 3249,
+  "媮": 3250,
+  "媯": 3251,
+  "媰": 3252,
+  "媱": 3253,
+  "媲": 3254,
+  "媳": 3255,
+  "媴": 3256,
+  "媵": 3257,
+  "媶": 3258,
+  "媷": 3259,
+  "媸": 3260,
+  "媹": 3261,
+  "媺": 3262,
+  "媻": 3263,
+  "媼": 3264,
+  "媽": 3265,
+  "媾": 3266,
+  "媿": 3267,
+  "嫀": 3268,
+  "嫁": 3269,
+  "嫂": 3270,
+  "嫃": 3271,
+  "嫄": 3272,
+  "嫅": 3273,
+  "嫆": 3274,
+  "嫇": 3275,
+  "嫈": 3276,
+  "嫉": 3277,
+  "嫊": 3278,
+  "嫋": 3279,
+  "嫌": 3280,
+  "嫍": 3281,
+  "嫎": 3282,
+  "嫏": 3283,
+  "嫐": 3284,
+  "嫑": 3285,
+  "嫒": 3286,
+  "嫓": 3287,
+  "嫔": 3288,
+  "嫕": 3289,
+  "嫖": 3290,
+  "嫗": 3291,
+  "嫘": 3292,
+  "嫙": 3293,
+  "嫚": 3294,
+  "嫛": 3295,
+  "嫜": 3296,
+  "嫝": 3297,
+  "嫞": 3298,
+  "嫟": 3299,
+  "嫠": 3300,
+  "嫡": 3301,
+  "嫢": 3302,
+  "嫣": 3303,
+  "嫤": 3304,
+  "嫥": 3305,
+  "嫦": 3306,
+  "嫧": 3307,
+  "嫨": 3308,
+  "嫩": 3309,
+  "嫪": 3310,
+  "嫫": 3311,
+  "嫬": 3312,
+  "嫭": 3313,
+  "嫮": 3314,
+  "嫯": 3315,
+  "嫰": 3316,
+  "嫱": 3317,
+  "嫲": 3318,
+  "嫳": 3319,
+  "嫴": 3320,
+  "嫵": 3321,
+  "嫶": 3322,
+  "嫷": 3323,
+  "嫸": 3324,
+  "嫹": 3325,
+  "嫺": 3326,
+  "嫻": 3327,
+  "嫼": 3328,
+  "嫽": 3329,
+  "嫾": 3330,
+  "嫿": 3331,
+  "嬀": 3332,
+  "嬁": 3333,
+  "嬂": 3334,
+  "嬃": 3335,
+  "嬄": 3336,
+  "嬅": 3337,
+  "嬆": 3338,
+  "嬇": 3339,
+  "嬈": 3340,
+  "嬉": 3341,
+  "嬊": 3342,
+  "嬋": 3343,
+  "嬌": 3344,
+  "嬍": 3345,
+  "嬎": 3346,
+  "嬏": 3347,
+  "嬐": 3348,
+  "嬑": 3349,
+  "嬒": 3350,
+  "嬓": 3351,
+  "嬔": 3352,
+  "嬕": 3353,
+  "嬖": 3354,
+  "嬗": 3355,
+  "嬘": 3356,
+  "嬙": 3357,
+  "嬚": 3358,
+  "嬛": 3359,
+  "嬜": 3360,
+  "嬝": 3361,
+  "嬞": 3362,
+  "嬟": 3363,
+  "嬠": 3364,
+  "嬡": 3365,
+  "嬢": 3366,
+  "嬣": 3367,
+  "嬤": 3368,
+  "嬥": 3369,
+  "嬦": 3370,
+  "嬧": 3371,
+  "嬨": 3372,
+  "嬩": 3373,
+  "嬪": 3374,
+  "嬫": 3375,
+  "嬬": 3376,
+  "嬭": 3377,
+  "嬮": 3378,
+  "嬯": 3379,
+  "嬰": 3380,
+  "嬱": 3381,
+  "嬲": 3382,
+  "嬳": 3383,
+  "嬴": 3384,
+  "嬵": 3385,
+  "嬶": 3386,
+  "嬷": 3387,
+  "嬸": 3388,
+  "嬹": 3389,
+  "嬺": 3390,
+  "嬻": 3391,
+  "嬼": 3392,
+  "嬽": 3393,
+  "嬾": 3394,
+  "嬿": 3395,
+  "孀": 3396,
+  "孁": 3397,
+  "孂": 3398,
+  "孃": 3399,
+  "孄": 3400,
+  "孅": 3401,
+  "孆": 3402,
+  "孇": 3403,
+  "孈": 3404,
+  "孉": 3405,
+  "孊": 3406,
+  "孋": 3407,
+  "孌": 3408,
+  "孍": 3409,
+  "孎": 3410,
+  "孏": 3411,
+  "子": 3412,
+  "孑": 3413,
+  "孒": 3414,
+  "孓": 3415,
+  "孔": 3416,
+  "孕": 3417,
+  "孖": 3418,
+  "字": 3419,
+  "存": 3420,
+  "孙": 3421,
+  "孚": 3422,
+  "孛": 3423,
+  "孜": 3424,
+  "孝": 3425,
+  "孞": 3426,
+  "孟": 3427,
+  "孠": 3428,
+  "孡": 3429,
+  "孢": 3430,
+  "季": 3431,
+  "孤": 3432,
+  "孥": 3433,
+  "学": 3434,
+  "孧": 3435,
+  "孨": 3436,
+  "孩": 3437,
+  "孪": 3438,
+  "孫": 3439,
+  "孬": 3440,
+  "孭": 3441,
+  "孮": 3442,
+  "孯": 3443,
+  "孰": 3444,
+  "孱": 3445,
+  "孲": 3446,
+  "孳": 3447,
+  "孴": 3448,
+  "孵": 3449,
+  "孶": 3450,
+  "孷": 3451,
+  "學": 3452,
+  "孹": 3453,
+  "孺": 3454,
+  "孻": 3455,
+  "孼": 3456,
+  "孽": 3457,
+  "孾": 3458,
+  "孿": 3459,
+  "宀": 3460,
+  "宁": 3461,
+  "宂": 3462,
+  "它": 3463,
+  "宄": 3464,
+  "宅": 3465,
+  "宆": 3466,
+  "宇": 3467,
+  "守": 3468,
+  "安": 3469,
+  "宊": 3470,
+  "宋": 3471,
+  "完": 3472,
+  "宍": 3473,
+  "宎": 3474,
+  "宏": 3475,
+  "宐": 3476,
+  "宑": 3477,
+  "宒": 3478,
+  "宓": 3479,
+  "宔": 3480,
+  "宕": 3481,
+  "宖": 3482,
+  "宗": 3483,
+  "官": 3484,
+  "宙": 3485,
+  "定": 3486,
+  "宛": 3487,
+  "宜": 3488,
+  "宝": 3489,
+  "实": 3490,
+  "実": 3491,
+  "宠": 3492,
+  "审": 3493,
+  "客": 3494,
+  "宣": 3495,
+  "室": 3496,
+  "宥": 3497,
+  "宦": 3498,
+  "宧": 3499,
+  "宨": 3500,
+  "宩": 3501,
+  "宪": 3502,
+  "宫": 3503,
+  "宬": 3504,
+  "宭": 3505,
+  "宮": 3506,
+  "宯": 3507,
+  "宰": 3508,
+  "宱": 3509,
+  "宲": 3510,
+  "害": 3511,
+  "宴": 3512,
+  "宵": 3513,
+  "家": 3514,
+  "宷": 3515,
+  "宸": 3516,
+  "容": 3517,
+  "宺": 3518,
+  "宻": 3519,
+  "宼": 3520,
+  "宽": 3521,
+  "宾": 3522,
+  "宿": 3523,
+  "寀": 3524,
+  "寁": 3525,
+  "寂": 3526,
+  "寃": 3527,
+  "寄": 3528,
+  "寅": 3529,
+  "密": 3530,
+  "寇": 3531,
+  "寈": 3532,
+  "寉": 3533,
+  "寊": 3534,
+  "寋": 3535,
+  "富": 3536,
+  "寍": 3537,
+  "寎": 3538,
+  "寏": 3539,
+  "寐": 3540,
+  "寑": 3541,
+  "寒": 3542,
+  "寓": 3543,
+  "寔": 3544,
+  "寕": 3545,
+  "寖": 3546,
+  "寗": 3547,
+  "寘": 3548,
+  "寙": 3549,
+  "寚": 3550,
+  "寛": 3551,
+  "寜": 3552,
+  "寝": 3553,
+  "寞": 3554,
+  "察": 3555,
+  "寠": 3556,
+  "寡": 3557,
+  "寢": 3558,
+  "寣": 3559,
+  "寤": 3560,
+  "寥": 3561,
+  "實": 3562,
+  "寧": 3563,
+  "寨": 3564,
+  "審": 3565,
+  "寪": 3566,
+  "寫": 3567,
+  "寬": 3568,
+  "寭": 3569,
+  "寮": 3570,
+  "寯": 3571,
+  "寰": 3572,
+  "寱": 3573,
+  "寲": 3574,
+  "寳": 3575,
+  "寴": 3576,
+  "寵": 3577,
+  "寶": 3578,
+  "寷": 3579,
+  "寸": 3580,
+  "对": 3581,
+  "寺": 3582,
+  "寻": 3583,
+  "导": 3584,
+  "寽": 3585,
+  "対": 3586,
+  "寿": 3587,
+  "尀": 3588,
+  "封": 3589,
+  "専": 3590,
+  "尃": 3591,
+  "射": 3592,
+  "尅": 3593,
+  "将": 3594,
+  "將": 3595,
+  "專": 3596,
+  "尉": 3597,
+  "尊": 3598,
+  "尋": 3599,
+  "尌": 3600,
+  "對": 3601,
+  "導": 3602,
+  "小": 3603,
+  "尐": 3604,
+  "少": 3605,
+  "尒": 3606,
+  "尓": 3607,
+  "尔": 3608,
+  "尕": 3609,
+  "尖": 3610,
+  "尗": 3611,
+  "尘": 3612,
+  "尙": 3613,
+  "尚": 3614,
+  "尛": 3615,
+  "尜": 3616,
+  "尝": 3617,
+  "尞": 3618,
+  "尟": 3619,
+  "尠": 3620,
+  "尡": 3621,
+  "尢": 3622,
+  "尣": 3623,
+  "尤": 3624,
+  "尥": 3625,
+  "尦": 3626,
+  "尧": 3627,
+  "尨": 3628,
+  "尩": 3629,
+  "尪": 3630,
+  "尫": 3631,
+  "尬": 3632,
+  "尭": 3633,
+  "尮": 3634,
+  "尯": 3635,
+  "尰": 3636,
+  "就": 3637,
+  "尲": 3638,
+  "尳": 3639,
+  "尴": 3640,
+  "尵": 3641,
+  "尶": 3642,
+  "尷": 3643,
+  "尸": 3644,
+  "尹": 3645,
+  "尺": 3646,
+  "尻": 3647,
+  "尼": 3648,
+  "尽": 3649,
+  "尾": 3650,
+  "尿": 3651,
+  "局": 3652,
+  "屁": 3653,
+  "层": 3654,
+  "屃": 3655,
+  "屄": 3656,
+  "居": 3657,
+  "屆": 3658,
+  "屇": 3659,
+  "屈": 3660,
+  "屉": 3661,
+  "届": 3662,
+  "屋": 3663,
+  "屌": 3664,
+  "屍": 3665,
+  "屎": 3666,
+  "屏": 3667,
+  "屐": 3668,
+  "屑": 3669,
+  "屒": 3670,
+  "屓": 3671,
+  "屔": 3672,
+  "展": 3673,
+  "屖": 3674,
+  "屗": 3675,
+  "屘": 3676,
+  "屙": 3677,
+  "屚": 3678,
+  "屛": 3679,
+  "屜": 3680,
+  "屝": 3681,
+  "属": 3682,
+  "屟": 3683,
+  "屠": 3684,
+  "屡": 3685,
+  "屢": 3686,
+  "屣": 3687,
+  "層": 3688,
+  "履": 3689,
+  "屦": 3690,
+  "屧": 3691,
+  "屨": 3692,
+  "屩": 3693,
+  "屪": 3694,
+  "屫": 3695,
+  "屬": 3696,
+  "屭": 3697,
+  "屮": 3698,
+  "屯": 3699,
+  "屰": 3700,
+  "山": 3701,
+  "屲": 3702,
+  "屳": 3703,
+  "屴": 3704,
+  "屵": 3705,
+  "屶": 3706,
+  "屷": 3707,
+  "屸": 3708,
+  "屹": 3709,
+  "屺": 3710,
+  "屻": 3711,
+  "屼": 3712,
+  "屽": 3713,
+  "屾": 3714,
+  "屿": 3715,
+  "岀": 3716,
+  "岁": 3717,
+  "岂": 3718,
+  "岃": 3719,
+  "岄": 3720,
+  "岅": 3721,
+  "岆": 3722,
+  "岇": 3723,
+  "岈": 3724,
+  "岉": 3725,
+  "岊": 3726,
+  "岋": 3727,
+  "岌": 3728,
+  "岍": 3729,
+  "岎": 3730,
+  "岏": 3731,
+  "岐": 3732,
+  "岑": 3733,
+  "岒": 3734,
+  "岓": 3735,
+  "岔": 3736,
+  "岕": 3737,
+  "岖": 3738,
+  "岗": 3739,
+  "岘": 3740,
+  "岙": 3741,
+  "岚": 3742,
+  "岛": 3743,
+  "岜": 3744,
+  "岝": 3745,
+  "岞": 3746,
+  "岟": 3747,
+  "岠": 3748,
+  "岡": 3749,
+  "岢": 3750,
+  "岣": 3751,
+  "岤": 3752,
+  "岥": 3753,
+  "岦": 3754,
+  "岧": 3755,
+  "岨": 3756,
+  "岩": 3757,
+  "岪": 3758,
+  "the": 3759,
+  "be": 3760,
+  "to": 3761,
+  "of": 3762,
+  "and": 3763,
+  "a": 3764,
+  "in": 3765,
+  "that": 3766,
+  "have": 3767,
+  "it": 3768,
+  "for": 3769,
+  "not": 3770,
+  "on": 3771,
+  "with": 3772,
+  "he": 3773,
+  "as": 3774,
+  "you": 3775,
+  "do": 3776,
+  "at": 3777,
+  "this": 3778,
+  "but": 3779,
+  "his": 3780,
+  "by": 3781,
+  "from": 3782,
+  "they": 3783,
+  "we": 3784,
+  "say": 3785,
+  "her": 3786,
+  "she": 3787,
+  "or": 3788,
+  "an": 3789,
+  "will": 3790,
+  "my": 3791,
+  "one": 3792,
+  "all": 3793,
+  "would": 3794,
+  "there": 3795,
+  "their": 3796,
+  "what": 3797,
+  "so": 3798,
+  "up": 3799,
+  "out": 3800,
+  "if": 3801,
+  "about": 3802,
+  "who": 3803,
+  "get": 3804,
+  "which": 3805,
+  "go": 3806,
+  "me": 3807,
+  "when": 3808,
+  "make": 3809,
+  "can": 3810,
+  "like": 3811,
+  "time": 3812,
+  "no": 3813,
+  "just": 3814,
+  "him": 3815,
+  "know": 3816,
+  "take": 3817,
+  "people": 3818,
+  "into": 3819,
+  "year": 3820,
+  "your": 3821,
+  "good": 3822,
+  "some": 3823,
+  "could": 3824,
+  "them": 3825,
+  "see": 3826,
+  "other": 3827,
+  "than": 3828,
+  "then": 3829,
+  "now": 3830,
+  "look": 3831,
+  "only": 3832,
+  "come": 3833,
+  "its": 3834,
+  "over": 3835,
+  "think": 3836,
+  "also": 3837,
+  "back": 3838,
+  "after": 3839,
+  "use": 3840,
+  "two": 3841,
+  "how": 3842,
+  "our": 3843,
+  "work": 3844,
+  "first": 3845,
+  "well": 3846,
+  "way": 3847,
+  "even": 3848,
+  "new": 3849,
+  "want": 3850,
+  "because": 3851,
+  "any": 3852,
+  "these": 3853,
+  "give": 3854,
+  "day": 3855,
+  "most": 3856,
+  "us": 3857,
+  "unthe": 3858,
+  "unbe": 3859,
+  "unto": 3860,
+  "unof": 3861,
+  "unand": 3862,
+  "una": 3863,
+  "unin": 3864,
+  "unthat": 3865,
+  "unhave": 3866,
+  "unit": 3867,
+  "unfor": 3868,
+  "unnot": 3869,
+  "unon": 3870,
+  "unwith": 3871,
+  "unhe": 3872,
+  "unas": 3873,
+  "unyou": 3874,
+  "undo": 3875,
+  "unat": 3876,
+  "unthis": 3877,
+  "unbut": 3878,
+  "unhis": 3879,
+  "unby": 3880,
+  "unfrom": 3881,
+  "unthey": 3882,
+  "unwe": 3883,
+  "unsay": 3884,
+  "unher": 3885,
+  "unshe": 3886,
+  "unor": 3887,
+  "unan": 3888,
+  "unwill": 3889,
+  "unmy": 3890,
+  "unone": 3891,
+  "unall": 3892,
+  "unwould": 3893,
+  "unthere": 3894,
+  "untheir": 3895,
+  "unwhat": 3896,
+  "unso": 3897,
+  "unup": 3898,
+  "unout": 3899,
+  "unif": 3900,
+  "unabout": 3901,
+  "unwho": 3902,
+  "unget": 3903,
+  "unwhich": 3904,
+  "ungo": 3905,
+  "unme": 3906,
+  "unwhen": 3907,
+  "rethe": 3908,
+  "rebe": 3909,
+  "reto": 3910,
+  "reof": 3911,
+  "reand": 3912,
+  "rea": 3913,
+  "rein": 3914,
+  "rethat": 3915,
+  "rehave": 3916,
+  "reit": 3917,
+  "refor": 3918,
+  "renot": 3919,
+  "reon": 3920,
+  "rewith": 3921,
+  "rehe": 3922,
+  "reas": 3923,
+  "reyou": 3924,
+  "redo": 3925,
+  "reat": 3926,
+  "rethis": 3927,
+  "rebut": 3928,
+  "rehis": 3929,
+  "reby": 3930,
+  "refrom": 3931,
+  "rethey": 3932,
+  "rewe": 3933,
+  "resay": 3934,
+  "reher": 3935,
+  "reshe": 3936,
+  "reor": 3937,
+  "rean": 3938,
+  "rewill": 3939,
+  "remy": 3940,
+  "reone": 3941,
+  "reall": 3942,
+  "rewould": 3943,
+  "rethere": 3944,
+  "retheir": 3945,
+  "rewhat": 3946,
+  "reso": 3947,
+  "reup": 3948,
+  "reout": 3949,
+  "reif": 3950,
+  "reabout": 3951,
+  "rewho": 3952,
+  "reget": 3953,
+  "rewhich": 3954,
+  "rego": 3955,
+  "reme": 3956,
+  "rewhen": 3957,
+  "prethe": 3958,
+  "prebe": 3959,
+  "preto": 3960,
+  "preof": 3961,
+  "preand": 3962,
+  "prea": 3963,
+  "prein": 3964,
+  "prethat": 3965,
+  "prehave": 3966,
+  "preit": 3967,
+  "prefor": 3968,
+  "prenot": 3969,
+  "preon": 3970,
+  "prewith": 3971,
+  "prehe": 3972,
+  "preas": 3973,
+  "preyou": 3974,
+  "predo": 3975,
+  "preat": 3976,
+  "prethis": 3977,
+  "prebut": 3978,
+  "prehis": 3979,
+  "preby": 3980,
+  "prefrom": 3981,
+  "prethey": 3982,
+  "prewe": 3983,
+  "presay": 3984,
+  "preher": 3985,
+  "preshe": 3986,
+  "preor": 3987,
+  "prean": 3988,
+  "prewill": 3989,
+  "premy": 3990,
+  "preone": 3991,
+  "preall": 3992,
+  "prewould": 3993,
+  "prethere": 3994,
+  "pretheir": 3995,
+  "prewhat": 3996,
+  "preso": 3997,
+  "preup": 3998,
+  "preout": 3999,
+  "preif": 4000,
+  "preabout": 4001,
+  "prewho": 4002,
+  "preget": 4003,
+  "prewhich": 4004,
+  "prego": 4005,
+  "preme": 4006,
+  "prewhen": 4007,
+  "disthe": 4008,
+  "disbe": 4009,
+  "disto": 4010,
+  "disof": 4011,
+  "disand": 4012,
+  "disa": 4013,
+  "disin": 4014,
+  "disthat": 4015,
+  "dishave": 4016,
+  "disit": 4017,
+  "disfor": 4018,
+  "disnot": 4019,
+  "dison": 4020,
+  "diswith": 4021,
+  "dishe": 4022,
+  "disas": 4023,
+  "disyou": 4024,
+  "disdo": 4025,
+  "disat": 4026,
+  "disthis": 4027,
+  "disbut": 4028,
+  "dishis": 4029,
+  "disby": 4030,
+  "disfrom": 4031,
+  "disthey": 4032,
+  "diswe": 4033,
+  "dissay": 4034,
+  "disher": 4035,
+  "disshe": 4036,
+  "disor": 4037,
+  "disan": 4038,
+  "diswill": 4039,
+  "dismy": 4040,
+  "disone": 4041,
+  "disall": 4042,
+  "diswould": 4043,
+  "disthere": 4044,
+  "distheir": 4045,
+  "diswhat": 4046,
+  "disso": 4047,
+  "disup": 4048,
+  "disout": 4049,
+  "disif": 4050,
+  "disabout": 4051,
+  "diswho": 4052,
+  "disget": 4053,
+  "diswhich": 4054,
+  "disgo": 4055,
+  "disme": 4056,
+  "diswhen": 4057,
+  "misthe": 4058,
+  "misbe": 4059,
+  "misto": 4060,
+  "misof": 4061,
+  "misand": 4062,
+  "misa": 4063,
+  "misin": 4064,
+  "misthat": 4065,
+  "mishave": 4066,
+  "misit": 4067,
+  "misfor": 4068,
+  "misnot": 4069,
+  "mison": 4070,
+  "miswith": 4071,
+  "mishe": 4072,
+  "misas": 4073,
+  "misyou": 4074,
+  "misdo": 4075,
+  "misat": 4076,
+  "misthis": 4077,
+  "misbut": 4078,
+  "mishis": 4079,
+  "misby": 4080,
+  "misfrom": 4081,
+  "misthey": 4082,
+  "miswe": 4083,
+  "missay": 4084,
+  "misher": 4085,
+  "misshe": 4086,
+  "misor": 4087,
+  "misan": 4088,
+  "miswill": 4089,
+  "mismy": 4090,
+  "misone": 4091,
+  "misall": 4092,
+  "miswould": 4093,
+  "misthere": 4094,
+  "mistheir": 4095,
+  "miswhat": 4096,
+  "misso": 4097,
+  "misup": 4098,
+  "misout": 4099,
+  "misif": 4100,
+  "misabout": 4101,
+  "miswho": 4102,
+  "misget": 4103,
+  "miswhich": 4104,
+  "misgo": 4105,
+  "misme": 4106,
+  "miswhen": 4107,
+  "overthe": 4108,
+  "overbe": 4109,
+  "overto": 4110,
+  "overof": 4111,
+  "overand": 4112,
+  "overa": 4113,
+  "overin": 4114,
+  "overthat": 4115,
+  "overhave": 4116,
+  "overit": 4117,
+  "overfor": 4118,
+  "overnot": 4119,
+  "overon": 4120,
+  "overwith": 4121,
+  "overhe": 4122,
+  "overas": 4123,
+  "overyou": 4124,
+  "overdo": 4125,
+  "overat": 4126,
+  "overthis": 4127,
+  "overbut": 4128,
+  "overhis": 4129,
+  "overby": 4130,
+  "overfrom": 4131,
+  "overthey": 4132,
+  "overwe": 4133,
+  "oversay": 4134,
+  "overher": 4135,
+  "overshe": 4136,
+  "overor": 4137,
+  "overan": 4138,
+  "overwill": 4139,
+  "overmy": 4140,
+  "overone": 4141,
+  "overall": 4142,
+  "overwould": 4143,
+  "overthere": 4144,
+  "overtheir": 4145,
+  "overwhat": 4146,
+  "overso": 4147,
+  "overup": 4148,
+  "overout": 4149,
+  "overif": 4150,
+  "overabout": 4151,
+  "overwho": 4152,
+  "overget": 4153,
+  "overwhich": 4154,
+  "overgo": 4155,
+  "overme": 4156,
+  "overwhen": 4157,
+  "underthe": 4158,
+  "underbe": 4159,
+  "underto": 4160,
+  "underof": 4161,
+  "underand": 4162,
+  "undera": 4163,
+  "underin": 4164,
+  "underthat": 4165,
+  "underhave": 4166,
+  "underit": 4167,
+  "underfor": 4168,
+  "undernot": 4169,
+  "underon": 4170,
+  "underwith": 4171,
+  "underhe": 4172,
+  "underas": 4173,
+  "underyou": 4174,
+  "underdo": 4175,
+  "underat": 4176,
+  "underthis": 4177,
+  "underbut": 4178,
+  "underhis": 4179,
+  "underby": 4180,
+  "underfrom": 4181,
+  "underthey": 4182,
+  "underwe": 4183,
+  "undersay": 4184,
+  "underher": 4185,
+  "undershe": 4186,
+  "underor": 4187,
+  "underan": 4188,
+  "underwill": 4189,
+  "undermy": 4190,
+  "underone": 4191,
+  "underall": 4192,
+  "underwould": 4193,
+  "underthere": 4194,
+  "undertheir": 4195,
+  "underwhat": 4196,
+  "underso": 4197,
+  "underup": 4198,
+  "underout": 4199,
+  "underif": 4200,
+  "underabout": 4201,
+  "underwho": 4202,
+  "underget": 4203,
+  "underwhich": 4204,
+  "undergo": 4205,
+  "underme": 4206,
+  "underwhen": 4207,
+  "outthe": 4208,
+  "outbe": 4209,
+  "outto": 4210,
+  "outof": 4211,
+  "outand": 4212,
+  "outa": 4213,
+  "outin": 4214,
+  "outthat": 4215,
+  "outhave": 4216,
+  "outit": 4217,
+  "outfor": 4218,
+  "outnot": 4219,
+  "outon": 4220,
+  "outwith": 4221,
+  "outhe": 4222,
+  "outas": 4223,
+  "outyou": 4224,
+  "outdo": 4225,
+  "outat": 4226,
+  "outthis": 4227,
+  "outbut": 4228,
+  "outhis": 4229,
+  "outby": 4230,
+  "outfrom": 4231,
+  "outthey": 4232,
+  "outwe": 4233,
+  "outsay": 4234,
+  "outher": 4235,
+  "outshe": 4236,
+  "outor": 4237,
+  "outan": 4238,
+  "outwill": 4239,
+  "outmy": 4240,
+  "outone": 4241,
+  "outall": 4242,
+  "outwould": 4243,
+  "outthere": 4244,
+  "outtheir": 4245,
+  "outwhat": 4246,
+  "outso": 4247,
+  "outup": 4248,
+  "outout": 4249,
+  "outif": 4250,
+  "outabout": 4251,
+  "outwho": 4252,
+  "outget": 4253,
+  "outwhich": 4254,
+  "outgo": 4255,
+  "outme": 4256,
+  "outwhen": 4257,
+  "subthe": 4258,
+  "。": 4259,
+  "，": 4260,
+  "、": 4278,
+  "；": 4262,
+  "：": 4263,
+  "！": 4264,
+  "？": 4265,
+  "…": 4266,
+  "—": 4267,
+  "'": 4286,
+  "（": 4270,
+  "）": 4271,
+  "【": 4272,
+  "】": 4273,
+  "《": 4274,
+  "》": 4275,
+  "·": 4276,
+  "～": 4277,
+  ".": 4279,
+  ",": 4280,
+  "!": 4281,
+  "?": 4282,
+  ";": 4283,
+  ":": 4284,
+  "\"": 4285,
+  "(": 4287,
+  ")": 4288,
+  "[": 4289,
+  "]": 4290,
+  "{": 4291,
+  "}": 4292,
+  "<": 4293,
+  ">": 4294,
+  "-": 4295,
+  "_": 4296,
+  "/": 4297,
+  "\\": 4298,
+  "|": 4299,
+  "@": 4300,
+  "#": 4301,
+  "$": 4302,
+  "%": 4303,
+  "^": 4304,
+  "&": 4305,
+  "*": 4306,
+  "+": 4307,
+  "=": 4308,
+  "~": 4309,
+  "`": 4310,
+  "0": 4311,
+  "1": 4312,
+  "2": 4313,
+  "3": 4314,
+  "4": 4315,
+  "5": 4316,
+  "6": 4317,
+  "7": 4318,
+  "8": 4319,
+  "9": 4320,
+  "10": 4321,
+  "00": 4322,
+  "01": 4323,
+  "20": 4324,
+  "30": 4325,
+  "50": 4326,
+  "100": 4327,
+  "200": 4328,
+  "500": 4329,
+  "1000": 4330,
+  "<extra_4331>": 4331,
+  "<extra_4332>": 4332,
+  "<extra_4333>": 4333,
+  "<extra_4334>": 4334,
+  "<extra_4335>": 4335,
+  "<extra_4336>": 4336,
+  "<extra_4337>": 4337,
+  "<extra_4338>": 4338,
+  "<extra_4339>": 4339,
+  "<extra_4340>": 4340,
+  "<extra_4341>": 4341,
+  "<extra_4342>": 4342,
+  "<extra_4343>": 4343,
+  "<extra_4344>": 4344,
+  "<extra_4345>": 4345,
+  "<extra_4346>": 4346,
+  "<extra_4347>": 4347,
+  "<extra_4348>": 4348,
+  "<extra_4349>": 4349,
+  "<extra_4350>": 4350,
+  "<extra_4351>": 4351,
+  "<extra_4352>": 4352,
+  "<extra_4353>": 4353,
+  "<extra_4354>": 4354,
+  "<extra_4355>": 4355,
+  "<extra_4356>": 4356,
+  "<extra_4357>": 4357,
+  "<extra_4358>": 4358,
+  "<extra_4359>": 4359,
+  "<extra_4360>": 4360,
+  "<extra_4361>": 4361,
+  "<extra_4362>": 4362,
+  "<extra_4363>": 4363,
+  "<extra_4364>": 4364,
+  "<extra_4365>": 4365,
+  "<extra_4366>": 4366,
+  "<extra_4367>": 4367,
+  "<extra_4368>": 4368,
+  "<extra_4369>": 4369,
+  "<extra_4370>": 4370,
+  "<extra_4371>": 4371,
+  "<extra_4372>": 4372,
+  "<extra_4373>": 4373,
+  "<extra_4374>": 4374,
+  "<extra_4375>": 4375,
+  "<extra_4376>": 4376,
+  "<extra_4377>": 4377,
+  "<extra_4378>": 4378,
+  "<extra_4379>": 4379,
+  "<extra_4380>": 4380,
+  "<extra_4381>": 4381,
+  "<extra_4382>": 4382,
+  "<extra_4383>": 4383,
+  "<extra_4384>": 4384,
+  "<extra_4385>": 4385,
+  "<extra_4386>": 4386,
+  "<extra_4387>": 4387,
+  "<extra_4388>": 4388,
+  "<extra_4389>": 4389,
+  "<extra_4390>": 4390,
+  "<extra_4391>": 4391,
+  "<extra_4392>": 4392,
+  "<extra_4393>": 4393,
+  "<extra_4394>": 4394,
+  "<extra_4395>": 4395,
+  "<extra_4396>": 4396,
+  "<extra_4397>": 4397,
+  "<extra_4398>": 4398,
+  "<extra_4399>": 4399,
+  "<extra_4400>": 4400,
+  "<extra_4401>": 4401,
+  "<extra_4402>": 4402,
+  "<extra_4403>": 4403,
+  "<extra_4404>": 4404,
+  "<extra_4405>": 4405,
+  "<extra_4406>": 4406,
+  "<extra_4407>": 4407,
+  "<extra_4408>": 4408,
+  "<extra_4409>": 4409,
+  "<extra_4410>": 4410,
+  "<extra_4411>": 4411,
+  "<extra_4412>": 4412,
+  "<extra_4413>": 4413,
+  "<extra_4414>": 4414,
+  "<extra_4415>": 4415,
+  "<extra_4416>": 4416,
+  "<extra_4417>": 4417,
+  "<extra_4418>": 4418,
+  "<extra_4419>": 4419,
+  "<extra_4420>": 4420,
+  "<extra_4421>": 4421,
+  "<extra_4422>": 4422,
+  "<extra_4423>": 4423,
+  "<extra_4424>": 4424,
+  "<extra_4425>": 4425,
+  "<extra_4426>": 4426,
+  "<extra_4427>": 4427,
+  "<extra_4428>": 4428,
+  "<extra_4429>": 4429,
+  "<extra_4430>": 4430,
+  "<extra_4431>": 4431,
+  "<extra_4432>": 4432,
+  "<extra_4433>": 4433,
+  "<extra_4434>": 4434,
+  "<extra_4435>": 4435,
+  "<extra_4436>": 4436,
+  "<extra_4437>": 4437,
+  "<extra_4438>": 4438,
+  "<extra_4439>": 4439,
+  "<extra_4440>": 4440,
+  "<extra_4441>": 4441,
+  "<extra_4442>": 4442,
+  "<extra_4443>": 4443,
+  "<extra_4444>": 4444,
+  "<extra_4445>": 4445,
+  "<extra_4446>": 4446,
+  "<extra_4447>": 4447,
+  "<extra_4448>": 4448,
+  "<extra_4449>": 4449,
+  "<extra_4450>": 4450,
+  "<extra_4451>": 4451,
+  "<extra_4452>": 4452,
+  "<extra_4453>": 4453,
+  "<extra_4454>": 4454,
+  "<extra_4455>": 4455,
+  "<extra_4456>": 4456,
+  "<extra_4457>": 4457,
+  "<extra_4458>": 4458,
+  "<extra_4459>": 4459,
+  "<extra_4460>": 4460,
+  "<extra_4461>": 4461,
+  "<extra_4462>": 4462,
+  "<extra_4463>": 4463,
+  "<extra_4464>": 4464,
+  "<extra_4465>": 4465,
+  "<extra_4466>": 4466,
+  "<extra_4467>": 4467,
+  "<extra_4468>": 4468,
+  "<extra_4469>": 4469,
+  "<extra_4470>": 4470,
+  "<extra_4471>": 4471,
+  "<extra_4472>": 4472,
+  "<extra_4473>": 4473,
+  "<extra_4474>": 4474,
+  "<extra_4475>": 4475,
+  "<extra_4476>": 4476,
+  "<extra_4477>": 4477,
+  "<extra_4478>": 4478,
+  "<extra_4479>": 4479,
+  "<extra_4480>": 4480,
+  "<extra_4481>": 4481,
+  "<extra_4482>": 4482,
+  "<extra_4483>": 4483,
+  "<extra_4484>": 4484,
+  "<extra_4485>": 4485,
+  "<extra_4486>": 4486,
+  "<extra_4487>": 4487,
+  "<extra_4488>": 4488,
+  "<extra_4489>": 4489,
+  "<extra_4490>": 4490,
+  "<extra_4491>": 4491,
+  "<extra_4492>": 4492,
+  "<extra_4493>": 4493,
+  "<extra_4494>": 4494,
+  "<extra_4495>": 4495,
+  "<extra_4496>": 4496,
+  "<extra_4497>": 4497,
+  "<extra_4498>": 4498,
+  "<extra_4499>": 4499,
+  "<extra_4500>": 4500,
+  "<extra_4501>": 4501,
+  "<extra_4502>": 4502,
+  "<extra_4503>": 4503,
+  "<extra_4504>": 4504,
+  "<extra_4505>": 4505,
+  "<extra_4506>": 4506,
+  "<extra_4507>": 4507,
+  "<extra_4508>": 4508,
+  "<extra_4509>": 4509,
+  "<extra_4510>": 4510,
+  "<extra_4511>": 4511,
+  "<extra_4512>": 4512,
+  "<extra_4513>": 4513,
+  "<extra_4514>": 4514,
+  "<extra_4515>": 4515,
+  "<extra_4516>": 4516,
+  "<extra_4517>": 4517,
+  "<extra_4518>": 4518,
+  "<extra_4519>": 4519,
+  "<extra_4520>": 4520,
+  "<extra_4521>": 4521,
+  "<extra_4522>": 4522,
+  "<extra_4523>": 4523,
+  "<extra_4524>": 4524,
+  "<extra_4525>": 4525,
+  "<extra_4526>": 4526,
+  "<extra_4527>": 4527,
+  "<extra_4528>": 4528,
+  "<extra_4529>": 4529,
+  "<extra_4530>": 4530,
+  "<extra_4531>": 4531,
+  "<extra_4532>": 4532,
+  "<extra_4533>": 4533,
+  "<extra_4534>": 4534,
+  "<extra_4535>": 4535,
+  "<extra_4536>": 4536,
+  "<extra_4537>": 4537,
+  "<extra_4538>": 4538,
+  "<extra_4539>": 4539,
+  "<extra_4540>": 4540,
+  "<extra_4541>": 4541,
+  "<extra_4542>": 4542,
+  "<extra_4543>": 4543,
+  "<extra_4544>": 4544,
+  "<extra_4545>": 4545,
+  "<extra_4546>": 4546,
+  "<extra_4547>": 4547,
+  "<extra_4548>": 4548,
+  "<extra_4549>": 4549,
+  "<extra_4550>": 4550,
+  "<extra_4551>": 4551,
+  "<extra_4552>": 4552,
+  "<extra_4553>": 4553,
+  "<extra_4554>": 4554,
+  "<extra_4555>": 4555,
+  "<extra_4556>": 4556,
+  "<extra_4557>": 4557,
+  "<extra_4558>": 4558,
+  "<extra_4559>": 4559,
+  "<extra_4560>": 4560,
+  "<extra_4561>": 4561,
+  "<extra_4562>": 4562,
+  "<extra_4563>": 4563,
+  "<extra_4564>": 4564,
+  "<extra_4565>": 4565,
+  "<extra_4566>": 4566,
+  "<extra_4567>": 4567,
+  "<extra_4568>": 4568,
+  "<extra_4569>": 4569,
+  "<extra_4570>": 4570,
+  "<extra_4571>": 4571,
+  "<extra_4572>": 4572,
+  "<extra_4573>": 4573,
+  "<extra_4574>": 4574,
+  "<extra_4575>": 4575,
+  "<extra_4576>": 4576,
+  "<extra_4577>": 4577,
+  "<extra_4578>": 4578,
+  "<extra_4579>": 4579,
+  "<extra_4580>": 4580,
+  "<extra_4581>": 4581,
+  "<extra_4582>": 4582,
+  "<extra_4583>": 4583,
+  "<extra_4584>": 4584,
+  "<extra_4585>": 4585,
+  "<extra_4586>": 4586,
+  "<extra_4587>": 4587,
+  "<extra_4588>": 4588,
+  "<extra_4589>": 4589,
+  "<extra_4590>": 4590,
+  "<extra_4591>": 4591,
+  "<extra_4592>": 4592,
+  "<extra_4593>": 4593,
+  "<extra_4594>": 4594,
+  "<extra_4595>": 4595,
+  "<extra_4596>": 4596,
+  "<extra_4597>": 4597,
+  "<extra_4598>": 4598,
+  "<extra_4599>": 4599,
+  "<extra_4600>": 4600,
+  "<extra_4601>": 4601,
+  "<extra_4602>": 4602,
+  "<extra_4603>": 4603,
+  "<extra_4604>": 4604,
+  "<extra_4605>": 4605,
+  "<extra_4606>": 4606,
+  "<extra_4607>": 4607,
+  "<extra_4608>": 4608,
+  "<extra_4609>": 4609,
+  "<extra_4610>": 4610,
+  "<extra_4611>": 4611,
+  "<extra_4612>": 4612,
+  "<extra_4613>": 4613,
+  "<extra_4614>": 4614,
+  "<extra_4615>": 4615,
+  "<extra_4616>": 4616,
+  "<extra_4617>": 4617,
+  "<extra_4618>": 4618,
+  "<extra_4619>": 4619,
+  "<extra_4620>": 4620,
+  "<extra_4621>": 4621,
+  "<extra_4622>": 4622,
+  "<extra_4623>": 4623,
+  "<extra_4624>": 4624,
+  "<extra_4625>": 4625,
+  "<extra_4626>": 4626,
+  "<extra_4627>": 4627,
+  "<extra_4628>": 4628,
+  "<extra_4629>": 4629,
+  "<extra_4630>": 4630,
+  "<extra_4631>": 4631,
+  "<extra_4632>": 4632,
+  "<extra_4633>": 4633,
+  "<extra_4634>": 4634,
+  "<extra_4635>": 4635,
+  "<extra_4636>": 4636,
+  "<extra_4637>": 4637,
+  "<extra_4638>": 4638,
+  "<extra_4639>": 4639,
+  "<extra_4640>": 4640,
+  "<extra_4641>": 4641,
+  "<extra_4642>": 4642,
+  "<extra_4643>": 4643,
+  "<extra_4644>": 4644,
+  "<extra_4645>": 4645,
+  "<extra_4646>": 4646,
+  "<extra_4647>": 4647,
+  "<extra_4648>": 4648,
+  "<extra_4649>": 4649,
+  "<extra_4650>": 4650,
+  "<extra_4651>": 4651,
+  "<extra_4652>": 4652,
+  "<extra_4653>": 4653,
+  "<extra_4654>": 4654,
+  "<extra_4655>": 4655,
+  "<extra_4656>": 4656,
+  "<extra_4657>": 4657,
+  "<extra_4658>": 4658,
+  "<extra_4659>": 4659,
+  "<extra_4660>": 4660,
+  "<extra_4661>": 4661,
+  "<extra_4662>": 4662,
+  "<extra_4663>": 4663,
+  "<extra_4664>": 4664,
+  "<extra_4665>": 4665,
+  "<extra_4666>": 4666,
+  "<extra_4667>": 4667,
+  "<extra_4668>": 4668,
+  "<extra_4669>": 4669,
+  "<extra_4670>": 4670,
+  "<extra_4671>": 4671,
+  "<extra_4672>": 4672,
+  "<extra_4673>": 4673,
+  "<extra_4674>": 4674,
+  "<extra_4675>": 4675,
+  "<extra_4676>": 4676,
+  "<extra_4677>": 4677,
+  "<extra_4678>": 4678,
+  "<extra_4679>": 4679,
+  "<extra_4680>": 4680,
+  "<extra_4681>": 4681,
+  "<extra_4682>": 4682,
+  "<extra_4683>": 4683,
+  "<extra_4684>": 4684,
+  "<extra_4685>": 4685,
+  "<extra_4686>": 4686,
+  "<extra_4687>": 4687,
+  "<extra_4688>": 4688,
+  "<extra_4689>": 4689,
+  "<extra_4690>": 4690,
+  "<extra_4691>": 4691,
+  "<extra_4692>": 4692,
+  "<extra_4693>": 4693,
+  "<extra_4694>": 4694,
+  "<extra_4695>": 4695,
+  "<extra_4696>": 4696,
+  "<extra_4697>": 4697,
+  "<extra_4698>": 4698,
+  "<extra_4699>": 4699,
+  "<extra_4700>": 4700,
+  "<extra_4701>": 4701,
+  "<extra_4702>": 4702,
+  "<extra_4703>": 4703,
+  "<extra_4704>": 4704,
+  "<extra_4705>": 4705,
+  "<extra_4706>": 4706,
+  "<extra_4707>": 4707,
+  "<extra_4708>": 4708,
+  "<extra_4709>": 4709,
+  "<extra_4710>": 4710,
+  "<extra_4711>": 4711,
+  "<extra_4712>": 4712,
+  "<extra_4713>": 4713,
+  "<extra_4714>": 4714,
+  "<extra_4715>": 4715,
+  "<extra_4716>": 4716,
+  "<extra_4717>": 4717,
+  "<extra_4718>": 4718,
+  "<extra_4719>": 4719,
+  "<extra_4720>": 4720,
+  "<extra_4721>": 4721,
+  "<extra_4722>": 4722,
+  "<extra_4723>": 4723,
+  "<extra_4724>": 4724,
+  "<extra_4725>": 4725,
+  "<extra_4726>": 4726,
+  "<extra_4727>": 4727,
+  "<extra_4728>": 4728,
+  "<extra_4729>": 4729,
+  "<extra_4730>": 4730,
+  "<extra_4731>": 4731,
+  "<extra_4732>": 4732,
+  "<extra_4733>": 4733,
+  "<extra_4734>": 4734,
+  "<extra_4735>": 4735,
+  "<extra_4736>": 4736,
+  "<extra_4737>": 4737,
+  "<extra_4738>": 4738,
+  "<extra_4739>": 4739,
+  "<extra_4740>": 4740,
+  "<extra_4741>": 4741,
+  "<extra_4742>": 4742,
+  "<extra_4743>": 4743,
+  "<extra_4744>": 4744,
+  "<extra_4745>": 4745,
+  "<extra_4746>": 4746,
+  "<extra_4747>": 4747,
+  "<extra_4748>": 4748,
+  "<extra_4749>": 4749,
+  "<extra_4750>": 4750,
+  "<extra_4751>": 4751,
+  "<extra_4752>": 4752,
+  "<extra_4753>": 4753,
+  "<extra_4754>": 4754,
+  "<extra_4755>": 4755,
+  "<extra_4756>": 4756,
+  "<extra_4757>": 4757,
+  "<extra_4758>": 4758,
+  "<extra_4759>": 4759,
+  "<extra_4760>": 4760,
+  "<extra_4761>": 4761,
+  "<extra_4762>": 4762,
+  "<extra_4763>": 4763,
+  "<extra_4764>": 4764,
+  "<extra_4765>": 4765,
+  "<extra_4766>": 4766,
+  "<extra_4767>": 4767,
+  "<extra_4768>": 4768,
+  "<extra_4769>": 4769,
+  "<extra_4770>": 4770,
+  "<extra_4771>": 4771,
+  "<extra_4772>": 4772,
+  "<extra_4773>": 4773,
+  "<extra_4774>": 4774,
+  "<extra_4775>": 4775,
+  "<extra_4776>": 4776,
+  "<extra_4777>": 4777,
+  "<extra_4778>": 4778,
+  "<extra_4779>": 4779,
+  "<extra_4780>": 4780,
+  "<extra_4781>": 4781,
+  "<extra_4782>": 4782,
+  "<extra_4783>": 4783,
+  "<extra_4784>": 4784,
+  "<extra_4785>": 4785,
+  "<extra_4786>": 4786,
+  "<extra_4787>": 4787,
+  "<extra_4788>": 4788,
+  "<extra_4789>": 4789,
+  "<extra_4790>": 4790,
+  "<extra_4791>": 4791,
+  "<extra_4792>": 4792,
+  "<extra_4793>": 4793,
+  "<extra_4794>": 4794,
+  "<extra_4795>": 4795,
+  "<extra_4796>": 4796,
+  "<extra_4797>": 4797,
+  "<extra_4798>": 4798,
+  "<extra_4799>": 4799,
+  "<extra_4800>": 4800,
+  "<extra_4801>": 4801,
+  "<extra_4802>": 4802,
+  "<extra_4803>": 4803,
+  "<extra_4804>": 4804,
+  "<extra_4805>": 4805,
+  "<extra_4806>": 4806,
+  "<extra_4807>": 4807,
+  "<extra_4808>": 4808,
+  "<extra_4809>": 4809,
+  "<extra_4810>": 4810,
+  "<extra_4811>": 4811,
+  "<extra_4812>": 4812,
+  "<extra_4813>": 4813,
+  "<extra_4814>": 4814,
+  "<extra_4815>": 4815,
+  "<extra_4816>": 4816,
+  "<extra_4817>": 4817,
+  "<extra_4818>": 4818,
+  "<extra_4819>": 4819,
+  "<extra_4820>": 4820,
+  "<extra_4821>": 4821,
+  "<extra_4822>": 4822,
+  "<extra_4823>": 4823,
+  "<extra_4824>": 4824,
+  "<extra_4825>": 4825,
+  "<extra_4826>": 4826,
+  "<extra_4827>": 4827,
+  "<extra_4828>": 4828,
+  "<extra_4829>": 4829,
+  "<extra_4830>": 4830,
+  "<extra_4831>": 4831,
+  "<extra_4832>": 4832,
+  "<extra_4833>": 4833,
+  "<extra_4834>": 4834,
+  "<extra_4835>": 4835,
+  "<extra_4836>": 4836,
+  "<extra_4837>": 4837,
+  "<extra_4838>": 4838,
+  "<extra_4839>": 4839,
+  "<extra_4840>": 4840,
+  "<extra_4841>": 4841,
+  "<extra_4842>": 4842,
+  "<extra_4843>": 4843,
+  "<extra_4844>": 4844,
+  "<extra_4845>": 4845,
+  "<extra_4846>": 4846,
+  "<extra_4847>": 4847,
+  "<extra_4848>": 4848,
+  "<extra_4849>": 4849,
+  "<extra_4850>": 4850,
+  "<extra_4851>": 4851,
+  "<extra_4852>": 4852,
+  "<extra_4853>": 4853,
+  "<extra_4854>": 4854,
+  "<extra_4855>": 4855,
+  "<extra_4856>": 4856,
+  "<extra_4857>": 4857,
+  "<extra_4858>": 4858,
+  "<extra_4859>": 4859,
+  "<extra_4860>": 4860,
+  "<extra_4861>": 4861,
+  "<extra_4862>": 4862,
+  "<extra_4863>": 4863,
+  "<extra_4864>": 4864,
+  "<extra_4865>": 4865,
+  "<extra_4866>": 4866,
+  "<extra_4867>": 4867,
+  "<extra_4868>": 4868,
+  "<extra_4869>": 4869,
+  "<extra_4870>": 4870,
+  "<extra_4871>": 4871,
+  "<extra_4872>": 4872,
+  "<extra_4873>": 4873,
+  "<extra_4874>": 4874,
+  "<extra_4875>": 4875,
+  "<extra_4876>": 4876,
+  "<extra_4877>": 4877,
+  "<extra_4878>": 4878,
+  "<extra_4879>": 4879,
+  "<extra_4880>": 4880,
+  "<extra_4881>": 4881,
+  "<extra_4882>": 4882,
+  "<extra_4883>": 4883,
+  "<extra_4884>": 4884,
+  "<extra_4885>": 4885,
+  "<extra_4886>": 4886,
+  "<extra_4887>": 4887,
+  "<extra_4888>": 4888,
+  "<extra_4889>": 4889,
+  "<extra_4890>": 4890,
+  "<extra_4891>": 4891,
+  "<extra_4892>": 4892,
+  "<extra_4893>": 4893,
+  "<extra_4894>": 4894,
+  "<extra_4895>": 4895,
+  "<extra_4896>": 4896,
+  "<extra_4897>": 4897,
+  "<extra_4898>": 4898,
+  "<extra_4899>": 4899,
+  "<extra_4900>": 4900,
+  "<extra_4901>": 4901,
+  "<extra_4902>": 4902,
+  "<extra_4903>": 4903,
+  "<extra_4904>": 4904,
+  "<extra_4905>": 4905,
+  "<extra_4906>": 4906,
+  "<extra_4907>": 4907,
+  "<extra_4908>": 4908,
+  "<extra_4909>": 4909,
+  "<extra_4910>": 4910,
+  "<extra_4911>": 4911,
+  "<extra_4912>": 4912,
+  "<extra_4913>": 4913,
+  "<extra_4914>": 4914,
+  "<extra_4915>": 4915,
+  "<extra_4916>": 4916,
+  "<extra_4917>": 4917,
+  "<extra_4918>": 4918,
+  "<extra_4919>": 4919,
+  "<extra_4920>": 4920,
+  "<extra_4921>": 4921,
+  "<extra_4922>": 4922,
+  "<extra_4923>": 4923,
+  "<extra_4924>": 4924,
+  "<extra_4925>": 4925,
+  "<extra_4926>": 4926,
+  "<extra_4927>": 4927,
+  "<extra_4928>": 4928,
+  "<extra_4929>": 4929,
+  "<extra_4930>": 4930,
+  "<extra_4931>": 4931,
+  "<extra_4932>": 4932,
+  "<extra_4933>": 4933,
+  "<extra_4934>": 4934,
+  "<extra_4935>": 4935,
+  "<extra_4936>": 4936,
+  "<extra_4937>": 4937,
+  "<extra_4938>": 4938,
+  "<extra_4939>": 4939,
+  "<extra_4940>": 4940,
+  "<extra_4941>": 4941,
+  "<extra_4942>": 4942,
+  "<extra_4943>": 4943,
+  "<extra_4944>": 4944,
+  "<extra_4945>": 4945,
+  "<extra_4946>": 4946,
+  "<extra_4947>": 4947,
+  "<extra_4948>": 4948,
+  "<extra_4949>": 4949,
+  "<extra_4950>": 4950,
+  "<extra_4951>": 4951,
+  "<extra_4952>": 4952,
+  "<extra_4953>": 4953,
+  "<extra_4954>": 4954,
+  "<extra_4955>": 4955,
+  "<extra_4956>": 4956,
+  "<extra_4957>": 4957,
+  "<extra_4958>": 4958,
+  "<extra_4959>": 4959,
+  "<extra_4960>": 4960,
+  "<extra_4961>": 4961,
+  "<extra_4962>": 4962,
+  "<extra_4963>": 4963,
+  "<extra_4964>": 4964,
+  "<extra_4965>": 4965,
+  "<extra_4966>": 4966,
+  "<extra_4967>": 4967,
+  "<extra_4968>": 4968,
+  "<extra_4969>": 4969,
+  "<extra_4970>": 4970,
+  "<extra_4971>": 4971,
+  "<extra_4972>": 4972,
+  "<extra_4973>": 4973,
+  "<extra_4974>": 4974,
+  "<extra_4975>": 4975,
+  "<extra_4976>": 4976,
+  "<extra_4977>": 4977,
+  "<extra_4978>": 4978,
+  "<extra_4979>": 4979,
+  "<extra_4980>": 4980,
+  "<extra_4981>": 4981,
+  "<extra_4982>": 4982,
+  "<extra_4983>": 4983,
+  "<extra_4984>": 4984,
+  "<extra_4985>": 4985,
+  "<extra_4986>": 4986,
+  "<extra_4987>": 4987,
+  "<extra_4988>": 4988,
+  "<extra_4989>": 4989,
+  "<extra_4990>": 4990,
+  "<extra_4991>": 4991,
+  "<extra_4992>": 4992,
+  "<extra_4993>": 4993,
+  "<extra_4994>": 4994,
+  "<extra_4995>": 4995,
+  "<extra_4996>": 4996,
+  "<extra_4997>": 4997,
+  "<extra_4998>": 4998,
+  "<extra_4999>": 4999
+}

configs/special_tokens_map.json

@@ -0,0 +1,16 @@
+{
+  "pad_token": "<pad>",
+  "pad_token_id": 0,
+  "bos_token": "<s>",
+  "bos_token_id": 1,
+  "eos_token": "</s>",
+  "eos_token_id": 2,
+  "unk_token": "<unk>",
+  "unk_token_id": 3,
+  "_comment": {
+    "pad_token": "填充token，用于batch对齐",
+    "bos_token": "序列起始token",
+    "eos_token": "序列结束token，生成时遇到此token停止",
+    "unk_token": "未知token，词表外字符映射到此"
+  }
+}

include/neuroflow/adamw.hpp

@@ -0,0 +1,46 @@
+#ifndef NEUROFLOW_ADAMW_HPP
+#define NEUROFLOW_ADAMW_HPP
+
+#include <cstddef>
+#include <vector>
+#include "tensor.hpp"
+
+namespace neuroflow {
+
+struct ParamState {
+    Tensor m;
+    Tensor v;
+};
+
+struct ParamGroup {
+    std::vector<Tensor*> params;
+    std::vector<Tensor*> grads;
+    float lr;
+    float weight_decay;
+};
+
+class AdamW {
+public:
+    float lr_;
+    float beta1_;
+    float beta2_;
+    float eps_;
+    float weight_decay_;
+    size_t step_;
+
+    std::vector<ParamGroup> param_groups_;
+    std::vector<std::vector<ParamState>> states_;
+
+    AdamW(float lr, float beta1 = 0.9f, float beta2 = 0.999f,
+          float eps = 1e-8f, float weight_decay = 0.01f);
+
+    void add_param_group(const ParamGroup& group);
+    void step();
+    void set_lr(float lr);
+    float get_lr() const { return lr_; }
+    size_t get_step() const { return step_; }
+};
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_ADAMW_HPP

include/neuroflow/alignment_common.hpp

@@ -0,0 +1,320 @@
+#ifndef NEUROFLOW_ALIGNMENT_COMMON_HPP
+#define NEUROFLOW_ALIGNMENT_COMMON_HPP
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "causal_lm.hpp"
+#include "tensor.hpp"
+
+namespace neuroflow {
+
+struct SFTSample {
+    std::string instruction;
+    std::string response;
+};
+
+struct DPOSample {
+    std::string instruction;
+    std::string chosen;
+    std::string rejected;
+};
+
+struct SFTTrainingTensors {
+    std::vector<size_t> input_ids;
+    std::vector<size_t> target_ids;
+    std::vector<float> loss_mask;
+    size_t instruction_len;
+};
+
+struct DPOTrainingTensors {
+    std::vector<size_t> chosen_ids;
+    std::vector<size_t> rejected_ids;
+    size_t chosen_prompt_len;
+    size_t rejected_prompt_len;
+};
+
+inline std::string extract_json_string(const std::string& json, const std::string& key) {
+    std::string search = "\"" + key + "\"";
+    size_t pos = json.find(search);
+    if (pos == std::string::npos) return "";
+    pos = json.find(':', pos + search.size());
+    if (pos == std::string::npos) return "";
+    pos++;
+    while (pos < json.size() && (json[pos] == ' ' || json[pos] == '\t')) pos++;
+    if (pos >= json.size() || json[pos] != '"') return "";
+    size_t end = pos + 1;
+    while (end < json.size()) {
+        if (json[end] == '"' && json[end - 1] != '\\') break;
+        if (json[end - 1] == '\\') { end++; continue; }
+        end++;
+    }
+    if (end >= json.size()) return "";
+    return json.substr(pos + 1, end - pos - 1);
+}
+
+inline size_t extract_json_number(const std::string& json, const std::string& key, size_t default_val = 0) {
+    std::string search = "\"" + key + "\"";
+    size_t pos = json.find(search);
+    if (pos == std::string::npos) return default_val;
+    pos = json.find(':', pos + search.size());
+    if (pos == std::string::npos) return default_val;
+    pos++;
+    while (pos < json.size() && (json[pos] == ' ' || json[pos] == '\t')) pos++;
+    size_t end = pos;
+    while (end < json.size() && (json[end] >= '0' && json[end] <= '9')) end++;
+    if (end == pos) return default_val;
+    return std::stoul(json.substr(pos, end - pos));
+}
+
+inline bool extract_json_bool(const std::string& json, const std::string& key, bool default_val = false) {
+    std::string search = "\"" + key + "\"";
+    size_t pos = json.find(search);
+    if (pos == std::string::npos) return default_val;
+    pos = json.find(':', pos + search.size());
+    if (pos == std::string::npos) return default_val;
+    pos++;
+    while (pos < json.size() && (json[pos] == ' ' || json[pos] == '\t')) pos++;
+    if (pos + 3 < json.size() && json.substr(pos, 4) == "true") return true;
+    if (pos + 4 < json.size() && json.substr(pos, 5) == "false") return false;
+    return default_val;
+}
+
+inline void unescape_json(std::string& s) {
+    size_t pos = 0;
+    while ((pos = s.find('\\', pos)) != std::string::npos) {
+        if (pos + 1 < s.size()) {
+            char c = s[pos + 1];
+            if (c == 'n') { s.replace(pos, 2, "\n"); pos++; }
+            else if (c == 't') { s.replace(pos, 2, "\t"); pos++; }
+            else if (c == 'r') { s.replace(pos, 2, "\r"); pos++; }
+            else if (c == '"') { s.replace(pos, 2, "\""); pos++; }
+            else if (c == '\\') { s.replace(pos, 2, "\\"); pos++; }
+            else if (c == '/') { s.replace(pos, 2, "/"); pos++; }
+            else if (c == 'u' && pos + 5 < s.size()) { pos += 6; }
+            else { pos += 2; }
+        } else { pos++; }
+    }
+}
+
+inline bool validate_path(const std::string& path) {
+    if (path.find("..") != std::string::npos) return false;
+    return true;
+}
+
+inline void save_lm_checkpoint(const CausalLMHead& model, const std::string& path) {
+    CausalLMHead& lm_head = const_cast<CausalLMHead&>(model);
+#ifdef USE_CUDA
+    auto sync_to_cpu = [](const Tensor& t) {
+        if (t.is_on_gpu()) { const_cast<Tensor&>(t).to_cpu(); }
+    };
+    sync_to_cpu(lm_head.w_embed_);
+    sync_to_cpu(lm_head.w_pos_);
+    sync_to_cpu(lm_head.dw_kernel_);
+    sync_to_cpu(lm_head.pw_conv_->weight);
+    sync_to_cpu(lm_head.sae_w_encode_->weight);
+    sync_to_cpu(lm_head.sae_w_decode_->weight);
+    sync_to_cpu(lm_head.ntm_w_read_->weight);
+    sync_to_cpu(lm_head.ntm_w_write_->weight);
+    sync_to_cpu(lm_head.ntm_w_erase_->weight);
+    sync_to_cpu(lm_head.ntm_memory_);
+    sync_to_cpu(lm_head.w_proj_->weight);
+    sync_to_cpu(lm_head.w_proj_->bias);
+    if (lm_head.bridge_) {
+        sync_to_cpu(lm_head.bridge_->weight);
+        sync_to_cpu(lm_head.bridge_->bias);
+    }
+    sync_to_cpu(lm_head.w_out_->weight);
+    sync_to_cpu(lm_head.w_out_->bias);
+    sync_to_cpu(lm_head.ln_->weight);
+    sync_to_cpu(lm_head.ln_->bias);
+    for (auto& attn : lm_head.attn_layers_) {
+        sync_to_cpu(attn->w_q->weight);
+        sync_to_cpu(attn->w_q->bias);
+        sync_to_cpu(attn->w_k->weight);
+        sync_to_cpu(attn->w_k->bias);
+        sync_to_cpu(attn->w_v->weight);
+        sync_to_cpu(attn->w_v->bias);
+        sync_to_cpu(attn->w_out->weight);
+        sync_to_cpu(attn->w_out->bias);
+        sync_to_cpu(attn->norm->weight);
+        sync_to_cpu(attn->norm->bias);
+    }
+#endif
+    auto sl = [](std::ofstream& o, const std::string& n, const Tensor& t) {
+        uint32_t nl = n.size(); o.write((char*)&nl, 4); o.write(n.data(), nl);
+        uint32_t nd = t.shape_.size(); o.write((char*)&nd, 4);
+        for (auto d : t.shape_) { uint32_t dd = d; o.write((char*)&dd, 4); }
+        uint32_t ds = t.data_size_; o.write((char*)&ds, 4);
+        o.write((char*)t.data_.get(), ds);
+    };
+    std::ofstream o(path, std::ios::binary);
+    o.write("LMH2", 4);
+    sl(o, "w_embed", lm_head.w_embed_);
+    sl(o, "w_pos", lm_head.w_pos_);
+    sl(o, "dw_kernel", lm_head.dw_kernel_);
+    sl(o, "pw_conv.weight", lm_head.pw_conv_->weight);
+    sl(o, "sae_encode.weight", lm_head.sae_w_encode_->weight);
+    sl(o, "sae_decode.weight", lm_head.sae_w_decode_->weight);
+    sl(o, "ntm_read.weight", lm_head.ntm_w_read_->weight);
+    sl(o, "ntm_write.weight", lm_head.ntm_w_write_->weight);
+    sl(o, "ntm_erase.weight", lm_head.ntm_w_erase_->weight);
+    sl(o, "ntm_memory", lm_head.ntm_memory_);
+    sl(o, "w_proj.weight", lm_head.w_proj_->weight);
+    sl(o, "w_proj.bias", lm_head.w_proj_->bias);
+    if (lm_head.bridge_) {
+        sl(o, "bridge.weight", lm_head.bridge_->weight);
+        sl(o, "bridge.bias", lm_head.bridge_->bias);
+    }
+    sl(o, "w_out.weight", lm_head.w_out_->weight);
+    if (lm_head.w_out_->bias.data_) sl(o, "w_out.bias", lm_head.w_out_->bias);
+    sl(o, "ln.weight", lm_head.ln_->weight);
+    sl(o, "ln.bias", lm_head.ln_->bias);
+    for (size_t i = 0; i < lm_head.attn_layers_.size(); ++i) {
+        std::string p = "attn" + std::to_string(i) + ".";
+        sl(o, p + "w_q.weight", lm_head.attn_layers_[i]->w_q->weight);
+        sl(o, p + "w_q.bias", lm_head.attn_layers_[i]->w_q->bias);
+        sl(o, p + "w_k.weight", lm_head.attn_layers_[i]->w_k->weight);
+        sl(o, p + "w_k.bias", lm_head.attn_layers_[i]->w_k->bias);
+        sl(o, p + "w_v.weight", lm_head.attn_layers_[i]->w_v->weight);
+        sl(o, p + "w_v.bias", lm_head.attn_layers_[i]->w_v->bias);
+        sl(o, p + "w_out.weight", lm_head.attn_layers_[i]->w_out->weight);
+        sl(o, p + "w_out.bias", lm_head.attn_layers_[i]->w_out->bias);
+        sl(o, p + "norm.weight", lm_head.attn_layers_[i]->norm->weight);
+        sl(o, p + "norm.bias", lm_head.attn_layers_[i]->norm->bias);
+    }
+    uint32_t z = 0; o.write((char*)&z, 4); o.close();
+}
+
+inline bool load_lm_checkpoint(CausalLMHead& lm_head, const std::string& lm_path) {
+    std::ifstream ifs(lm_path, std::ios::binary);
+    if (!ifs) {
+        std::cerr << "LM Head checkpoint未找到: " << lm_path << std::endl;
+        return false;
+    }
+
+    char magic[5] = {0};
+    ifs.read(magic, 4);
+    if (std::string(magic) != "LMH2" && std::string(magic) != "LMH1") {
+        std::cerr << "LM Head格式不匹配: " << std::string(magic) << " (期望LMH2/LMH1)" << std::endl;
+        ifs.close();
+        return false;
+    }
+
+    auto read_named_tensor = [&]() -> std::pair<std::string, Tensor> {
+        uint32_t nl = 0; ifs.read((char*)&nl, 4);
+        if (nl == 0 || ifs.eof()) return {"", Tensor()};
+        std::string name(nl, '\0'); ifs.read(&name[0], nl);
+        uint32_t nd = 0; ifs.read((char*)&nd, 4);
+        std::vector<size_t> shape(nd);
+        for (size_t i = 0; i < nd; ++i) { uint32_t d = 0; ifs.read((char*)&d, 4); shape[i] = d; }
+        uint32_t ds = 0; ifs.read((char*)&ds, 4);
+        Tensor t(shape, QuantType::FP32);
+        if (t.data_size_ == ds) {
+            ifs.read((char*)t.data_.get(), ds);
+        } else {
+            ifs.seekg(ds, std::ios::cur);
+            t = Tensor();
+        }
+        return {name, std::move(t)};
+    };
+
+    std::unordered_map<std::string, Tensor*> tensor_map;
+    tensor_map["w_embed"] = &lm_head.w_embed_;
+    tensor_map["w_pos"] = &lm_head.w_pos_;
+    tensor_map["dw_kernel"] = &lm_head.dw_kernel_;
+    tensor_map["pw_conv.weight"] = &lm_head.pw_conv_->weight;
+    tensor_map["sae_encode.weight"] = &lm_head.sae_w_encode_->weight;
+    tensor_map["sae_decode.weight"] = &lm_head.sae_w_decode_->weight;
+    tensor_map["ntm_read.weight"] = &lm_head.ntm_w_read_->weight;
+    tensor_map["ntm_write.weight"] = &lm_head.ntm_w_write_->weight;
+    tensor_map["ntm_erase.weight"] = &lm_head.ntm_w_erase_->weight;
+    tensor_map["ntm_memory"] = &lm_head.ntm_memory_;
+    tensor_map["w_proj.weight"] = &lm_head.w_proj_->weight;
+    tensor_map["w_proj.bias"] = &lm_head.w_proj_->bias;
+    if (lm_head.bridge_) {
+        tensor_map["bridge.weight"] = &lm_head.bridge_->weight;
+        tensor_map["bridge.bias"] = &lm_head.bridge_->bias;
+    }
+    tensor_map["w_out.weight"] = &lm_head.w_out_->weight;
+    tensor_map["w_out.bias"] = &lm_head.w_out_->bias;
+    tensor_map["ln.weight"] = &lm_head.ln_->weight;
+    tensor_map["ln.bias"] = &lm_head.ln_->bias;
+    for (size_t i = 0; i < lm_head.attn_layers_.size(); ++i) {
+        std::string p = "attn" + std::to_string(i) + ".";
+        tensor_map[p + "w_q.weight"] = &lm_head.attn_layers_[i]->w_q->weight;
+        tensor_map[p + "w_q.bias"] = &lm_head.attn_layers_[i]->w_q->bias;
+        tensor_map[p + "w_k.weight"] = &lm_head.attn_layers_[i]->w_k->weight;
+        tensor_map[p + "w_k.bias"] = &lm_head.attn_layers_[i]->w_k->bias;
+        tensor_map[p + "w_v.weight"] = &lm_head.attn_layers_[i]->w_v->weight;
+        tensor_map[p + "w_v.bias"] = &lm_head.attn_layers_[i]->w_v->bias;
+        tensor_map[p + "w_out.weight"] = &lm_head.attn_layers_[i]->w_out->weight;
+        tensor_map[p + "w_out.bias"] = &lm_head.attn_layers_[i]->w_out->bias;
+        tensor_map[p + "norm.weight"] = &lm_head.attn_layers_[i]->norm->weight;
+        tensor_map[p + "norm.bias"] = &lm_head.attn_layers_[i]->norm->bias;
+    }
+
+    size_t loaded = 0;
+    while (ifs) {
+        auto [name, tensor] = read_named_tensor();
+        if (name.empty()) break;
+        auto it = tensor_map.find(name);
+        if (it != tensor_map.end() && tensor.numel() > 0) {
+            if (it->second->shape_ == tensor.shape_) {
+                memcpy(it->second->data_.get(), tensor.data_.get(), tensor.data_size_);
+                loaded++;
+            } else {
+                std::cerr << "  跳过 '" << name << "': 形状不匹配" << std::endl;
+            }
+        } else if (name.find("w_qkv.") != std::string::npos) {
+            std::string base = name.substr(0, name.find("w_qkv."));
+            std::string suffix = name.substr(name.find("w_qkv.") + 6);
+            for (size_t i = 0; i < lm_head.attn_layers_.size(); ++i) {
+                std::string p = "attn" + std::to_string(i) + ".";
+                if (base != p) continue;
+                size_t d_model = lm_head.config_.d_model;
+                size_t head_dim = d_model / lm_head.config_.num_attn_heads;
+                size_t n_q = lm_head.attn_layers_[i]->n_q_heads_;
+                size_t n_kv = lm_head.attn_layers_[i]->n_kv_heads_;
+                if (suffix == "weight" && tensor.shape_.size() == 2 && tensor.shape_[0] == 3 * d_model) {
+                    const float* src = tensor.as_fp32();
+                    float* dq = lm_head.attn_layers_[i]->w_q->weight.as_fp32();
+                    float* dk = lm_head.attn_layers_[i]->w_k->weight.as_fp32();
+                    float* dv = lm_head.attn_layers_[i]->w_v->weight.as_fp32();
+                    for (size_t r = 0; r < d_model; ++r) {
+                        memcpy(dq + r * n_q * head_dim, src + r * 3 * d_model, n_q * head_dim * sizeof(float));
+                        memcpy(dk + r * n_kv * head_dim, src + r * 3 * d_model + d_model, n_kv * head_dim * sizeof(float));
+                        memcpy(dv + r * n_kv * head_dim, src + r * 3 * d_model + 2 * d_model, n_kv * head_dim * sizeof(float));
+                    }
+                    loaded++;
+                    std::cerr << "  拆分旧格式 '" << name << "' -> w_q/w_k/w_v" << std::endl;
+                } else if (suffix == "bias" && tensor.shape_[0] == 3 * d_model) {
+                    const float* src = tensor.as_fp32();
+                    float* bq = lm_head.attn_layers_[i]->w_q->bias.as_fp32();
+                    float* bk = lm_head.attn_layers_[i]->w_k->bias.as_fp32();
+                    float* bv = lm_head.attn_layers_[i]->w_v->bias.as_fp32();
+                    memcpy(bq, src, n_q * head_dim * sizeof(float));
+                    memcpy(bk, src + d_model, n_kv * head_dim * sizeof(float));
+                    memcpy(bv, src + 2 * d_model, n_kv * head_dim * sizeof(float));
+                    loaded++;
+                    std::cerr << "  拆分旧格式 '" << name << "' -> w_q/w_k/w_v bias" << std::endl;
+                }
+            }
+        }
+    }
+
+    ifs.close();
+    if (lm_head.config_.weight_tying) lm_head.tie_weights();
+    std::cerr << "LM Head已恢复: " << loaded << " 个张量从 " << lm_path << std::endl;
+    return loaded > 0;
+}
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_ALIGNMENT_COMMON_HPP

include/neuroflow/backprop.hpp

@@ -0,0 +1,609 @@
+/**
+ * NeuroFlow 完整反向传播实现
+ * 
+ * 实现完整模型的梯度反向传播链：
+ * - Output fusion -> SN -> ECN/DMN -> Memory -> Input projection
+ */
+
+#pragma once
+
+// C++ 标准库
+#include <cmath>
+#include <memory>
+#include <unordered_map>
+#include <vector>
+
+// 第三方库
+#ifdef _OPENMP
+#include <omp.h>
+#endif
+
+// 项目头文件
+#include "memory.hpp"
+#include "model.hpp"
+#include "networks.hpp"
+#include "tensor.hpp"
+
+namespace neuroflow {
+
+/**
+ * 完整反向传播引擎
+ */
+class BackpropEngine {
+public:
+    // 缓存前向传播中间结果
+    struct ForwardCache {
+        Tensor input;
+        Tensor h;  // 输入投影后
+        Tensor h_normed;
+        Tensor h_gelu;
+        
+        // SN中间结果
+        Tensor sn_h1, sn_h2;
+        Tensor sn_gate_h;
+        
+        // ECN中间结果
+        std::vector<Tensor> ecn_hidden;
+        Tensor ecn_ofc_v;
+        Tensor ecn_vmpfc_d;
+        
+        // DMN中间结果
+        Tensor dmn_encoded;
+        Tensor dmn_latent;
+        std::vector<Tensor> dmn_associations;
+        Tensor dmn_vision;
+        
+        // Memory结果
+        Tensor memory_encoded;
+        Tensor memory_query;
+        Tensor memory_retrieved;
+        
+        // Output fusion
+        Tensor combined;
+        Tensor fused_pre_norm;
+        Tensor fused_bn;
+        Tensor fused_bn_pre_relu;
+        Tensor fused;
+    };
+    
+    ForwardCache cache;
+    NeuroFlowModel& model;
+    
+    BackpropEngine(NeuroFlowModel& m) : model(m) {}
+    
+    // 前向传播（带缓存）
+    NeuroFlowModel::Output forward_with_cache(const Tensor& x) {
+        cache.input = x.clone();
+        
+        size_t batch = x.shape_[0];
+        
+        // Input projection
+        cache.h = model.input_proj_linear->forward(x);
+        cache.h_normed = model.input_proj_norm->forward(cache.h);
+        cache.h_gelu = model.input_proj_gelu->forward(cache.h_normed);
+        
+        // SN
+        auto sn_out = model.sn->forward(cache.h_gelu);
+        
+        // ECN
+        Tensor h_for_ecn = cache.h_gelu.clone();
+        for (size_t i = 0; i < model.ecn->num_layers; ++i) {
+            h_for_ecn = model.ecn->dlpfc_linear[i]->forward(h_for_ecn);
+            h_for_ecn = model.ecn->dlpfc_norm[i]->forward(h_for_ecn);
+            h_for_ecn = model.ecn->dlpfc_gelu[i]->forward(h_for_ecn);
+            cache.ecn_hidden.push_back(h_for_ecn.clone());
+        }
+        
+        // OFC
+        cache.ecn_ofc_v = model.ecn->ofc1->forward(cache.ecn_hidden.back());
+        TensorOps::gelu(cache.ecn_ofc_v);
+        
+        // vmPFC
+        cache.ecn_vmpfc_d = model.ecn->vmpfc1->forward(cache.ecn_hidden.back());
+        TensorOps::gelu(cache.ecn_vmpfc_d);
+        
+        // DMN
+        cache.memory_encoded = model.memory->encode(cache.h_gelu);
+        cache.dmn_encoded = model.dmn->mem_encoder1->forward(cache.memory_encoded);
+        TensorOps::gelu(cache.dmn_encoded);
+        cache.dmn_latent = model.dmn->mem_encoder2->forward(cache.dmn_encoded);
+        
+        for (auto& [h1, h2] : model.dmn->association_heads) {
+            Tensor assoc = h1->forward(cache.dmn_latent);
+            TensorOps::gelu(assoc);
+            assoc = h2->forward(assoc);
+            cache.dmn_associations.push_back(assoc.clone());
+        }
+        
+        cache.dmn_vision = TensorOps::concat(cache.dmn_associations, 1);
+        cache.dmn_vision = model.dmn->future_proj1->forward(cache.dmn_vision);
+        cache.dmn_vision = model.dmn->future_norm->forward(cache.dmn_vision);
+        cache.dmn_vision = model.dmn->future_gelu->forward(cache.dmn_vision);
+        
+        // Memory retrieval
+        cache.memory_query = model.memory->query_proj->forward(cache.h_gelu);
+        auto mem_out = model.memory->forward(cache.h_gelu);
+        cache.memory_retrieved = mem_out.retrieved;
+        
+        // Output fusion
+        NeuroFlowModel::Output out;
+        out.saliency = sn_out.saliency;
+        out.gates = sn_out.gates;
+        out.anomaly = sn_out.anomaly;
+        
+        // 提取门控
+        const float* gates = out.gates.as_fp32();
+        Tensor ecn_gate({batch, 1}, QuantType::FP32);
+        Tensor dmn_gate({batch, 1}, QuantType::FP32);
+        for (size_t i = 0; i < batch; ++i) {
+            ecn_gate.as_fp32()[i] = gates[i * 2];
+            dmn_gate.as_fp32()[i] = gates[i * 2 + 1];
+        }
+        out.ecn_gate = ecn_gate;
+        out.dmn_gate = dmn_gate;
+        
+        // ECN决策
+        Tensor ecn_decision = model.ecn->vmpfc2->forward(cache.ecn_vmpfc_d);
+        out.decision = ecn_decision;
+        out.value = model.ecn->ofc2->forward(cache.ecn_ofc_v);
+        
+        // 加权融合
+        Tensor ecn_weighted({batch, model.config.output_dim}, QuantType::FP32);
+        Tensor dmn_weighted({batch, model.config.output_dim}, QuantType::FP32);
+        Tensor mem_weighted({batch, model.config.output_dim}, QuantType::FP32);
+        
+        float* ew = ecn_weighted.as_fp32();
+        float* dw = dmn_weighted.as_fp32();
+        float* mw = mem_weighted.as_fp32();
+        const float* ed = out.decision.as_fp32();
+        const float* dv = cache.dmn_vision.as_fp32();
+        const float* mr = cache.memory_retrieved.as_fp32();
+        const float* eg = ecn_gate.as_fp32();
+        const float* dg = dmn_gate.as_fp32();
+        
+        for (size_t i = 0; i < batch; ++i) {
+            for (size_t j = 0; j < model.config.output_dim; ++j) {
+                ew[i * model.config.output_dim + j] = ed[i * model.config.output_dim + j] * eg[i];
+                if (j < cache.dmn_vision.shape_[1]) {
+                    dw[i * model.config.output_dim + j] = dv[i * cache.dmn_vision.shape_[1] + j] * dg[i];
+                }
+                if (j < cache.memory_retrieved.shape_[1]) {
+                    mw[i * model.config.output_dim + j] = mr[i * cache.memory_retrieved.shape_[1] + j];
+                }
+            }
+        }
+        
+        std::vector<Tensor> to_concat = {ecn_weighted, dmn_weighted, mem_weighted};
+        cache.combined = TensorOps::concat(to_concat, 1);
+        
+        cache.fused_bn = model.output_fusion_down->forward(cache.combined);
+        cache.fused_bn = model.output_fusion_bottleneck_norm->forward(cache.fused_bn);
+        cache.fused_bn_pre_relu = cache.fused_bn.clone();
+        TensorOps::relu(cache.fused_bn);
+        cache.fused_pre_norm = model.output_fusion_up->forward(cache.fused_bn);
+        cache.fused = model.output_fusion_norm->forward(cache.fused_pre_norm);
+        out.output = cache.fused;
+        
+        return out;
+    }
+    
+    // 完整反向传播
+    struct Gradients {
+        // Input projection
+        Tensor input_proj_weight_grad;
+        Tensor input_proj_bias_grad;
+        Tensor input_proj_norm_weight_grad;
+        Tensor input_proj_norm_bias_grad;
+        
+        // SN
+        Tensor sn_saliency_grads;
+        Tensor sn_gate_grads;
+        
+        // ECN
+        std::vector<Tensor> ecn_weight_grads;
+        std::vector<Tensor> ecn_bias_grads;
+        Tensor ecn_ofc_grads;
+        Tensor ecn_vmpfc_grads;
+        
+        // DMN
+        Tensor dmn_encoder_grads;
+        std::vector<Tensor> dmn_association_grads;
+        
+        // Memory
+        Tensor memory_encode_grads;
+        Tensor memory_query_grads;
+        Tensor memory_retrieve_grads;
+        
+        // Output fusion
+        Tensor output_fusion_down_weight_grad;
+        Tensor output_fusion_down_bias_grad;
+        Tensor output_fusion_up_weight_grad;
+        Tensor output_fusion_up_bias_grad;
+
+        Tensor output_fusion_norm_grads;
+        
+        // Input gradient (用于传播到上游)
+        Tensor input_grad;
+    };
+    
+    Gradients backward(const Tensor& output_grad) {
+        Gradients grads;
+        size_t batch = output_grad.shape_[0];
+        
+        // ===== Output fusion backward =====
+        // LayerNorm backward
+        Tensor norm_grad = layernorm_backward(cache.fused, output_grad);
+        
+        // Linear backward (output_fusion)
+        grads.output_fusion_up_weight_grad = Tensor(
+            {model.output_fusion_up->weight.shape_[0], 
+             model.output_fusion_up->weight.shape_[1]}, QuantType::FP32);
+        
+        // 计算权重梯度: output_grad.T @ combined
+        // 这里简化处理，使用近似
+        const float* og = output_grad.as_fp32();
+        const float* cb = cache.combined.as_fp32();
+        float* wg = grads.output_fusion_up_weight_grad.as_fp32();
+        
+        size_t out_dim = model.config.output_dim;
+        size_t combined_dim = cache.combined.shape_[1];
+        
+#ifdef USE_CBLAS
+        cblas_sgemm(CblasRowMajor, CblasTrans, CblasNoTrans,
+                    out_dim, combined_dim, batch,
+                    1.0f / batch, og, out_dim, cb, combined_dim,
+                    0.0f, wg, combined_dim);
+#else
+        for (size_t i = 0; i < out_dim; ++i) {
+            for (size_t j = 0; j < combined_dim; ++j) {
+                float sum = 0;
+                for (size_t b = 0; b < batch; ++b) {
+                    sum += og[b * out_dim + i] * cb[b * combined_dim + j];
+                }
+                wg[i * combined_dim + j] = sum / batch;
+            }
+        }
+#endif
+        
+        // Input gradient for combined
+        Tensor combined_grad({batch, combined_dim}, QuantType::FP32);
+        const float* ow = model.output_fusion_up->weight.as_fp32();
+        float* cg = combined_grad.as_fp32();
+        
+#ifdef USE_CBLAS
+        cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans,
+                    batch, combined_dim, out_dim,
+                    1.0f, og, out_dim, ow, combined_dim,
+                    0.0f, cg, combined_dim);
+#else
+        for (size_t b = 0; b < batch; ++b) {
+            for (size_t j = 0; j < combined_dim; ++j) {
+                float sum = 0;
+                for (size_t i = 0; i < out_dim; ++i) {
+                    sum += og[b * out_dim + i] * ow[i * combined_dim + j];
+                }
+                cg[b * combined_dim + j] = sum;
+            }
+        }
+#endif
+        
+        // ===== Split gradients =====
+        // combined = [ecn_weighted, dmn_weighted, mem_weighted]
+        Tensor ecn_grad({batch, model.config.output_dim}, QuantType::FP32);
+        Tensor dmn_grad({batch, model.config.output_dim}, QuantType::FP32);
+        Tensor mem_grad({batch, model.config.output_dim}, QuantType::FP32);
+        
+        float* eg = ecn_grad.as_fp32();
+        float* dg = dmn_grad.as_fp32();
+        float* mg = mem_grad.as_fp32();
+        
+        for (size_t b = 0; b < batch; ++b) {
+            for (size_t j = 0; j < model.config.output_dim; ++j) {
+                eg[b * model.config.output_dim + j] = cg[b * combined_dim + j];
+                dg[b * model.config.output_dim + j] = cg[b * combined_dim + model.config.output_dim + j];
+                mg[b * model.config.output_dim + j] = cg[b * combined_dim + 2 * model.config.output_dim + j];
+            }
+        }
+        
+        // ===== ECN backward =====
+        // vmpfc2 backward
+        // ... (简化，继续传播)
+        
+        // ===== Input projection backward =====
+        // 传播到输入
+        grads.input_grad = Tensor({batch, model.config.input_dim}, QuantType::FP32);
+        
+        // 简化：假设所有梯度汇集到输入
+        float* ig = grads.input_grad.as_fp32();
+        for (size_t b = 0; b < batch; ++b) {
+            float total_grad = 0;
+            for (size_t j = 0; j < model.config.output_dim; ++j) {
+                total_grad += og[b * model.config.output_dim + j];
+            }
+            for (size_t j = 0; j < model.config.input_dim; ++j) {
+                ig[b * model.config.input_dim + j] = total_grad / model.config.output_dim * 
+                    cache.input.as_fp32()[b * model.config.input_dim + j] * 0.01f;
+            }
+        }
+        
+        return grads;
+    }
+    
+private:
+    // LayerNorm backward
+    Tensor layernorm_backward(const Tensor& input, const Tensor& output_grad, float eps = 1e-5f) {
+        size_t batch = input.shape_[0];
+        size_t dim = input.shape_[1];
+        
+        Tensor input_grad({batch, dim}, QuantType::FP32);
+        
+        const float* inp = input.as_fp32();
+        const float* og = output_grad.as_fp32();
+        float* ig = input_grad.as_fp32();
+        
+        #pragma omp parallel for
+        for (long long b = 0; b < static_cast<long long>(batch); ++b) {
+            float mean = 0.0f;
+            for (size_t d = 0; d < dim; ++d) {
+                mean += inp[b * dim + d];
+            }
+            mean /= dim;
+            
+            float var = 0.0f;
+            for (size_t d = 0; d < dim; ++d) {
+                float diff = inp[b * dim + d] - mean;
+                var += diff * diff;
+            }
+            var /= dim;
+            float std = std::sqrt(var + eps);
+            
+            float sum_grad = 0.0f;
+            float sum_grad_x = 0.0f;
+            for (size_t d = 0; d < dim; ++d) {
+                float normalized = (inp[b * dim + d] - mean) / std;
+                sum_grad += og[b * dim + d];
+                sum_grad_x += og[b * dim + d] * normalized;
+            }
+            
+            for (size_t d = 0; d < dim; ++d) {
+                float normalized = (inp[b * dim + d] - mean) / std;
+                ig[b * dim + d] = (og[b * dim + d] - sum_grad / dim - normalized * sum_grad_x / dim) / std;
+            }
+        }
+        
+        return input_grad;
+    }
+};
+
+/**
+ * 训练器 - 完整训练流程
+ */
+class Trainer {
+public:
+    NeuroFlowModel& model;
+    float learning_rate;
+    BackpropEngine backprop;
+    
+    Trainer(NeuroFlowModel& m, float lr = 0.001f)
+        : model(m), learning_rate(lr), backprop(m) {}
+    
+    // 单步训练
+    struct TrainStep {
+        float loss;
+        float grad_norm;
+    };
+    
+    TrainStep train_step(const Tensor& input, const Tensor& target) {
+        TrainStep result;
+        
+        // Forward with cache
+        auto output = backprop.forward_with_cache(input);
+        
+        // Compute loss (MSE inline)
+        float loss = 0.0f;
+        const float* pred_p = output.output.as_fp32();
+        const float* tgt_p = target.as_fp32();
+        size_t n = output.output.numel();
+        for (size_t i = 0; i < n; ++i) {
+            float diff = pred_p[i] - tgt_p[i];
+            loss += diff * diff;
+        }
+        result.loss = loss / n;
+        
+        // Compute output gradient
+        Tensor output_grad({input.shape_[0], target.shape_[1]}, QuantType::FP32);
+        const float* pred = output.output.as_fp32();
+        const float* tgt = target.as_fp32();
+        float* og = output_grad.as_fp32();
+        
+        float scale = 2.0f / output.output.numel();
+        float grad_norm = 0.0f;
+        for (size_t i = 0; i < output.output.numel(); ++i) {
+            og[i] = scale * (pred[i] - tgt[i]);
+            grad_norm += og[i] * og[i];
+        }
+        result.grad_norm = std::sqrt(grad_norm);
+        
+        // Backward
+        auto grads = backprop.backward(output_grad);
+        
+        // Update weights (SGD inline)
+        auto sgd_step = [&](Tensor& param, const Tensor& grad) {
+            float* p = param.as_fp32();
+            const float* g = grad.as_fp32();
+            size_t sz = param.numel();
+            for (size_t i = 0; i < sz; ++i) p[i] -= learning_rate * g[i];
+        };
+        sgd_step(model.output_fusion_up->weight, grads.output_fusion_up_weight_grad);
+        
+        // Memory consolidation
+        model.memory->consolidate(input);
+        
+        return result;
+    }
+    
+    // 训练循环
+    std::vector<float> train(const std::vector<Tensor>& inputs,
+                              const std::vector<Tensor>& targets,
+                              int epochs = 10) {
+        std::vector<float> losses;
+        
+        for (int e = 0; e < epochs; ++e) {
+            float epoch_loss = 0.0f;
+            
+            for (size_t i = 0; i < inputs.size(); ++i) {
+                auto step = train_step(inputs[i], targets[i]);
+                epoch_loss += step.loss;
+            }
+            
+            epoch_loss /= inputs.size();
+            losses.push_back(epoch_loss);
+        }
+        
+        return losses;
+    }
+};
+
+/**
+ * FullBackpropEngine - 完整全链路反向传播引擎
+ * 
+ * 支持所有层的梯度计算和参数更新：
+ * - Output fusion (Linear + LayerNorm)
+ * - ECN (dlPFC多层 + OFC + vmPFC)
+ * - SN门控梯度传播
+ * - Input projection (Linear + LayerNorm + GELU)
+ */
+class FullBackpropEngine {
+public:
+    struct ForwardCache {
+        Tensor input;
+        Tensor input_proj_pre;
+        Tensor input_proj_post;
+        Tensor h;
+
+        Tensor sn_gates;
+        Tensor sn_gate_h;       // gate1 pre-gelu 输出 (修复: gelu_backward需要pre-gelu)
+        Tensor sn_gate_h_post;  // gate1 post-gelu 输出 (gate2 forward用)
+
+        std::vector<Tensor> ecn_pre_linear;
+        std::vector<Tensor> ecn_pre_norm;
+        std::vector<Tensor> ecn_hidden;
+        Tensor ecn_vmpfc_pre;
+        Tensor ecn_vmpfc_d;
+        Tensor ecn_decision;  // vmpfc2 output [batch, output_dim=2048] (修复: SN gate梯度用)
+        Tensor ecn_ofc_pre;
+        Tensor ecn_ofc_v;
+
+        Tensor memory_encoded;
+        Tensor dmn_encoded;
+        Tensor dmn_latent;
+        std::vector<Tensor> dmn_associations;
+        Tensor dmn_vision;
+        std::vector<Tensor> dmn_head1_outs;  // head1 gelu outputs (修复: 用于head2反向传播)
+        Tensor memory_retrieved;
+
+        Tensor combined;
+        Tensor fused_pre_norm;
+        Tensor fused_bn;
+        Tensor fused_bn_pre_relu;
+        Tensor fused_bn_pre_norm;  // pre-norm input for bottleneck norm
+        Tensor fused;
+    };
+
+    ForwardCache cache;
+    NeuroFlowModel& model;
+
+    FullBackpropEngine(NeuroFlowModel& m);
+
+    NeuroFlowModel::Output forward_with_cache(const Tensor& x);
+
+    struct Gradients {
+        Tensor input_proj_weight_grad;
+        Tensor input_proj_bias_grad;
+        Tensor input_proj_norm_weight_grad;
+        Tensor input_proj_norm_bias_grad;
+
+        Tensor output_fusion_down_weight_grad;
+        Tensor output_fusion_down_bias_grad;
+        Tensor output_fusion_up_weight_grad;
+        Tensor output_fusion_up_bias_grad;
+        Tensor output_fusion_norm_weight_grad;
+        Tensor output_fusion_norm_bias_grad;
+        Tensor output_fusion_bottleneck_norm_weight_grad;
+        Tensor output_fusion_bottleneck_norm_bias_grad;
+
+        Tensor ecn_vmpfc2_weight_grad;
+        Tensor ecn_vmpfc2_bias_grad;
+        Tensor ecn_vmpfc1_weight_grad;
+        Tensor ecn_vmpfc1_bias_grad;
+
+        std::vector<Tensor> ecn_dlpfc_weight_grads;
+        std::vector<Tensor> ecn_dlpfc_bias_grads;
+        std::vector<Tensor> ecn_dlpfc_norm_weight_grads;
+        std::vector<Tensor> ecn_dlpfc_norm_bias_grads;
+
+        // SN gate gradients (修复: gate1/gate2 之前没有梯度)
+        Tensor sn_gate2_weight_grad;
+        Tensor sn_gate2_bias_grad;
+        Tensor sn_gate1_weight_grad;
+        Tensor sn_gate1_bias_grad;
+
+        // DMN gradients (修复: DMN 之前没有梯度)
+        Tensor dmn_future_proj1_weight_grad;
+        Tensor dmn_future_proj1_bias_grad;
+        std::vector<Tensor> dmn_head2_weight_grads;
+        std::vector<Tensor> dmn_head2_bias_grads;
+        std::vector<Tensor> dmn_head1_weight_grads;
+        std::vector<Tensor> dmn_head1_bias_grads;
+        Tensor dmn_mem_encoder2_weight_grad;
+        Tensor dmn_mem_encoder2_bias_grad;
+        Tensor dmn_mem_encoder1_weight_grad;
+        Tensor dmn_mem_encoder1_bias_grad;
+
+        // Memory gradients (修复: memory encode/query 之前没有梯度)
+        Tensor mem_encode_proj_weight_grad;
+        Tensor mem_encode_proj_bias_grad;
+        Tensor mem_query_proj_weight_grad;
+        Tensor mem_query_proj_bias_grad;
+
+        Tensor input_grad;
+    };
+
+    Gradients backward(const Tensor& output_grad);
+};
+
+/**
+ * FullTrainer - 全链路训练器
+ * 
+ * 使用FullBackpropEngine进行完整的梯度计算和参数更新
+ */
+class FullTrainer {
+public:
+    NeuroFlowModel& model;
+    float learning_rate;
+    FullBackpropEngine backprop;
+
+    FullTrainer(NeuroFlowModel& m, float lr = 0.001f);
+
+    struct TrainStep {
+        float loss;
+        float grad_norm;
+    };
+
+    TrainStep train_step(const Tensor& input, const Tensor& target);
+    std::vector<float> train(const std::vector<Tensor>& inputs,
+                              const std::vector<Tensor>& targets,
+                              int epochs = 10);
+
+    TrainStep accumulate_step(const Tensor& input, const Tensor& target);
+    void apply_accumulated_gradients(int accum_steps);
+
+private:
+    void apply_gradients(FullBackpropEngine::Gradients& grads, float lr);
+    FullBackpropEngine::Gradients accum_grads_;
+    bool accum_initialized_ = false;
+    float accum_loss_ = 0.0f;
+    float accum_grad_norm_ = 0.0f;
+};
+
+} // namespace neuroflow

include/neuroflow/causal_lm.hpp

@@ -0,0 +1,253 @@
+#ifndef NEUROFLOW_CAUSAL_LM_HPP
+#define NEUROFLOW_CAUSAL_LM_HPP
+
+#include <memory>
+#include <string>
+#include <vector>
+#include "memory.hpp"
+#include "model.hpp"
+#include "networks.hpp"
+#include "tensor.hpp"
+#include "rope.hpp"
+#include "swiglu.hpp"
+#include "rms_norm.hpp"
+#include "adamw.hpp"
+
+#ifdef USE_CUDA
+#include "cuda_kernels.hpp"
+#endif
+
+namespace neuroflow {
+
+enum class FinishReason : uint8_t {
+    EOS_TOKEN = 0,
+    MAX_LENGTH = 1,
+    GEN_ERROR = 2
+};
+
+struct CausalLMConfig {
+    size_t vocab_size = 128000;
+    size_t d_model = 512;
+    size_t max_seq_len = 512;
+    size_t causal_window_size = 32;
+    size_t sae_k = 128;
+    size_t ntm_memory_slots = 32;
+    bool use_mla = true;
+    size_t mla_latent_dim = 64;
+    size_t mla_n_heads = 8;
+    size_t mla_max_cache_len = 4096;
+    bool use_quantization = false;
+    bool weight_tying = true;
+    bool use_rope = true;
+    bool use_bridge = true;
+    bool use_swiglu = true;
+    size_t swiglu_intermediate_size = 0;
+    bool use_qk_norm = true;
+    size_t num_attn_layers = 4;
+    size_t num_attn_heads = 8;
+    size_t n_kv_heads = 2;
+    int padding_id = -1;
+    std::string pooling = "last";
+};
+
+struct CacheStats {
+    size_t cache_len = 0;
+    size_t memory_bytes = 0;
+    float saving_ratio = 0.0f;
+    size_t sliding_window_drops = 0;
+};
+
+struct GenerateOutput {
+    std::string text;
+    std::vector<size_t> token_ids;
+    std::vector<Tensor> logits_history;
+    FinishReason finish_reason = FinishReason::MAX_LENGTH;
+    CacheStats cache_stats;
+};
+
+class CausalSelfAttention {
+public:
+    size_t n_q_heads_;
+    size_t n_kv_heads_;
+    size_t n_rep_;
+    size_t d_model_;
+    size_t head_dim_;
+
+    std::shared_ptr<Linear> w_q;
+    std::shared_ptr<Linear> w_k;
+    std::shared_ptr<Linear> w_v;
+    std::shared_ptr<Linear> w_out;
+    std::shared_ptr<LayerNorm> norm;
+    std::unique_ptr<RoPE> rope_;
+    bool use_rope_;
+    size_t max_seq_len_;
+    bool use_qk_norm_;
+    std::unique_ptr<RMSNorm> q_norm_;
+
+    std::unique_ptr<RMSNorm> k_norm_;
+    bool training_mode_ = false;
+    float yarn_temp_scale_ = 1.0f;
+
+    struct Cache {
+        Tensor input;
+        Tensor q_proj;
+        Tensor k_proj;
+        Tensor v_proj;
+        Tensor attn_weights;
+        Tensor attn_output;
+        Tensor w_out_input;
+        Tensor residual;
+    };
+    Cache cache_;
+
+    CausalSelfAttention(size_t d_model, size_t n_q_heads, size_t n_kv_heads,
+                        bool use_rope = true, size_t max_seq_len = 128, bool use_qk_norm = true);
+    Tensor forward(const Tensor& x, const Tensor* padding_mask = nullptr);
+
+    void train() { training_mode_ = true; }
+    void eval() { training_mode_ = false; }
+
+    struct Gradients {
+        Tensor w_q_weight_grad;
+        Tensor w_q_bias_grad;
+        Tensor w_k_weight_grad;
+        Tensor w_k_bias_grad;
+        Tensor w_v_weight_grad;
+        Tensor w_v_bias_grad;
+        Tensor w_out_weight_grad;
+        Tensor w_out_bias_grad;
+        Tensor input_grad;
+    };
+
+    Gradients backward(const Tensor& output_grad);
+
+private:
+    Tensor layernorm_backward_impl(const Tensor& input, const Tensor& weight, const Tensor& output_grad, float eps = 1e-5f);
+    Tensor linear_backward_weight_impl(const Tensor& input, const Tensor& output_grad);
+    Tensor linear_backward_input_impl(const Tensor& output_grad, const Tensor& weight);
+    Tensor bias_backward_impl(const Tensor& output_grad);
+};
+
+class CausalLMHead {
+public:
+    CausalLMConfig config_;
+
+    Tensor w_embed_;
+    Tensor w_pos_;
+    Tensor dw_kernel_;
+    std::shared_ptr<Linear> pw_conv_;
+    std::shared_ptr<Linear> sae_w_encode_;
+    std::shared_ptr<Linear> sae_w_decode_;
+    std::shared_ptr<Linear> ntm_w_read_;
+    std::shared_ptr<Linear> ntm_w_write_;
+    std::shared_ptr<Linear> ntm_w_erase_;
+    Tensor ntm_memory_;
+    Tensor shadow_memory_;
+    bool training_mode_ = false;
+    bool mode_set_ = false;
+    std::shared_ptr<Linear> w_proj_;
+    std::shared_ptr<Linear> bridge_;
+    std::shared_ptr<Linear> w_out_;
+    std::shared_ptr<LayerNorm> ln_;
+    std::shared_ptr<LatentKVCache> kv_cache_;
+    Tensor last_hidden_;
+    Tensor last_projected_;
+    std::vector<std::unique_ptr<CausalSelfAttention>> attn_layers_;
+    std::unique_ptr<SwiGLUFFN> swiglu_;
+
+    struct TrainingCache {
+        std::vector<size_t> input_ids;
+        Tensor x_embed;
+        Tensor x_pos;
+        std::vector<Tensor> attn_inputs;
+        std::vector<Tensor> attn_outputs;
+        Tensor x_gate_in;
+        Tensor x_gate_pre_sigmoid;
+        Tensor x_after_gate;
+        Tensor x_after_swiglu;
+        Tensor x_sae_encoded;
+        Tensor x_after_sae;
+        Tensor x_ntm_read_weights;
+        Tensor x_ntm_read_content;
+        Tensor x_ntm_h;
+        Tensor x_ntm_erase;
+        Tensor x_ntm_write;
+        Tensor x_after_ntm;
+        Tensor x_after_ln;
+        Tensor x_pooled;
+        Tensor x_bridge;
+        Tensor x_projected;
+    };
+    TrainingCache train_cache_;
+
+    size_t sliding_window_drops_;
+
+    void tie_weights();
+
+    CausalLMHead(const CausalLMConfig& config);
+
+    void train();
+    void eval();
+    bool is_training() const { return training_mode_; }
+    void set_yarn_scale(float scale_factor);
+
+    Tensor embed_lookup(const std::vector<size_t>& ids);
+    Tensor positional_encode(const Tensor& x, size_t offset = 0);
+    Tensor causal_window_gate(const Tensor& x);
+    Tensor sae_sparse(const Tensor& x);
+    Tensor ntm_memory_access(const Tensor& x);
+    Tensor last_token_pool(const Tensor& x);
+    Tensor mean_pool(const Tensor& x);
+    Tensor pool(const Tensor& x);
+
+    Tensor make_padding_mask(const std::vector<size_t>& token_ids) const;
+
+    Tensor forward(const std::vector<size_t>& token_ids);
+    Tensor forward_step(size_t token_id, size_t pos);
+    Tensor forward_for_training(const std::vector<size_t>& token_ids);
+
+    struct LMGradients {
+        std::vector<CausalSelfAttention::Gradients> attn_grads;
+        Tensor w_proj_weight_grad;
+        Tensor w_proj_bias_grad;
+        Tensor bridge_weight_grad;
+        Tensor bridge_bias_grad;
+        Tensor w_out_weight_grad;
+        Tensor w_out_bias_grad;
+        Tensor embed_grad;
+        std::vector<size_t> used_token_ids;
+        Tensor ln_weight_grad;
+        Tensor ln_bias_grad;
+        Tensor ntm_read_weight_grad;
+        Tensor ntm_write_weight_grad;
+        Tensor ntm_erase_weight_grad;
+        Tensor sae_encode_weight_grad;
+        Tensor sae_decode_weight_grad;
+        Tensor dw_kernel_grad;
+        Tensor pw_conv_weight_grad;
+        Tensor pw_conv_bias_grad;
+        SwiGLUFFN::Gradients swiglu_grads;
+    };
+
+    LMGradients backward_from_logits(const Tensor& logits_grad);
+    void apply_lm_gradients(LMGradients& grads, float lr);
+
+    // 将所有可训练参数注册到 AdamW 优化器（按 weight/bias 分组）
+    // 注意：不注册 attn.norm（backward_from_logits 不计算其梯度），不注册 w_embed_/w_out_（weight_tying 下共享，由 embed_grad 单独处理）
+    void register_trainable_params(AdamW& opt, float lr, float weight_decay);
+    // 按 register_trainable_params 的注册顺序，将 lm_grads 中的梯度填入优化器的 param_groups_[].grads
+    void assign_grads_to_optimizer(AdamW& opt, LMGradients& grads);
+
+    void clear_cache();
+    CacheStats cache_stats() const;
+
+private:
+    Tensor ln_backward_impl(const Tensor& input, const Tensor& weight, const Tensor& output_grad, float eps = 1e-5f);
+    Tensor lm_head_linear_backward_input(const Tensor& output_grad, const Tensor& weight);
+    Tensor lm_head_linear_backward_weight(const Tensor& input, const Tensor& output_grad);
+    Tensor lm_head_bias_backward(const Tensor& output_grad);
+};
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_CAUSAL_LM_HPP

include/neuroflow/cuda_context.hpp

@@ -0,0 +1,171 @@
+#ifndef NEUROFLOW_CUDA_CONTEXT_HPP
+#define NEUROFLOW_CUDA_CONTEXT_HPP
+
+#ifdef USE_CUDA
+
+#include <cstddef>
+#include <stdexcept>
+#include <string>
+
+#include <cuda_runtime.h>
+#include <cublas_v2.h>
+
+#define CUDA_CHECK(call)                                                       \
+    do {                                                                        \
+        cudaError_t err = (call);                                               \
+        if (err != cudaSuccess) {                                               \
+            throw std::runtime_error(                                           \
+                std::string("[CUDA ERROR] ") + cudaGetErrorString(err)          \
+                + " at " + __FILE__ + ":" + std::to_string(__LINE__));          \
+        }                                                                       \
+    } while (0)
+
+#define CUBLAS_CHECK(call)                                                     \
+    do {                                                                        \
+        cublasStatus_t status = (call);                                         \
+        if (status != CUBLAS_STATUS_SUCCESS) {                                  \
+            throw std::runtime_error(                                           \
+                std::string("[CUBLAS ERROR] code=") + std::to_string(status)    \
+                + " at " + __FILE__ + ":" + std::to_string(__LINE__));          \
+        }                                                                       \
+    } while (0)
+
+namespace neuroflow {
+
+class CudaContext {
+public:
+    static CudaContext& instance() {
+        static CudaContext ctx;
+        return ctx;
+    }
+
+    bool initialize(int device_id = 0) {
+        if (initialized_) return true;
+
+        int device_count = 0;
+        cudaError_t err = cudaGetDeviceCount(&device_count);
+        if (err != cudaSuccess || device_count == 0) {
+            std::cerr << "[CUDA WARNING] No CUDA-capable GPU detected" << std::endl;
+            return false;
+        }
+
+        if (device_id >= device_count) {
+            std::cerr << "[CUDA WARNING] Device " << device_id
+                      << " not available (count=" << device_count << ")" << std::endl;
+            return false;
+        }
+
+        CUDA_CHECK(cudaSetDevice(device_id));
+
+        cudaDeviceProp prop;
+        CUDA_CHECK(cudaGetDeviceProperties(&prop, device_id));
+        if (prop.major < 8) {
+            std::cerr << "[CUDA WARNING] GPU Compute Capability " << prop.major
+                      << "." << prop.minor << " < 8.0 (Ampere required)" << std::endl;
+            return false;
+        }
+
+        CUDA_CHECK(cudaStreamCreate(&stream_));
+        CUBLAS_CHECK(cublasCreate(&cublas_handle_));
+        CUBLAS_CHECK(cublasSetStream(cublas_handle_, stream_));
+
+        device_id_ = device_id;
+        initialized_ = true;
+
+        std::cerr << "[CUDA] Initialized on " << prop.name
+                  << " (CC " << prop.major << "." << prop.minor
+                  << ", " << prop.totalGlobalMem / (1024*1024) << " MB)" << std::endl;
+        return true;
+    }
+
+    void finalize() {
+        if (!initialized_) return;
+        CUBLAS_CHECK(cublasDestroy(cublas_handle_));
+        CUDA_CHECK(cudaStreamDestroy(stream_));
+        initialized_ = false;
+        device_id_ = -1;
+    }
+
+    void sgemm(bool transA, bool transB,
+               int M, int N, int K,
+               float alpha, const float* d_A, int lda,
+               const float* d_B, int ldb,
+               float beta, float* d_C, int ldc) {
+        cublasOperation_t opA = transA ? CUBLAS_OP_T : CUBLAS_OP_N;
+        cublasOperation_t opB = transB ? CUBLAS_OP_T : CUBLAS_OP_N;
+        CUBLAS_CHECK(cublasSgemm(cublas_handle_, opA, opB,
+                                 M, N, K, &alpha, d_A, lda, d_B, ldb, &beta, d_C, ldc));
+    }
+
+    void sgemm_rowmajor(bool transA, bool transB,
+                        int M, int N, int K,
+                        float alpha, const float* d_A, int lda,
+                        const float* d_B, int ldb,
+                        float beta, float* d_C, int ldc) {
+        cublasOperation_t opA = transB ? CUBLAS_OP_T : CUBLAS_OP_N;
+        cublasOperation_t opB = transA ? CUBLAS_OP_T : CUBLAS_OP_N;
+        CUBLAS_CHECK(cublasSgemm(cublas_handle_, opA, opB,
+                                 N, M, K, &alpha, d_B, ldb, d_A, lda, &beta, d_C, ldc));
+    }
+
+    void* alloc(size_t bytes) {
+        void* d_ptr = nullptr;
+        CUDA_CHECK(cudaMalloc(&d_ptr, bytes));
+        return d_ptr;
+    }
+
+    void free(void* d_ptr) {
+        if (d_ptr) CUDA_CHECK(cudaFree(d_ptr));
+    }
+
+    void copy_h2d(void* dst, const void* src, size_t bytes) {
+        CUDA_CHECK(cudaMemcpyAsync(dst, src, bytes, cudaMemcpyHostToDevice, stream_));
+    }
+
+    void copy_d2h(void* dst, const void* src, size_t bytes) {
+        CUDA_CHECK(cudaMemcpyAsync(dst, src, bytes, cudaMemcpyDeviceToHost, stream_));
+    }
+
+    void copy_d2d(void* dst, const void* src, size_t bytes) {
+        CUDA_CHECK(cudaMemcpyAsync(dst, src, bytes, cudaMemcpyDeviceToDevice, stream_));
+    }
+
+    void synchronize() {
+        CUDA_CHECK(cudaStreamSynchronize(stream_));
+    }
+
+    bool is_available() const { return initialized_; }
+
+    size_t free_memory() const {
+        if (!initialized_) return 0;
+        size_t free = 0, total = 0;
+        CUDA_CHECK(cudaMemGetInfo(&free, &total));
+        return free;
+    }
+
+    size_t total_memory() const {
+        if (!initialized_) return 0;
+        size_t free = 0, total = 0;
+        CUDA_CHECK(cudaMemGetInfo(&free, &total));
+        return total;
+    }
+
+    cudaStream_t stream() const { return stream_; }
+    cublasHandle_t cublas_handle() const { return cublas_handle_; }
+
+private:
+    CudaContext() = default;
+    ~CudaContext() { finalize(); }
+    CudaContext(const CudaContext&) = delete;
+    CudaContext& operator=(const CudaContext&) = delete;
+
+    cublasHandle_t cublas_handle_ = nullptr;
+    cudaStream_t stream_ = nullptr;
+    bool initialized_ = false;
+    int device_id_ = -1;
+};
+
+} // namespace neuroflow
+
+#endif // USE_CUDA
+#endif // NEUROFLOW_CUDA_CONTEXT_HPP

include/neuroflow/cuda_kernels.hpp

@@ -0,0 +1,1152 @@
+#ifndef NEUROFLOW_CUDA_KERNELS_HPP
+#define NEUROFLOW_CUDA_KERNELS_HPP
+
+#ifdef USE_CUDA
+
+#include <cstddef>
+#include <cstdio>
+#include <cmath>
+#include <cuda_runtime.h>
+#include "cuda_context.hpp"
+
+namespace neuroflow {
+
+__device__ inline void safe_atomic_max_float(float* addr, float val) {
+    unsigned int old = atomicCAS(reinterpret_cast<unsigned int*>(addr),
+                                  __float_as_uint(val),
+                                  __float_as_uint(val));
+    while (val > __uint_as_float(old)) {
+        unsigned int assumed = old;
+        old = atomicCAS(reinterpret_cast<unsigned int*>(addr),
+                         assumed,
+                         __float_as_uint(val));
+        if (old == assumed) break;
+    }
+}
+
+// ═══════════════════════════════════════════════════════
+// CUDA Kernel 实现
+// ═══════════════════════════════════════════════════════
+
+// --- GELU ---
+__global__ void kernel_gelu_impl(float* data, size_t n) {
+    const float SQRT_2_PI = 0.7978845608f;
+    const float COEFF = 0.044715f;
+    size_t idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= n) return;
+    float v = data[idx];
+    float inner = SQRT_2_PI * (v + COEFF * v * v * v);
+    data[idx] = 0.5f * v * (1.0f + tanhf(inner));
+}
+
+inline void launch_gelu(float* d_data, size_t n, cudaStream_t stream) {
+    int block = 256;
+    int grid = (n + block - 1) / block;
+    kernel_gelu_impl<<<grid, block, 0, stream>>>(d_data, n);
+}
+
+// --- Sigmoid ---
+__global__ void kernel_sigmoid_impl(float* data, size_t n) {
+    size_t idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= n) return;
+    data[idx] = 1.0f / (1.0f + expf(-data[idx]));
+}
+
+inline void launch_sigmoid(float* d_data, size_t n, cudaStream_t stream) {
+    int block = 256;
+    int grid = (n + block - 1) / block;
+    kernel_sigmoid_impl<<<grid, block, 0, stream>>>(d_data, n);
+}
+
+// --- Element-wise Add (residual) ---
+__global__ void kernel_add_impl(float* out, const float* a, const float* b, size_t n) {
+    size_t idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= n) return;
+    out[idx] = a[idx] + b[idx];
+}
+
+inline void launch_add(float* d_out, const float* d_a, const float* d_b, size_t n, cudaStream_t stream) {
+    int block = 256;
+    int grid = (n + block - 1) / block;
+    kernel_add_impl<<<grid, block, 0, stream>>>(d_out, d_a, d_b, n);
+}
+
+// --- Scale + Add ---
+__global__ void kernel_scale_add_impl(float* out, const float* a, float scale, const float* b, size_t n) {
+    size_t idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= n) return;
+    out[idx] = a[idx] * scale + b[idx];
+}
+
+inline void launch_scale_add(float* d_out, const float* d_a, float scale, const float* d_b, size_t n, cudaStream_t stream) {
+    int block = 256;
+    int grid = (n + block - 1) / block;
+    kernel_scale_add_impl<<<grid, block, 0, stream>>>(d_out, d_a, scale, d_b, n);
+}
+
+// --- Bias Add ---
+__global__ void kernel_bias_add_impl(float* out, const float* bias, int rows, int cols) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    int total = rows * cols;
+    if (idx >= total) return;
+    int j = idx % cols;
+    out[idx] += bias[j];
+}
+
+inline void launch_bias_add(float* d_out, const float* d_bias, int rows, int cols, cudaStream_t stream) {
+    int block = 256;
+    int total = rows * cols;
+    int grid = (total + block - 1) / block;
+    kernel_bias_add_impl<<<grid, block, 0, stream>>>(d_out, d_bias, rows, cols);
+}
+
+// --- Softmax (online, per-row) ---
+__global__ void kernel_softmax_impl(float* data, int rows, int cols) {
+    int row = blockIdx.x;
+    if (row >= rows) return;
+    float* row_data = data + row * cols;
+
+    float max_val = -1e30f;
+    for (int j = threadIdx.x; j < cols; j += blockDim.x) {
+        max_val = fmaxf(max_val, row_data[j]);
+    }
+    __shared__ float s_max;
+    if (threadIdx.x == 0) s_max = -1e30f;
+    __syncthreads();
+    safe_atomic_max_float(&s_max, max_val);
+    __syncthreads();
+    max_val = s_max;
+
+    float sum = 0.0f;
+    for (int j = threadIdx.x; j < cols; j += blockDim.x) {
+        row_data[j] = expf(row_data[j] - max_val);
+        sum += row_data[j];
+    }
+    __shared__ float s_sum;
+    if (threadIdx.x == 0) s_sum = 0.0f;
+    __syncthreads();
+    atomicAdd(&s_sum, sum);
+    __syncthreads();
+
+    for (int j = threadIdx.x; j < cols; j += blockDim.x) {
+        row_data[j] /= s_sum;
+    }
+}
+
+inline void launch_softmax(float* d_data, int rows, int cols, cudaStream_t stream) {
+    kernel_softmax_impl<<<rows, 256, 0, stream>>>(d_data, rows, cols);
+}
+
+// --- Causal Softmax (per-row, zero out j > i) ---
+__global__ void kernel_causal_softmax_impl(float* data, int seq_len) {
+    int i = blockIdx.x;
+    if (i >= seq_len) return;
+    float* row = data + i * seq_len;
+
+    for (int j = threadIdx.x; j <= i; j += blockDim.x) {
+        // keep
+    }
+    // Zero future positions
+    for (int j = i + 1 + threadIdx.x; j < seq_len; j += blockDim.x) {
+        row[j] = 0.0f;
+    }
+    __syncthreads();
+
+    float max_val = -1e30f;
+    for (int j = threadIdx.x; j <= i; j += blockDim.x) {
+        max_val = fmaxf(max_val, row[j]);
+    }
+    __shared__ float s_max;
+    if (threadIdx.x == 0) s_max = -1e30f;
+    __syncthreads();
+    safe_atomic_max_float(&s_max, max_val);
+    __syncthreads();
+    max_val = s_max;
+
+    float sum = 0.0f;
+    for (int j = threadIdx.x; j <= i; j += blockDim.x) {
+        row[j] = expf(row[j] - max_val);
+        sum += row[j];
+    }
+    __shared__ float s_sum;
+    if (threadIdx.x == 0) s_sum = 0.0f;
+    __syncthreads();
+    atomicAdd(&s_sum, sum);
+    __syncthreads();
+
+    for (int j = threadIdx.x; j <= i; j += blockDim.x) {
+        row[j] /= s_sum;
+    }
+}
+
+inline void launch_causal_softmax(float* d_data, int seq_len, cudaStream_t stream) {
+    kernel_causal_softmax_impl<<<seq_len, 256, 0, stream>>>(d_data, seq_len);
+}
+
+// --- LayerNorm Forward ---
+__global__ void kernel_layer_norm_impl(float* out, const float* inp, const float* w, const float* b, int rows, int cols, float eps) {
+    int row = blockIdx.x;
+    if (row >= rows) return;
+    const float* row_in = inp + row * cols;
+    float* row_out = out + row * cols;
+
+    float mean = 0.0f;
+    for (int j = threadIdx.x; j < cols; j += blockDim.x) {
+        mean += row_in[j];
+    }
+    __shared__ float s_mean;
+    if (threadIdx.x == 0) s_mean = 0.0f;
+    __syncthreads();
+    atomicAdd(&s_mean, mean);
+    __syncthreads();
+    mean = s_mean / cols;
+
+    float var = 0.0f;
+    for (int j = threadIdx.x; j < cols; j += blockDim.x) {
+        float diff = row_in[j] - mean;
+        var += diff * diff;
+    }
+    __shared__ float s_var;
+    if (threadIdx.x == 0) s_var = 0.0f;
+    __syncthreads();
+    atomicAdd(&s_var, var);
+    __syncthreads();
+    var = s_var / cols;
+
+    float inv_std = 1.0f / sqrtf(var + eps);
+    for (int j = threadIdx.x; j < cols; j += blockDim.x) {
+        float norm = (row_in[j] - mean) * inv_std;
+        row_out[j] = w[j] * norm + b[j];
+    }
+}
+
+inline void launch_layer_norm(float* d_out, const float* d_inp, const float* d_w, const float* d_b, int rows, int cols, float eps, cudaStream_t stream) {
+    kernel_layer_norm_impl<<<rows, 256, 0, stream>>>(d_out, d_inp, d_w, d_b, rows, cols, eps);
+}
+
+// --- Embed Lookup ---
+__global__ void kernel_embed_lookup_impl(float* out, const float* embed, const int* token_ids, int seq_len, int d_model, float scale) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    int total = seq_len * d_model;
+    if (idx >= total) return;
+    int i = idx / d_model;
+    int d = idx % d_model;
+    int tid = token_ids[i];
+    out[idx] = embed[tid * d_model + d] * scale;
+}
+
+inline void launch_embed_lookup(float* d_out, const float* d_embed, const int* d_token_ids, int seq_len, int d_model, float scale, cudaStream_t stream) {
+    int total = seq_len * d_model;
+    int block = 256;
+    int grid = (total + block - 1) / block;
+    kernel_embed_lookup_impl<<<grid, block, 0, stream>>>(d_out, d_embed, d_token_ids, seq_len, d_model, scale);
+}
+
+// --- Positional Encode (add) ---
+__global__ void kernel_positional_encode_impl(float* out, const float* pos_enc, int seq_len, int d_model, int offset) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    int total = seq_len * d_model;
+    if (idx >= total) return;
+    int i = idx / d_model;
+    int d = idx % d_model;
+    int p = offset + i;
+    out[idx] += pos_enc[p * d_model + d];
+}
+
+inline void launch_positional_encode(float* d_out, const float* d_pos_enc, int seq_len, int d_model, int offset, cudaStream_t stream) {
+    int total = seq_len * d_model;
+    int block = 256;
+    int grid = (total + block - 1) / block;
+    kernel_positional_encode_impl<<<grid, block, 0, stream>>>(d_out, d_pos_enc, seq_len, d_model, offset);
+}
+
+// --- SGD Update ---
+__global__ void kernel_sgd_update_impl(float* param, const float* grad, size_t n, float lr) {
+    size_t idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= n) return;
+    float g = grad[idx];
+    if (isfinite(g)) {
+        param[idx] -= lr * g;
+    }
+}
+
+inline void launch_sgd_update(float* d_param, const float* d_grad, size_t n, float lr, cudaStream_t stream) {
+    int block = 256;
+    int grid = (static_cast<int>(n) + block - 1) / block;
+    kernel_sgd_update_impl<<<grid, block, 0, stream>>>(d_param, d_grad, n, lr);
+}
+
+// --- Sparse Embed Update ---
+__global__ void kernel_sparse_embed_update_impl(float* embed, const float* grad, const int* token_ids, int seq_len, int d_model, float lr) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    int total = seq_len * d_model;
+    if (idx >= total) return;
+    int i = idx / d_model;
+    int d = idx % d_model;
+    int tid = token_ids[i];
+    float g = grad[i * d_model + d];
+    if (isfinite(g)) {
+        atomicAdd(&embed[tid * d_model + d], -lr * g);
+    }
+}
+
+inline void launch_sparse_embed_update(float* d_embed, const float* d_grad, const int* d_token_ids, int seq_len, int d_model, float lr, cudaStream_t stream) {
+    int total = seq_len * d_model;
+    int block = 256;
+    int grid = (total + block - 1) / block;
+    kernel_sparse_embed_update_impl<<<grid, block, 0, stream>>>(d_embed, d_grad, d_token_ids, seq_len, d_model, lr);
+}
+
+// --- Cross Entropy Forward (returns loss on GPU) ---
+__global__ void kernel_cross_entropy_impl(float* loss, const float* logits, int target_id, int vocab_size) {
+    __shared__ float s_max;
+    __shared__ float s_sum;
+
+    float local_max = -1e30f;
+    for (int j = threadIdx.x; j < vocab_size; j += blockDim.x) {
+        local_max = fmaxf(local_max, logits[j]);
+    }
+    if (threadIdx.x == 0) s_max = -1e30f;
+    __syncthreads();
+    safe_atomic_max_float(&s_max, local_max);
+    __syncthreads();
+
+    float local_sum = 0.0f;
+    for (int j = threadIdx.x; j < vocab_size; j += blockDim.x) {
+        local_sum += expf(logits[j] - s_max);
+    }
+    if (threadIdx.x == 0) s_sum = 0.0f;
+    __syncthreads();
+    atomicAdd(&s_sum, local_sum);
+    __syncthreads();
+
+    if (threadIdx.x == 0) {
+        float log_sum_exp = s_max + logf(s_sum);
+        *loss = -(logits[target_id] - log_sum_exp);
+    }
+}
+
+inline void launch_cross_entropy(float* d_loss, const float* d_logits, int target_id, int vocab_size, cudaStream_t stream) {
+    kernel_cross_entropy_impl<<<1, 256, 0, stream>>>(d_loss, d_logits, target_id, vocab_size);
+}
+
+// --- Cross Entropy Backward (softmax - one_hot) ---
+__global__ void kernel_cross_entropy_backward_impl(float* grad, const float* logits, int target_id, int vocab_size) {
+    __shared__ float s_max;
+    __shared__ float s_sum;
+
+    float local_max = -1e30f;
+    for (int j = threadIdx.x; j < vocab_size; j += blockDim.x) {
+        local_max = fmaxf(local_max, logits[j]);
+    }
+    if (threadIdx.x == 0) s_max = -1e30f;
+    __syncthreads();
+    safe_atomic_max_float(&s_max, local_max);
+    __syncthreads();
+
+    float local_sum = 0.0f;
+    for (int j = threadIdx.x; j < vocab_size; j += blockDim.x) {
+        float val = expf(logits[j] - s_max);
+        local_sum += val;
+    }
+    if (threadIdx.x == 0) s_sum = 0.0f;
+    __syncthreads();
+    atomicAdd(&s_sum, local_sum);
+    __syncthreads();
+
+    for (int j = threadIdx.x; j < vocab_size; j += blockDim.x) {
+        float softmax_val = expf(logits[j] - s_max) / s_sum;
+        grad[j] = softmax_val;
+        if (j == target_id) grad[j] -= 1.0f;
+    }
+}
+
+inline void launch_cross_entropy_backward(float* d_grad, const float* d_logits, int target_id, int vocab_size, cudaStream_t stream) {
+    kernel_cross_entropy_backward_impl<<<1, 256, 0, stream>>>(d_grad, d_logits, target_id, vocab_size);
+}
+
+// --- Mean Pool ---
+__global__ void kernel_mean_pool_impl(float* out, const float* inp, int seq_len, int d_model) {
+    int d = blockIdx.x * blockDim.x + threadIdx.x;
+    if (d >= d_model) return;
+    float sum = 0.0f;
+    for (int i = 0; i < seq_len; ++i) {
+        sum += inp[i * d_model + d];
+    }
+    out[d] = sum / seq_len;
+}
+
+inline void launch_mean_pool(float* d_out, const float* d_inp, int seq_len, int d_model, cudaStream_t stream) {
+    int block = 256;
+    int grid = (d_model + block - 1) / block;
+    kernel_mean_pool_impl<<<grid, block, 0, stream>>>(d_out, d_inp, seq_len, d_model);
+}
+
+// --- Last Token Pool ---
+__global__ void kernel_last_token_pool_impl(float* out, const float* inp, int seq_len, int d_model) {
+    int d = blockIdx.x * blockDim.x + threadIdx.x;
+    if (d >= d_model) return;
+    out[d] = inp[(seq_len - 1) * d_model + d];
+}
+
+inline void launch_last_token_pool(float* d_out, const float* d_inp, int seq_len, int d_model, cudaStream_t stream) {
+    int block = 256;
+    int grid = (d_model + block - 1) / block;
+    kernel_last_token_pool_impl<<<grid, block, 0, stream>>>(d_out, d_inp, seq_len, d_model);
+}
+
+// --- Fill Zero ---
+__global__ void kernel_fill_zero_impl(float* data, size_t n) {
+    size_t idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= n) return;
+    data[idx] = 0.0f;
+}
+
+inline void launch_fill_zero(float* d_data, size_t n, cudaStream_t stream) {
+    int block = 256;
+    int grid = (static_cast<int>(n) + block - 1) / block;
+    kernel_fill_zero_impl<<<grid, block, 0, stream>>>(d_data, n);
+}
+
+// --- Causal Mask Zero (zero out j > i in [seq_len, seq_len]) ---
+__global__ void kernel_causal_mask_zero_impl(float* data, int seq_len) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    int total = seq_len * seq_len;
+    if (idx >= total) return;
+    int i = idx / seq_len;
+    int j = idx % seq_len;
+    if (j > i) data[idx] = 0.0f;
+}
+
+inline void launch_causal_mask_zero(float* d_data, int seq_len, cudaStream_t stream) {
+    int total = seq_len * seq_len;
+    int block = 256;
+    int grid = (total + block - 1) / block;
+    kernel_causal_mask_zero_impl<<<grid, block, 0, stream>>>(d_data, seq_len);
+}
+
+// --- Scatter QKV gradients ---
+__global__ void kernel_scatter_qkv_impl(float* d_qkv, const float* d_q, const float* d_k, const float* d_v,
+                                         int seq_len, int d_model, int head_dim, int n_heads, int h) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    int total = seq_len * head_dim;
+    if (idx >= total) return;
+    int i = idx / head_dim;
+    int d = idx % head_dim;
+    size_t q_off = h * head_dim;
+    size_t k_off = d_model + h * head_dim;
+    size_t v_off = 2 * d_model + h * head_dim;
+    d_qkv[i * 3 * d_model + q_off + d] += d_q[i * head_dim + d];
+    d_qkv[i * 3 * d_model + k_off + d] += d_k[i * head_dim + d];
+    d_qkv[i * 3 * d_model + v_off + d] += d_v[i * head_dim + d];
+}
+
+inline void launch_scatter_qkv(float* d_qkv, const float* d_q, const float* d_k, const float* d_v,
+                                int seq_len, int d_model, int head_dim, int n_heads, int h, cudaStream_t stream) {
+    int total = seq_len * head_dim;
+    int block = 256;
+    int grid = (total + block - 1) / block;
+    kernel_scatter_qkv_impl<<<grid, block, 0, stream>>>(d_qkv, d_q, d_k, d_v, seq_len, d_model, head_dim, n_heads, h);
+}
+
+// --- Softmax Backward (for attention) ---
+__global__ void kernel_softmax_backward_impl(float* d_scores, const float* attn_weights, const float* d_attn_weights,
+                                              int seq_len, float inv_scale) {
+    int i = blockIdx.x;
+    if (i >= seq_len) return;
+    const float* aw_row = attn_weights + i * seq_len;
+    const float* daw_row = d_attn_weights + i * seq_len;
+    float* ds_row = d_scores + i * seq_len;
+
+    float dot = 0.0f;
+    for (int j = threadIdx.x; j <= i; j += blockDim.x) {
+        dot += aw_row[j] * daw_row[j];
+    }
+    __shared__ float s_dot;
+    if (threadIdx.x == 0) s_dot = 0.0f;
+    __syncthreads();
+    atomicAdd(&s_dot, dot);
+    __syncthreads();
+
+    for (int j = threadIdx.x; j <= i; j += blockDim.x) {
+        ds_row[j] = aw_row[j] * (daw_row[j] - s_dot) * inv_scale;
+    }
+    for (int j = i + 1 + threadIdx.x; j < seq_len; j += blockDim.x) {
+        ds_row[j] = 0.0f;
+    }
+}
+
+inline void launch_softmax_backward(float* d_scores, const float* d_attn_weights, const float* d_daw,
+                                     int seq_len, float inv_scale, cudaStream_t stream) {
+    kernel_softmax_backward_impl<<<seq_len, 256, 0, stream>>>(d_scores, d_attn_weights, d_daw, seq_len, inv_scale);
+}
+
+// --- LayerNorm Backward ---
+__global__ void kernel_layer_norm_backward_impl(float* input_grad, const float* input, const float* weight,
+                                                  const float* output_grad, int rows, int cols, float eps) {
+    int row = blockIdx.x;
+    if (row >= rows) return;
+    const float* inp_row = input + row * cols;
+    const float* og_row = output_grad + row * cols;
+    float* ig_row = input_grad + row * cols;
+
+    float mean = 0.0f;
+    for (int d = threadIdx.x; d < cols; d += blockDim.x) mean += inp_row[d];
+    __shared__ float s_mean;
+    if (threadIdx.x == 0) s_mean = 0.0f;
+    __syncthreads();
+    atomicAdd(&s_mean, mean);
+    __syncthreads();
+    mean = s_mean / cols;
+
+    float var = 0.0f;
+    for (int d = threadIdx.x; d < cols; d += blockDim.x) {
+        float diff = inp_row[d] - mean;
+        var += diff * diff;
+    }
+    __shared__ float s_var;
+    if (threadIdx.x == 0) s_var = 0.0f;
+    __syncthreads();
+    atomicAdd(&s_var, var);
+    __syncthreads();
+    var = s_var / cols;
+
+    float inv_std = 1.0f / sqrtf(var + eps);
+
+    float sum_gn = 0.0f, sum_gnx = 0.0f;
+    for (int d = threadIdx.x; d < cols; d += blockDim.x) {
+        float norm = (inp_row[d] - mean) * inv_std;
+        float gn = og_row[d] * weight[d];
+        sum_gn += gn;
+        sum_gnx += gn * norm;
+    }
+    __shared__ float s_sum_gn;
+    __shared__ float s_sum_gnx;
+    if (threadIdx.x == 0) { s_sum_gn = 0.0f; s_sum_gnx = 0.0f; }
+    __syncthreads();
+    atomicAdd(&s_sum_gn, sum_gn);
+    atomicAdd(&s_sum_gnx, sum_gnx);
+    __syncthreads();
+
+    for (int d = threadIdx.x; d < cols; d += blockDim.x) {
+        float norm = (inp_row[d] - mean) * inv_std;
+        float gn = og_row[d] * weight[d];
+        ig_row[d] = inv_std * (gn - s_sum_gn / cols - norm * s_sum_gnx / cols);
+    }
+}
+
+inline void launch_layer_norm_backward(float* d_input_grad, const float* d_input, const float* d_weight,
+                                         const float* d_output_grad, int rows, int cols, float eps, cudaStream_t stream) {
+    kernel_layer_norm_backward_impl<<<rows, 256, 0, stream>>>(d_input_grad, d_input, d_weight, d_output_grad, rows, cols, eps);
+}
+
+// --- Pool Backward (mean) ---
+__global__ void kernel_mean_pool_backward_impl(float* x_grad, const float* pooled_grad, int seq_len, int d_model) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    int total = seq_len * d_model;
+    if (idx >= total) return;
+    int d = idx % d_model;
+    float inv_n = 1.0f / static_cast<float>(seq_len);
+    x_grad[idx] = pooled_grad[d] * inv_n;
+}
+
+inline void launch_mean_pool_backward(float* d_x_grad, const float* d_pooled_grad, int seq_len, int d_model, cudaStream_t stream) {
+    int total = seq_len * d_model;
+    int block = 256;
+    int grid = (total + block - 1) / block;
+    kernel_mean_pool_backward_impl<<<grid, block, 0, stream>>>(d_x_grad, d_pooled_grad, seq_len, d_model);
+}
+
+// --- Bias Backward ---
+__global__ void kernel_bias_backward_impl(float* bias_grad, const float* output_grad, int batch, int dim) {
+    int j = blockIdx.x * blockDim.x + threadIdx.x;
+    if (j >= dim) return;
+    float sum = 0.0f;
+    for (int b = 0; b < batch; ++b) {
+        sum += output_grad[b * dim + j];
+    }
+    bias_grad[j] = sum / batch;
+}
+
+inline void launch_bias_backward(float* d_bias_grad, const float* d_output_grad, int batch, int dim, cudaStream_t stream) {
+    int block = 256;
+    int grid = (dim + block - 1) / block;
+    kernel_bias_backward_impl<<<grid, block, 0, stream>>>(d_bias_grad, d_output_grad, batch, dim);
+}
+
+// --- Sigmoid Backward (for causal gate) ---
+__global__ void kernel_sigmoid_backward_impl(float* gate_grad, const float* gate_output, const float* input, const float* x_grad, size_t n) {
+    size_t idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= n) return;
+    float g = gate_output[idx];
+    float sig = 1.0f / (1.0f + expf(-g));
+    gate_grad[idx] = x_grad[idx] * sig * (1.0f - sig) * input[idx] + x_grad[idx] * sig;
+}
+
+inline void launch_sigmoid_backward(float* d_gate_grad, const float* d_gate_output, const float* d_input, const float* d_x_grad, size_t n, cudaStream_t stream) {
+    int block = 256;
+    int grid = (static_cast<int>(n) + block - 1) / block;
+    kernel_sigmoid_backward_impl<<<grid, block, 0, stream>>>(d_gate_grad, d_gate_output, d_input, d_x_grad, n);
+}
+
+// --- Extract per-head Q, K, V from QKV ---
+__global__ void kernel_extract_qkv_impl(float* Q_h, float* K_h, float* V_h,
+                                          const float* qkv, int seq_len, int d_model, int head_dim, int h) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    int total = seq_len * head_dim;
+    if (idx >= total) return;
+    int i = idx / head_dim;
+    int d = idx % head_dim;
+    size_t q_off = h * head_dim;
+    size_t k_off = d_model + h * head_dim;
+    size_t v_off = 2 * d_model + h * head_dim;
+    Q_h[i * head_dim + d] = qkv[i * 3 * d_model + q_off + d];
+    K_h[i * head_dim + d] = qkv[i * 3 * d_model + k_off + d];
+    V_h[i * head_dim + d] = qkv[i * 3 * d_model + v_off + d];
+}
+
+inline void launch_extract_qkv(float* d_Q_h, float* d_K_h, float* d_V_h,
+                                const float* d_qkv, int seq_len, int d_model, int head_dim, int h, cudaStream_t stream) {
+    int total = seq_len * head_dim;
+    int block = 256;
+    int grid = (total + block - 1) / block;
+    kernel_extract_qkv_impl<<<grid, block, 0, stream>>>(d_Q_h, d_K_h, d_V_h, d_qkv, seq_len, d_model, head_dim, h);
+}
+
+__global__ void kernel_extract_head_impl(float* out, const float* src, int seq_len, int n_heads, int head_dim, int h) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    int total = seq_len * head_dim;
+    if (idx >= total) return;
+    int i = idx / head_dim;
+    int d = idx % head_dim;
+    out[i * head_dim + d] = src[i * n_heads * head_dim + h * head_dim + d];
+}
+
+inline void launch_extract_head(float* out, const float* src, int seq_len, int n_heads, int head_dim, int h, cudaStream_t stream) {
+    int total = seq_len * head_dim;
+    int block = 256;
+    int grid = (total + block - 1) / block;
+    kernel_extract_head_impl<<<grid, block, 0, stream>>>(out, src, seq_len, n_heads, head_dim, h);
+}
+
+// --- Extract per-head attn_out_grad ---
+__global__ void kernel_extract_attn_out_grad_impl(float* aog_h, const float* aog, int seq_len, int d_model, int head_dim, int h) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    int total = seq_len * head_dim;
+    if (idx >= total) return;
+    int i = idx / head_dim;
+    int d = idx % head_dim;
+    aog_h[i * head_dim + d] = aog[i * d_model + h * head_dim + d];
+}
+
+inline void launch_extract_attn_out_grad(float* d_aog_h, const float* d_aog, int seq_len, int d_model, int head_dim, int h, cudaStream_t stream) {
+    int total = seq_len * head_dim;
+    int block = 256;
+    int grid = (total + block - 1) / block;
+    kernel_extract_attn_out_grad_impl<<<grid, block, 0, stream>>>(d_aog_h, d_aog, seq_len, d_model, head_dim, h);
+}
+
+// --- NTM softmax (per-batch) ---
+__global__ void kernel_ntm_softmax_impl(float* data, int batch, int slots) {
+    int b = blockIdx.x;
+    if (b >= batch) return;
+    float* row = data + b * slots;
+    float max_val = -1e30f;
+    for (int s = threadIdx.x; s < slots; s += blockDim.x) {
+        max_val = fmaxf(max_val, row[s]);
+    }
+    __shared__ float s_max;
+    if (threadIdx.x == 0) s_max = -1e30f;
+    __syncthreads();
+    safe_atomic_max_float(&s_max, max_val);
+    __syncthreads();
+
+    float sum = 0.0f;
+    for (int s = threadIdx.x; s < slots; s += blockDim.x) {
+        row[s] = expf(row[s] - s_max);
+        sum += row[s];
+    }
+    __shared__ float s_sum;
+    if (threadIdx.x == 0) s_sum = 0.0f;
+    __syncthreads();
+    atomicAdd(&s_sum, sum);
+    __syncthreads();
+
+    for (int s = threadIdx.x; s < slots; s += blockDim.x) {
+        row[s] /= s_sum;
+    }
+}
+
+inline void launch_ntm_softmax(float* d_data, int batch, int slots, cudaStream_t stream) {
+    kernel_ntm_softmax_impl<<<batch, 256, 0, stream>>>(d_data, batch, slots);
+}
+
+// --- NTM read content (read_weights @ memory) ---
+__global__ void kernel_ntm_read_impl(float* read_content, const float* read_weights, const float* memory, int batch, int slots, int d_model) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    int total = batch * d_model;
+    if (idx >= total) return;
+    int b = idx / d_model;
+    int d = idx % d_model;
+    float val = 0.0f;
+    for (int s = 0; s < slots; ++s) {
+        val += read_weights[b * slots + s] * memory[s * d_model + d];
+    }
+    read_content[idx] = val;
+}
+
+inline void launch_ntm_read(float* d_read_content, const float* d_read_weights, const float* d_memory, int batch, int slots, int d_model, cudaStream_t stream) {
+    int total = batch * d_model;
+    int block = 256;
+    int grid = (total + block - 1) / block;
+    kernel_ntm_read_impl<<<grid, block, 0, stream>>>(d_read_content, d_read_weights, d_memory, batch, slots, d_model);
+}
+
+// --- NTM write (memory update) ---
+__global__ void kernel_ntm_write_impl(float* memory, const float* read_weights, const float* erase, const float* write_val, int batch, int slots, int d_model) {
+    int s = blockIdx.x;
+    int d = threadIdx.x;
+    if (s >= slots || d >= d_model) return;
+    for (int b = 0; b < batch; ++b) {
+        float rw = read_weights[b * slots + s];
+        float e = 1.0f / (1.0f + expf(-erase[b * d_model + d]));
+        float w = tanhf(write_val[b * d_model + d]);
+        memory[s * d_model + d] = memory[s * d_model + d] * (1.0f - rw * e) + rw * w;
+    }
+}
+
+inline void launch_ntm_write(float* d_memory, const float* d_read_weights, const float* d_erase, const float* d_write_val, int batch, int slots, int d_model, cudaStream_t stream) {
+    kernel_ntm_write_impl<<<slots, d_model, 0, stream>>>(d_memory, d_read_weights, d_erase, d_write_val, batch, slots, d_model);
+}
+
+// --- SAE Top-K mask (bitonic sort approach for small n) ---
+// Step 1: compute abs values into a temp buffer
+// Step 2: bitonic sort to find the k-th largest absolute value (threshold)
+// Step 3: zero out elements whose abs value < threshold
+
+__global__ void kernel_sae_topk_mask_impl(float* data, size_t n, size_t k) {
+    // For small n (typical d_model=128), use single-block bitonic sort
+    // Each thread handles multiple elements if n > blockDim
+    extern __shared__ float s_abs[];
+
+    // Load abs values into shared memory
+    for (size_t i = threadIdx.x; i < n; i += blockDim.x) {
+        s_abs[i] = fabsf(data[i]);
+    }
+    __syncthreads();
+
+    // Bitonic sort descending in shared memory
+    for (size_t stage = 1; stage < n; stage *= 2) {
+        for (size_t step = stage; step >= 1; step /= 2) {
+            for (size_t i = threadIdx.x; i < n / 2; i += blockDim.x) {
+                size_t dir = ((i / stage) % 2) == 0 ? 1 : 0;
+                size_t pair = i ^ step;
+                if (pair > i && pair < n) {
+                    float a = s_abs[i];
+                    float b = s_abs[pair];
+                    if ((dir && a < b) || (!dir && a > b)) {
+                        s_abs[i] = b;
+                        s_abs[pair] = a;
+                    }
+                }
+            }
+            __syncthreads();
+        }
+    }
+
+    // k-th largest absolute value is at index k-1 (0-indexed)
+    float threshold = 0.0f;
+    if (k > 0 && k <= n) {
+        threshold = s_abs[k - 1];
+    }
+    __syncthreads();
+
+    // Zero out elements below threshold
+    for (size_t i = threadIdx.x; i < n; i += blockDim.x) {
+        if (fabsf(data[i]) < threshold) {
+            data[i] = 0.0f;
+        }
+    }
+}
+
+inline void launch_sae_topk_mask(float* d_data, size_t n, size_t k, cudaStream_t stream) {
+    int block = 256;
+    size_t shared_mem = n * sizeof(float);
+    kernel_sae_topk_mask_impl<<<1, block, shared_mem, stream>>>(d_data, n, k);
+}
+
+// --- SAE Top-K mask for backward (same logic but on grad, using cached encoded values) ---
+__global__ void kernel_sae_topk_mask_backward_impl(float* grad, const float* encoded, size_t n, size_t k) {
+    extern __shared__ float s_abs[];
+
+    for (size_t i = threadIdx.x; i < n; i += blockDim.x) {
+        s_abs[i] = fabsf(encoded[i]);
+    }
+    __syncthreads();
+
+    for (size_t stage = 1; stage < n; stage *= 2) {
+        for (size_t step = stage; step >= 1; step /= 2) {
+            for (size_t i = threadIdx.x; i < n / 2; i += blockDim.x) {
+                size_t dir = ((i / stage) % 2) == 0 ? 1 : 0;
+                size_t pair = i ^ step;
+                if (pair > i && pair < n) {
+                    float a = s_abs[i];
+                    float b = s_abs[pair];
+                    if ((dir && a < b) || (!dir && a > b)) {
+                        s_abs[i] = b;
+                        s_abs[pair] = a;
+                    }
+                }
+            }
+            __syncthreads();
+        }
+    }
+
+    float threshold = 0.0f;
+    if (k > 0 && k <= n) {
+        threshold = s_abs[k - 1];
+    }
+    __syncthreads();
+
+    for (size_t i = threadIdx.x; i < n; i += blockDim.x) {
+        if (fabsf(encoded[i]) < threshold) {
+            grad[i] = 0.0f;
+        }
+    }
+}
+
+inline void launch_sae_topk_mask_backward(float* d_grad, const float* d_encoded, size_t n, size_t k, cudaStream_t stream) {
+    int block = 256;
+    size_t shared_mem = n * sizeof(float);
+    kernel_sae_topk_mask_backward_impl<<<1, block, shared_mem, stream>>>(d_grad, d_encoded, n, k);
+}
+
+// --- Gradient clipping (compute norm + scale) ---
+// Returns total squared norm on GPU (single float)
+__global__ void kernel_grad_norm_sq_impl(float* norm_sq, const float* const* grads, const size_t* sizes, int n_tensors) {
+    float local_sum = 0.0f;
+    for (int t = 0; t < n_tensors; ++t) {
+        const float* g = grads[t];
+        size_t sz = sizes[t];
+        for (size_t i = threadIdx.x; i < sz; i += blockDim.x) {
+            local_sum += g[i] * g[i];
+        }
+    }
+    __shared__ float s_sum;
+    if (threadIdx.x == 0) s_sum = 0.0f;
+    __syncthreads();
+    atomicAdd(&s_sum, local_sum);
+    __syncthreads();
+    if (threadIdx.x == 0) *norm_sq = s_sum;
+}
+
+// --- Scale multiple gradients by a single factor ---
+__global__ void kernel_scale_grads_impl(float** grads, const size_t* sizes, int n_tensors, float scale) {
+    int t = blockIdx.y;
+    if (t >= n_tensors) return;
+    size_t idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= sizes[t]) return;
+    grads[t][idx] *= scale;
+}
+
+// --- SGD update with gradient clipping (fused: scale + update) ---
+__global__ void kernel_sgd_update_clipped_impl(float* param, const float* grad, size_t n, float lr, float clip_scale) {
+    size_t idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= n) return;
+    float g = grad[idx] * clip_scale;
+    if (isfinite(g)) {
+        param[idx] -= lr * g;
+    }
+}
+
+inline void launch_sgd_update_clipped(float* d_param, const float* d_grad, size_t n, float lr, float clip_scale, cudaStream_t stream) {
+    int block = 256;
+    int grid = (static_cast<int>(n) + block - 1) / block;
+    kernel_sgd_update_clipped_impl<<<grid, block, 0, stream>>>(d_param, d_grad, n, lr, clip_scale);
+}
+
+// --- Padding Mask Generation ---
+__global__ void kernel_padding_mask_impl(float* mask, const int* token_ids, int seq_len, int padding_id) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= seq_len) return;
+    mask[idx] = (token_ids[idx] == padding_id) ? 0.0f : 1.0f;
+}
+
+inline void launch_padding_mask(float* d_mask, const int* d_token_ids, int seq_len, int padding_id, cudaStream_t stream) {
+    int block = 256;
+    int grid = (seq_len + block - 1) / block;
+    kernel_padding_mask_impl<<<grid, block, 0, stream>>>(d_mask, d_token_ids, seq_len, padding_id);
+}
+
+// --- Fused Causal + Padding Mask (applied to attention scores before softmax) ---
+// Sets attn_scores[i][j] = -inf if (j > i) OR (padding_mask[j] == 0)
+__global__ void kernel_fused_causal_padding_mask_impl(float* attn_scores, const float* padding_mask,
+                                                       int seq_len, int n_heads) {
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    int total = n_heads * seq_len * seq_len;
+    if (idx >= total) return;
+    int h = idx / (seq_len * seq_len);
+    int rem = idx % (seq_len * seq_len);
+    int i = rem / seq_len;
+    int j = rem % seq_len;
+    if (j > i || padding_mask[j] == 0.0f) {
+        attn_scores[idx] = -1e30f;
+    }
+}
+
+inline void launch_fused_causal_padding_mask(float* d_attn_scores, const float* d_padding_mask,
+                                              int seq_len, int n_heads, cudaStream_t stream) {
+    int total = n_heads * seq_len * seq_len;
+    int block = 256;
+    int grid = (total + block - 1) / block;
+    kernel_fused_causal_padding_mask_impl<<<grid, block, 0, stream>>>(d_attn_scores, d_padding_mask, seq_len, n_heads);
+}
+
+
+// --- GPU Top-K/Top-P Sampling ---
+// Performs temperature scaling, top-k filtering, top-p filtering, softmax, and sampling
+// entirely on GPU. Only the sampled token ID is copied back to CPU.
+// Uses a simple deterministic pseudo-random based on block/thread index + seed.
+
+__global__ void kernel_topk_topp_sampling_impl(
+    const float* d_logits, int* d_output_token,
+    int vocab_size, int top_k, float top_p, float temperature,
+    unsigned int seed, int num_candidates) {
+
+    extern __shared__ float smem[];
+
+    float* sm_scores = smem;
+    int* sm_indices = (int*)(sm_scores + num_candidates);
+
+    int tid = threadIdx.x;
+
+    for (int i = tid; i < num_candidates; ++i) {
+        sm_scores[i] = -1e30f;
+        sm_indices[i] = 0;
+    }
+    __syncthreads();
+
+    for (int i = tid; i < vocab_size; i += blockDim.x) {
+        float val = d_logits[i] / temperature;
+        if (val > sm_scores[num_candidates - 1]) {
+            int pos = num_candidates - 1;
+            while (pos > 0 && val > sm_scores[pos - 1]) {
+                sm_scores[pos] = sm_scores[pos - 1];
+                sm_indices[pos] = sm_indices[pos - 1];
+                pos--;
+            }
+            sm_scores[pos] = val;
+            sm_indices[pos] = i;
+        }
+    }
+    __syncthreads();
+
+    if (tid == 0) {
+        int k = min(top_k, vocab_size);
+        k = min(k, num_candidates);
+
+        float max_val = sm_scores[0];
+        float sum = 0.0f;
+        for (int i = 0; i < k; ++i) {
+            sm_scores[i] = expf(sm_scores[i] - max_val);
+            sum += sm_scores[i];
+        }
+        for (int i = 0; i < k; ++i) {
+            sm_scores[i] /= sum;
+        }
+
+        float cumulative = 0.0f;
+        int top_p_cutoff = k;
+        for (int i = 0; i < k; ++i) {
+            cumulative += sm_scores[i];
+            if (cumulative >= top_p) {
+                top_p_cutoff = i + 1;
+                break;
+            }
+        }
+
+        float renorm_sum = 0.0f;
+        for (int i = 0; i < top_p_cutoff; ++i) {
+            renorm_sum += sm_scores[i];
+        }
+
+        unsigned int rng_state = seed + blockIdx.x * 12345;
+        rng_state = rng_state * 1103515245 + 12345;
+        float r = ((rng_state >> 16) & 0x7fff) / 32768.0f;
+
+        float threshold = r * renorm_sum;
+        cumulative = 0.0f;
+        int chosen = 0;
+        for (int i = 0; i < top_p_cutoff; ++i) {
+            cumulative += sm_scores[i];
+            if (cumulative >= threshold) {
+                chosen = sm_indices[i];
+                break;
+            }
+        }
+
+        *d_output_token = chosen;
+    }
+}
+
+inline bool launch_topk_topp_sampling(const float* d_logits, int* d_output_token,
+                                       int vocab_size, int top_k, float top_p,
+                                       float temperature, unsigned int seed,
+                                       cudaStream_t stream) {
+    int num_candidates = min(top_k, vocab_size);
+    num_candidates = max(num_candidates, 1);
+
+    int block = 256;
+    size_t smem_size = sizeof(float) * num_candidates + sizeof(int) * num_candidates;
+
+    if (smem_size > 48 * 1024) {
+        return false;
+    }
+
+    kernel_topk_topp_sampling_impl<<<1, block, smem_size, stream>>>(
+        d_logits, d_output_token, vocab_size, top_k, top_p, temperature, seed, num_candidates);
+
+    return true;
+}
+
+// --- FP16/BF16 Conversion Kernels ---
+
+__global__ void kernel_fp32_to_fp16(uint16_t* out, const float* in, size_t n) {
+    size_t idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= n) return;
+    out[idx] = __float2half(in[idx]);
+}
+
+__global__ void kernel_fp16_to_fp32(float* out, const uint16_t* in, size_t n) {
+    size_t idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= n) return;
+    out[idx] = __half2float(in[idx]);
+}
+
+__global__ void kernel_fp32_to_bf16(uint16_t* out, const float* in, size_t n) {
+    size_t idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= n) return;
+    uint32_t bits;
+    memcpy(&bits, &in[idx], 4);
+    out[idx] = static_cast<uint16_t>(bits >> 16);
+}
+
+__global__ void kernel_bf16_to_fp32(float* out, const uint16_t* in, size_t n) {
+    size_t idx = blockIdx.x * blockDim.x + threadIdx.x;
+    if (idx >= n) return;
+    uint32_t bits = static_cast<uint32_t>(in[idx]) << 16;
+    memcpy(&out[idx], &bits, 4);
+}
+
+inline void launch_fp32_to_fp16(uint16_t* d_out, const float* d_in, size_t n, cudaStream_t stream) {
+    int block = 256;
+    int grid = (static_cast<int>(n) + block - 1) / block;
+    kernel_fp32_to_fp16<<<grid, block, 0, stream>>>(d_out, d_in, n);
+}
+
+inline void launch_fp16_to_fp32(float* d_out, const uint16_t* d_in, size_t n, cudaStream_t stream) {
+    int block = 256;
+    int grid = (static_cast<int>(n) + block - 1) / block;
+    kernel_fp16_to_fp32<<<grid, block, 0, stream>>>(d_out, d_in, n);
+}
+
+inline void launch_fp32_to_bf16(uint16_t* d_out, const float* d_in, size_t n, cudaStream_t stream) {
+    int block = 256;
+    int grid = (static_cast<int>(n) + block - 1) / block;
+    kernel_fp32_to_bf16<<<grid, block, 0, stream>>>(d_out, d_in, n);
+}
+
+inline void launch_bf16_to_fp32(float* d_out, const uint16_t* d_in, size_t n, cudaStream_t stream) {
+    int block = 256;
+    int grid = (static_cast<int>(n) + block - 1) / block;
+    kernel_bf16_to_fp32<<<grid, block, 0, stream>>>(d_out, d_in, n);
+}
+
+// --- Flash Attention (Tiled Online Softmax) ---
+
+template<int BLOCK_SIZE = 64, int HEAD_DIM = 64>
+__global__ void kernel_flash_attention_forward(
+    const float* Q, const float* K, const float* V,
+    float* O, int seq_len, float scale) {
+
+    int h = blockIdx.x;
+    int i = blockIdx.y * blockDim.x + threadIdx.x;
+    if (i >= seq_len) return;
+
+    extern __shared__ char smem[];
+    float* s_K = reinterpret_cast<float*>(smem);
+    float* s_V = reinterpret_cast<float*>(smem + BLOCK_SIZE * HEAD_DIM * sizeof(float));
+
+    float m = -1e30f;
+    float l = 0.0f;
+    float o[HEAD_DIM];
+    for (int d = 0; d < HEAD_DIM; ++d) o[d] = 0.0f;
+
+    const float* q_row = Q + (h * seq_len + i) * HEAD_DIM;
+
+    for (int jb = 0; jb < seq_len; jb += BLOCK_SIZE) {
+        int j_end = min(jb + BLOCK_SIZE, seq_len);
+
+        for (int j = threadIdx.x; j < (j_end - jb) * HEAD_DIM; j += blockDim.x) {
+            int jj = j / HEAD_DIM;
+            int dd = j % HEAD_DIM;
+            s_K[jj * HEAD_DIM + dd] = K[(h * seq_len + jb + jj) * HEAD_DIM + dd];
+            s_V[jj * HEAD_DIM + dd] = V[(h * seq_len + jb + jj) * HEAD_DIM + dd];
+        }
+        __syncthreads();
+
+        for (int jj = 0; jj < (j_end - jb); ++jj) {
+            int j = jb + jj;
+            if (j > i) break;
+
+            float dot = 0.0f;
+            for (int d = 0; d < HEAD_DIM; ++d) {
+                dot += q_row[d] * s_K[jj * HEAD_DIM + d];
+            }
+            dot *= scale;
+
+            float m_new = max(m, dot);
+            float l_new = expf(m - m_new) * l + expf(dot - m_new);
+
+            for (int d = 0; d < HEAD_DIM; ++d) {
+                o[d] = expf(m - m_new) * o[d] + expf(dot - m_new) * s_V[jj * HEAD_DIM + d];
+            }
+            m = m_new;
+            l = l_new;
+        }
+        __syncthreads();
+    }
+
+    if (l > 0.0f) {
+        for (int d = 0; d < HEAD_DIM; ++d) {
+            O[(h * seq_len + i) * HEAD_DIM + d] = o[d] / l;
+        }
+    } else {
+        for (int d = 0; d < HEAD_DIM; ++d) {
+            O[(h * seq_len + i) * HEAD_DIM + d] = 0.0f;
+        }
+    }
+}
+
+inline void launch_flash_attention(const float* d_Q, const float* d_K, const float* d_V,
+                                   float* d_O, int n_heads, int seq_len,
+                                   int head_dim, float scale, cudaStream_t stream) {
+    dim3 grid(n_heads, (seq_len + 63) / 64);
+    int block = 64;
+    size_t smem = 64 * head_dim * 2 * sizeof(float);
+    if (head_dim <= 64) {
+        kernel_flash_attention_forward<64, 64><<<grid, block, smem, stream>>>(
+            d_Q, d_K, d_V, d_O, seq_len, scale);
+    } else if (head_dim <= 128) {
+        kernel_flash_attention_forward<64, 128><<<grid, block, smem, stream>>>(
+            d_Q, d_K, d_V, d_O, seq_len, scale);
+    } else {
+        kernel_flash_attention_forward<64, 256><<<grid, block, smem, stream>>>(
+            d_Q, d_K, d_V, d_O, seq_len, scale);
+    }
+}
+
+} // namespace neuroflow
+
+#endif // USE_CUDA
+#endif // NEUROFLOW_CUDA_KERNELS_HPP

include/neuroflow/dpo.hpp

@@ -0,0 +1,91 @@
+#ifndef NEUROFLOW_DPO_HPP
+#define NEUROFLOW_DPO_HPP
+
+#include <cstddef>
+#include <random>
+#include <string>
+#include <vector>
+
+#include "adamw.hpp"
+#include "alignment_common.hpp"
+#include "causal_lm.hpp"
+#include "scheduler.hpp"
+#include "tokenizer.hpp"
+
+namespace neuroflow {
+
+struct DPOTrainConfig {
+    std::string data_path;
+    std::string sft_ckpt_path;
+    std::string tokenizer_path;
+    std::string output_dir;
+    float learning_rate = 1e-6f;
+    int epochs = 3;
+    size_t max_seq_len = 512;
+    float warmup_ratio = 0.05f;
+    float weight_decay = 0.0f;
+    float grad_clip = 1.0f;
+    float beta = 0.1f;
+    float adam_beta1 = 0.9f;
+    float adam_beta2 = 0.999f;
+    float adam_eps = 1e-8f;
+    size_t save_interval = 1000;
+    size_t log_interval = 10;
+    unsigned seed = 42;
+};
+
+class DPODataLoader {
+public:
+    DPODataLoader(const std::string& jsonl_path, size_t max_samples = 0);
+
+    bool has_next() const;
+    DPOSample next();
+    void reset();
+    void shuffle(std::mt19937& rng);
+    size_t total_samples() const { return samples_.size(); }
+    size_t invalid_count() const { return invalid_count_; }
+
+private:
+    std::vector<DPOSample> samples_;
+    size_t cursor_ = 0;
+    size_t invalid_count_ = 0;
+};
+
+struct DPOLossOutput {
+    float loss;
+    float alpha;
+    float reward_chosen;
+    float reward_rejected;
+};
+
+float compute_log_prob(CausalLMHead& model, const std::vector<size_t>& token_ids,
+                       size_t prompt_len, size_t vocab_size);
+
+DPOLossOutput compute_dpo_loss(float log_prob_chosen_policy,
+                                float log_prob_rejected_policy,
+                                float log_prob_chosen_ref,
+                                float log_prob_rejected_ref,
+                                float beta);
+
+class DPOTrainer {
+public:
+    DPOTrainConfig config;
+
+    DPOTrainer(const DPOTrainConfig& cfg);
+
+    void train();
+    float train_on_sample(const DPOSample& sample);
+
+private:
+    std::unique_ptr<CausalLMHead> policy_;
+    std::unique_ptr<CausalLMHead> reference_;
+    std::unique_ptr<BPETokenizer> tokenizer_;
+    std::unique_ptr<AdamW> optimizer_;
+    std::unique_ptr<CosineScheduler> scheduler_;
+
+    float compute_w_embed_checksum();
+};
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_DPO_HPP

include/neuroflow/generative.hpp

@@ -0,0 +1,19 @@
+#ifndef NEUROFLOW_GENERATIVE_HPP
+#define NEUROFLOW_GENERATIVE_HPP
+
+/**
+ * NeuroFlow 生成式语言模型模块 - 聚合头文件
+ *
+ * 为类脑模块化网络增加因果语言模型能力：
+ * - CausalLMHead: 词嵌入+因果门控+SAE+NTM+投影输出
+ * - Tokenizer: BPE/WordPiece分词器
+ * - SamplingStrategy: 贪心/Top-K/Top-P采样
+ * - GenerativeModel: 编排层，与ECN/DMN/SN协同
+ */
+
+#include "causal_lm.hpp"
+#include "tokenizer.hpp"
+#include "sampling.hpp"
+#include "generative_model.hpp"
+
+#endif // NEUROFLOW_GENERATIVE_HPP

include/neuroflow/generative_model.hpp

@@ -0,0 +1,44 @@
+#ifndef NEUROFLOW_GENERATIVE_MODEL_HPP
+#define NEUROFLOW_GENERATIVE_MODEL_HPP
+
+#include <memory>
+#include <random>
+#include <string>
+#include <vector>
+#include "model.hpp"
+#include "tensor.hpp"
+#include "causal_lm.hpp"
+#include "tokenizer.hpp"
+#include "sampling.hpp"
+
+namespace neuroflow {
+
+class GenerativeModel {
+public:
+    std::unique_ptr<CausalLMHead> lm_head_;
+    std::unique_ptr<Tokenizer> tokenizer_;
+    std::unique_ptr<SamplingStrategy> sampler_;
+    NeuroFlowModel* neuroflow_model_;
+
+    GenerativeModel(const CausalLMConfig& lm_config,
+                    std::unique_ptr<Tokenizer> tokenizer,
+                    NeuroFlowModel* nf_model = nullptr);
+
+    GenerateOutput generate(const std::string& prompt, const GenerateConfig& config);
+
+    Tensor apply_sn_gating(const Tensor& hidden, const Tensor& logits);
+    Tensor inject_memory(const Tensor& query, const Tensor& logits);
+
+    Tensor apply_repetition_penalty(Tensor logits, const GenerateConfig& config,
+                                     const std::vector<size_t>& generated);
+    Tensor apply_punct_penalty(Tensor logits, const GenerateConfig& config);
+
+    void set_strategy(SamplingStrategyType type);
+
+    void clear_cache();
+    CacheStats cache_stats() const;
+};
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_GENERATIVE_MODEL_HPP

include/neuroflow/grad_scaler.hpp

@@ -0,0 +1,36 @@
+#ifndef NEUROFLOW_GRAD_SCALER_HPP
+#define NEUROFLOW_GRAD_SCALER_HPP
+
+#include <cstddef>
+#include <vector>
+#include "tensor.hpp"
+
+namespace neuroflow {
+
+class GradScaler {
+public:
+    float scale_;
+    float growth_factor_;
+    float backoff_factor_;
+    size_t growth_interval_;
+    size_t growth_tracker_;
+
+    GradScaler(float init_scale = 65536.0f,
+               float growth_factor = 2.0f,
+               float backoff_factor = 0.5f,
+               size_t growth_interval = 2000);
+
+    float get_scale() const { return scale_; }
+
+    bool has_inf_or_nan(const std::vector<Tensor*>& grads) const;
+
+    void unscale(std::vector<Tensor*>& grads);
+
+    void scale_loss(Tensor& loss);
+
+    void update(bool found_inf);
+};
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_GRAD_SCALER_HPP

include/neuroflow/memory.hpp

@@ -0,0 +1,555 @@
+#ifndef NEUROFLOW_MEMORY_HPP
+#define NEUROFLOW_MEMORY_HPP
+
+/**
+ * NeuroFlow 记忆系统
+ * 
+ * 核心技术：
+ * 1. MLA (Multi-head Latent Attention) - DeepSeek KV压缩
+ * 2. 滑动窗口长记忆
+ * 3. 记忆分页与磁盘溢出
+ * 4. 记忆巩固 (LTP模拟)
+ */
+
+#include <fstream>
+#include <memory>
+#include <queue>
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include "networks.hpp"
+#include "tensor.hpp"
+
+namespace neuroflow {
+
+/**
+ * MLA压缩KV Cache
+ * 
+ * DeepSeek核心技术：将KV压缩到潜在空间
+ * 内存节省：87.5%+
+ */
+class LatentKVCache {
+public:
+    size_t d_model;
+    size_t n_heads;
+    size_t d_latent;  // 压缩维度
+    size_t head_dim;
+    
+    // 投影矩阵
+    std::shared_ptr<Linear> W_q;      // Q投影
+    std::shared_ptr<Linear> W_dkv;    // KV压缩投影
+    std::shared_ptr<Linear> W_uk;     // K解压
+    std::shared_ptr<Linear> W_uv;     // V解压
+    std::shared_ptr<Linear> W_o;      // 输出投影
+    
+    // Cache存储 (压缩形式)
+    Tensor cache;                     // (max_seq, d_latent)
+    size_t cache_len;
+    size_t max_cache_len;
+    
+    LatentKVCache(size_t model_dim, size_t heads, size_t latent_dim, size_t max_len = 4096)
+        : d_model(model_dim), n_heads(heads), d_latent(latent_dim),
+          head_dim(model_dim / heads), max_cache_len(max_len), cache_len(0) {
+        
+        W_q = std::make_shared<Linear>(d_model, d_model, false);
+        W_dkv = std::make_shared<Linear>(d_model, d_latent, false);  // 压缩！
+        W_uk = std::make_shared<Linear>(d_latent, d_model, false);
+        W_uv = std::make_shared<Linear>(d_latent, d_model, false);
+        W_o = std::make_shared<Linear>(d_model, d_model, false);
+        
+        // 初始化cache
+        cache = Tensor({max_len, d_latent}, QuantType::FP32);
+    }
+    
+    // 前向传播 (带cache)
+    Tensor forward(const Tensor& x, bool use_cache = true) {
+        size_t batch = x.shape_[0];
+        size_t seq_len = x.shape_.size() > 1 ? x.shape_[1] : 1;
+        size_t input_dim = x.shape_.size() > 2 ? x.shape_[2] : (x.shape_.size() > 1 ? x.shape_[1] : d_model);
+        
+        // 确定实际维度
+        if (x.shape_.size() == 2 && x.shape_[1] == d_model) {
+            // 输入是 {batch, d_model}，seq_len=1
+            seq_len = 1;
+            input_dim = d_model;
+        } else if (x.shape_.size() == 2) {
+            // 输入可能是 {batch, seq_len} 但缺少 d_model
+            // 将 seq_len 视为实际序列长度，假设每个位置是 d_model 维
+            // 这需要特殊处理
+            seq_len = 1;
+            input_dim = x.shape_[1];
+        }
+        
+        // Q投影 - 输入需要是 {batch * seq_len, d_model}
+        size_t flat_batch = batch * seq_len;
+        Tensor x_flat({flat_batch, d_model}, QuantType::FP32);
+        float* xf = x_flat.as_fp32();
+        const float* xd = x.as_fp32();
+        
+        // 如果输入维度小于 d_model，补零
+        size_t copy_size = std::min(input_dim, d_model);
+        for (size_t i = 0; i < flat_batch; ++i) {
+            for (size_t j = 0; j < copy_size; ++j) {
+                xf[i * d_model + j] = xd[i * input_dim + j];
+            }
+            for (size_t j = copy_size; j < d_model; ++j) {
+                xf[i * d_model + j] = 0.0f;
+            }
+        }
+        
+        Tensor q = W_q->forward(x_flat);
+        q = q.reshape({batch, seq_len, n_heads, head_dim});
+        
+        // KV压缩到潜在空间 (MLA核心!)
+        Tensor c_kv = W_dkv->forward(x_flat);
+        c_kv = c_kv.reshape({batch, seq_len, d_latent});
+        
+        // 拼接历史cache
+        if (use_cache && cache_len > 0) {
+            size_t new_len = cache_len + seq_len;
+            Tensor new_cache({new_len, d_latent}, QuantType::FP32);
+            float* nc = new_cache.as_fp32();
+            float* old = cache.as_fp32();
+            
+            // 拷贝旧cache
+            memcpy(nc, old, cache_len * d_latent * sizeof(float));
+            
+            // 拷贝新cache (取batch=0)
+            float* new_kv = c_kv.as_fp32();
+            for (size_t s = 0; s < seq_len; ++s) {
+                memcpy(nc + (cache_len + s) * d_latent,
+                       new_kv + s * d_latent,
+                       d_latent * sizeof(float));
+            }
+            
+            c_kv = new_cache.reshape({1, new_len, d_latent});
+        }
+        
+        // 解压K, V
+        size_t total_len = use_cache && cache_len > 0 ? (cache_len + seq_len) : seq_len;
+        
+        // 正确reshape c_kv到二维 - 注意batch维度处理
+        // 当有历史cache时，c_kv 被 reshape 到 {1, new_len, d_latent}
+        // 需要正确处理batch扩展
+        size_t c_kv_batch = use_cache && cache_len > 0 ? 1 : batch;
+        size_t actual_elements = c_kv_batch * total_len * d_latent;
+        
+        Tensor c_kv_flat({batch * total_len, d_latent}, QuantType::FP32);
+        float* ckf = c_kv_flat.as_fp32();
+        const float* ck = c_kv.as_fp32();
+        
+        // 正确拷贝：只拷贝实际存在的数据
+        for (size_t b = 0; b < batch; ++b) {
+            for (size_t t = 0; t < total_len; ++t) {
+                for (size_t d = 0; d < d_latent; ++d) {
+                    // 当有历史cache时，所有batch共享同一份cache数据
+                    size_t src_idx = (c_kv_batch == 1 ? t : b * total_len + t) * d_latent + d;
+                    size_t dst_idx = (b * total_len + t) * d_latent + d;
+                    ckf[dst_idx] = ck[src_idx];
+                }
+            }
+        }
+        
+        Tensor k = W_uk->forward(c_kv_flat);
+        Tensor v = W_uv->forward(c_kv_flat);
+        
+        k = k.reshape({batch, total_len, n_heads, head_dim});
+        v = v.reshape({batch, total_len, n_heads, head_dim});
+        
+        // 注意力计算
+        Tensor output({batch, seq_len, d_model}, QuantType::FP32);
+        float* out = output.as_fp32();
+        float* qp = q.as_fp32();
+        float* kp = k.as_fp32();
+        float* vp = v.as_fp32();
+        
+        float scale = 1.0f / std::sqrt(static_cast<float>(head_dim));
+        
+        for (size_t b = 0; b < batch; ++b) {
+            for (size_t h = 0; h < n_heads; ++h) {
+                for (size_t s = 0; s < seq_len; ++s) {
+                    // 计算注意力分数
+                    std::vector<float> scores(total_len);
+                    for (size_t t = 0; t < total_len; ++t) {
+                        float dot = 0;
+                        for (size_t d = 0; d < head_dim; ++d) {
+                            dot += qp[b * seq_len * n_heads * head_dim + s * n_heads * head_dim + h * head_dim + d]
+                                 * kp[b * total_len * n_heads * head_dim + t * n_heads * head_dim + h * head_dim + d];
+                        }
+                        scores[t] = dot * scale;
+                    }
+                    
+                    // Softmax
+                    float max_s = scores[0];
+                    for (auto& sc : scores) max_s = std::max(max_s, sc);
+                    float sum = 0;
+                    for (auto& sc : scores) {
+                        sc = std::exp(sc - max_s);
+                        sum += sc;
+                    }
+                    for (auto& sc : scores) sc /= sum;
+                    
+                    // 加权求和
+                    for (size_t d = 0; d < head_dim; ++d) {
+                        float val = 0;
+                        for (size_t t = 0; t < total_len; ++t) {
+                            val += scores[t] * vp[b * total_len * n_heads * head_dim + t * n_heads * head_dim + h * head_dim + d];
+                        }
+                        out[b * seq_len * d_model + s * d_model + h * head_dim + d] = val;
+                    }
+                }
+            }
+        }
+        
+        output = W_o->forward(output.reshape({batch * seq_len, d_model}));
+        output = output.reshape({batch, seq_len, d_model});
+        
+        // 更新cache
+        if (use_cache) {
+            float* c = cache.as_fp32();
+            float* nk = c_kv.as_fp32();
+            // 只保留最新的部分
+            size_t keep = std::min(seq_len, max_cache_len - cache_len);
+            if (cache_len + seq_len > max_cache_len) {
+                // 滑动：丢弃旧的
+                size_t shift = cache_len + seq_len - max_cache_len;
+                memmove(c, c + shift * d_latent, (cache_len - shift) * d_latent * sizeof(float));
+                cache_len -= shift;
+            }
+            memcpy(c + cache_len * d_latent, nk, seq_len * d_latent * sizeof(float));
+            cache_len += seq_len;
+        }
+        
+        return output.reshape({batch, d_model});
+    }
+    
+    // 清空cache
+    void clear_cache() {
+        cache_len = 0;
+        memset(cache.data_.get(), 0, cache.data_size_);
+    }
+    
+    // 获取cache大小 (字节)
+    size_t cache_size_bytes() const {
+        return cache_len * d_latent * sizeof(float);
+    }
+    
+    // 相比传统KV节省的内存比例
+    float memory_saving_ratio() const {
+        size_t traditional_size = cache_len * d_model * 2 * sizeof(float);  // K + V
+        size_t mla_size = cache_len * d_latent * sizeof(float);
+        return 1.0f - static_cast<float>(mla_size) / traditional_size;
+    }
+};
+
+/**
+ * MemoryConsolidationModule
+ * 
+ * 模拟海马体记忆巩固：
+ * 1. Encoding - 记忆编码
+ * 2. Retrieval - 注意力检索
+ * 3. Consolidation - LTP增强
+ */
+class MemoryConsolidationModule {
+public:
+    size_t memory_slots;
+    size_t memory_dim;
+    float ltp_rate;
+    
+    // 记忆库
+    Tensor memory_bank;  // (slots, dim)
+    
+    // 投影
+    std::shared_ptr<Linear> encode_proj;
+    std::shared_ptr<Linear> retrieve_proj;
+    std::shared_ptr<Linear> query_proj;
+    
+    MemoryConsolidationModule(size_t input_dim, size_t slots = 64, size_t dim = 128, float ltp = 0.01f)
+        : memory_slots(slots), memory_dim(dim), ltp_rate(ltp) {
+        
+        memory_bank = Tensor({slots, dim}, QuantType::FP32);
+        float* m = memory_bank.as_fp32();
+        std::mt19937 init_rng(42);
+        std::uniform_real_distribution<float> init_dist(-0.02f, 0.02f);
+        for (size_t i = 0; i < memory_bank.numel(); ++i) {
+            m[i] = init_dist(init_rng);
+        }
+        
+        encode_proj = std::make_shared<Linear>(input_dim, dim);
+        retrieve_proj = std::make_shared<Linear>(dim, input_dim);
+        query_proj = std::make_shared<Linear>(input_dim, dim);
+    }
+    
+    // 编码
+    Tensor encode(const Tensor& x) {
+        return encode_proj->forward(x);
+    }
+    
+    // 检索
+    struct RetrievalResult {
+        Tensor retrieved;
+        Tensor attention;
+    };
+    
+    RetrievalResult retrieve(const Tensor& query) {
+        RetrievalResult result;
+        
+        Tensor q = query_proj->forward(query);  // (batch, dim)
+        
+        // 注意力: query @ memory_bank.T
+        size_t batch = q.shape_[0];
+        result.attention = Tensor({batch, memory_slots}, QuantType::FP32);
+        
+        float* qp = q.as_fp32();
+        float* mp = memory_bank.as_fp32();
+        float* ap = result.attention.as_fp32();
+        
+        float scale = 1.0f / std::sqrt(static_cast<float>(memory_dim));
+        
+        for (size_t b = 0; b < batch; ++b) {
+            // 计算分数
+            std::vector<float> scores(memory_slots);
+            for (size_t s = 0; s < memory_slots; ++s) {
+                float dot = 0;
+                for (size_t d = 0; d < memory_dim; ++d) {
+                    dot += qp[b * memory_dim + d] * mp[s * memory_dim + d];
+                }
+                scores[s] = dot * scale;
+            }
+            
+            // Softmax
+            float max_s = scores[0];
+            for (auto& sc : scores) max_s = std::max(max_s, sc);
+            float sum = 0;
+            for (auto& sc : scores) {
+                sc = std::exp(sc - max_s);
+                sum += sc;
+            }
+            for (size_t s = 0; s < memory_slots; ++s) {
+                ap[b * memory_slots + s] = scores[s] / sum;
+            }
+        }
+        
+        // 检索: attention @ memory_bank
+        Tensor retrieved_mem({batch, memory_dim}, QuantType::FP32);
+        float* rp = retrieved_mem.as_fp32();
+        
+        for (size_t b = 0; b < batch; ++b) {
+            for (size_t d = 0; d < memory_dim; ++d) {
+                float val = 0;
+                for (size_t s = 0; s < memory_slots; ++s) {
+                    val += ap[b * memory_slots + s] * mp[s * memory_dim + d];
+                }
+                rp[b * memory_dim + d] = val;
+            }
+        }
+        
+        result.retrieved = retrieve_proj->forward(retrieved_mem);
+        return result;
+    }
+    
+    // 记忆巩固 (LTP模拟)
+    void consolidate(const Tensor& x) {
+        Tensor encoded = encode(x);
+        Tensor q = query_proj->forward(x);
+        
+        float* qp = q.as_fp32();
+        float* mp = memory_bank.as_fp32();
+        float* ep = encoded.as_fp32();
+        
+        size_t batch = x.shape_[0];
+        
+        // 计算注意力
+        std::vector<std::vector<float>> attentions(batch);
+        for (size_t b = 0; b < batch; ++b) {
+            attentions[b].resize(memory_slots);
+            for (size_t s = 0; s < memory_slots; ++s) {
+                float dot = 0;
+                for (size_t d = 0; d < memory_dim; ++d) {
+                    dot += qp[b * memory_dim + d] * mp[s * memory_dim + d];
+                }
+                attentions[b][s] = dot;
+            }
+            
+            float max_s = attentions[b][0];
+            for (auto& sc : attentions[b]) max_s = std::max(max_s, sc);
+            float sum = 0;
+            for (auto& sc : attentions[b]) {
+                sc = std::exp(sc - max_s);
+                sum += sc;
+            }
+            for (auto& sc : attentions[b]) sc /= sum;
+        }
+        
+        // 更新记忆槽 (加权平均)
+        for (size_t s = 0; s < memory_slots; ++s) {
+            float update = 0;
+            float weight_sum = 0;
+            for (size_t b = 0; b < batch; ++b) {
+                float w = attentions[b][s];
+                weight_sum += w;
+                for (size_t d = 0; d < memory_dim; ++d) {
+                    update += w * ep[b * memory_dim + d];
+                }
+            }
+            if (weight_sum > 0) {
+                for (size_t d = 0; d < memory_dim; ++d) {
+                    mp[s * memory_dim + d] += ltp_rate * (update / weight_sum - mp[s * memory_dim + d]);
+                }
+            }
+        }
+    }
+    
+    // 前向
+    RetrievalResult forward(const Tensor& x) {
+        auto result = retrieve(x);
+        return result;
+    }
+};
+
+/**
+ * PagedMemoryManager
+ * 
+ * 支持长记忆的分页系统：
+ * 1. 内存中的活跃页
+ * 2. 磁盘上的历史页
+ * 3. 自动页换入换出
+ */
+class PagedMemoryManager {
+public:
+    struct MemoryPage {
+        Tensor data;
+        size_t page_id;
+        size_t access_count;
+        bool in_memory;
+        std::string disk_path;
+    };
+    
+    size_t page_size;        // 每页槽数量
+    size_t max_memory_pages; // 内存最大页数
+    size_t memory_dim;
+    
+    std::unordered_map<size_t, MemoryPage> pages;
+    std::queue<size_t> page_order;  // 用于LRU
+    
+    size_t next_page_id;
+    std::string disk_dir;
+    
+    PagedMemoryManager(size_t page_sz, size_t max_pages, size_t dim, const std::string& dir = "/tmp/neuroflow_mem")
+        : page_size(page_sz), max_memory_pages(max_pages), memory_dim(dim), 
+          next_page_id(0), disk_dir(dir) {
+        // 创建磁盘目录
+        // mkdir(disk_dir.c_str(), 0755);  // 实际应用中添加
+    }
+    
+    // 创建新页
+    size_t create_page() {
+        size_t id = next_page_id++;
+        MemoryPage page;
+        page.page_id = id;
+        page.data = Tensor({page_size, memory_dim}, QuantType::FP32);
+        page.access_count = 0;
+        page.in_memory = true;
+        page.disk_path = disk_dir + "/page_" + std::to_string(id) + ".bin";
+        
+        pages[id] = page;
+        page_order.push(id);
+        
+        // 如果超过内存限制，换出最旧页
+        if (pages.size() > max_memory_pages) {
+            evict_oldest();
+        }
+        
+        return id;
+    }
+    
+    // 获取页数据
+    Tensor* get_page(size_t id) {
+        if (pages.find(id) == pages.end()) return nullptr;
+        
+        auto& page = pages[id];
+        page.access_count++;
+        
+        // 如果在磁盘，换入
+        if (!page.in_memory) {
+            load_from_disk(id);
+        }
+        
+        return &page.data;
+    }
+    
+    // 换出最旧页
+    void evict_oldest() {
+        while (page_order.size() > max_memory_pages) {
+            size_t old_id = page_order.front();
+            page_order.pop();
+            
+            auto& page = pages[old_id];
+            if (page.in_memory) {
+                save_to_disk(old_id);
+                page.in_memory = false;
+            }
+        }
+    }
+    
+    // 保存到磁盘
+    void save_to_disk(size_t id) {
+        auto& page = pages[id];
+        if (page.data.dtype_ != QuantType::FP32)
+            throw std::runtime_error("save_to_disk: page " + std::to_string(id) + " is not FP32");
+        if (!page.data.data_ || page.data.data_size_ == 0)
+            throw std::runtime_error("save_to_disk: page " + std::to_string(id) + " has no data");
+        std::ofstream f(page.disk_path, std::ios::binary);
+        if (!f) throw std::runtime_error("Cannot save page to disk: " + page.disk_path);
+        const float* data = page.data.as_fp32();
+        f.write(reinterpret_cast<const char*>(data), page.data.data_size_);
+        if (!f.good()) throw std::runtime_error("Write error saving page: " + page.disk_path);
+        f.close();
+    }
+    
+    void load_from_disk(size_t id) {
+        auto& page = pages[id];
+        if (page.data.dtype_ != QuantType::FP32)
+            throw std::runtime_error("load_from_disk: page " + std::to_string(id) + " is not FP32");
+        if (!page.data.data_ || page.data.data_size_ == 0)
+            throw std::runtime_error("load_from_disk: page " + std::to_string(id) + " has no data");
+        std::ifstream f(page.disk_path, std::ios::binary);
+        if (!f) throw std::runtime_error("Cannot load page from disk: " + page.disk_path);
+        float* data = page.data.as_fp32();
+        f.read(reinterpret_cast<char*>(data), page.data.data_size_);
+        if (!f.good()) throw std::runtime_error("Read error loading page: " + page.disk_path);
+        f.close();
+        page.in_memory = true;
+        page_order.push(id);
+    }
+    
+    // 获取统计
+    struct Stats {
+        size_t total_pages;
+        size_t in_memory_pages;
+        size_t on_disk_pages;
+        size_t total_memory_bytes;
+    };
+    
+    Stats get_stats() {
+        Stats s;
+        s.total_pages = pages.size();
+        s.in_memory_pages = 0;
+        s.on_disk_pages = 0;
+        s.total_memory_bytes = 0;
+        
+        for (auto& [id, page] : pages) {
+            if (page.in_memory) {
+                s.in_memory_pages++;
+                s.total_memory_bytes += page.data.data_size_;
+            } else {
+                s.on_disk_pages++;
+            }
+        }
+        return s;
+    }
+};
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_MEMORY_HPP

include/neuroflow/model.hpp

@@ -0,0 +1,680 @@
+#ifndef NEUROFLOW_MODEL_HPP
+#define NEUROFLOW_MODEL_HPP
+
+/**
+ * NeuroFlowModel - 主模型类
+ * 
+ * 整合三大网络：
+ * 1. ExecutiveControlNetwork (ECN)
+ * 2. DefaultModeNetwork (DMN)
+ * 3. SalienceNetwork (SN)
+ * 
+ * + 记忆模块
+ * + 神经流形分析
+ */
+
+#include <memory>
+#include <unordered_map>
+#include <vector>
+#include "memory.hpp"
+#include "networks.hpp"
+#include "tensor.hpp"
+
+namespace neuroflow {
+
+/**
+ * NeuroFlowModel - 类脑模块化神经网络
+ */
+class NeuroFlowModel {
+public:
+    // 配置
+    struct Config {
+        size_t input_dim = 512;
+        size_t hidden_dim = 2048;
+        size_t output_dim = 2048;
+        size_t memory_dim = 512;
+        size_t memory_slots = 64;
+        size_t num_layers = 12;
+        size_t num_associations = 8;
+        bool use_quantization = false;
+        bool use_mla = false;
+        size_t mla_latent_dim = 32;
+        bool use_causal_lm = false;
+        size_t vocab_size = 128000;
+        size_t max_seq_len = 512;
+        std::string tokenizer_path = "";
+        size_t causal_window_size = 64;
+        size_t sae_k = 128;
+        size_t ntm_memory_slots = 32;
+        size_t fusion_bottleneck_dim = 256;
+        size_t lm_num_attn_layers = 4;
+        size_t lm_num_attn_heads = 8;
+        size_t lm_n_kv_heads = 2;
+        bool lm_use_rope = true;
+        bool lm_use_qk_norm = true;
+        bool lm_use_swiglu = true;
+        bool lm_use_bridge = true;
+        std::string lm_pooling = "last";
+        size_t mla_n_heads = 8;
+        size_t mla_max_cache_len = 4096;
+        size_t d_model = 512;
+    };
+    
+    Config config;
+    
+    // 输入投影
+    std::shared_ptr<Linear> input_proj_linear;
+    std::shared_ptr<LayerNorm> input_proj_norm;
+    std::shared_ptr<GELU> input_proj_gelu;
+    
+    // 三大核心网络
+    std::unique_ptr<ExecutiveControlNetwork> ecn;
+    std::unique_ptr<DefaultModeNetwork> dmn;
+    std::unique_ptr<SalienceNetwork> sn;
+    
+    // 记忆模块
+    std::unique_ptr<MemoryConsolidationModule> memory;
+    std::unique_ptr<LatentKVCache> mla_cache;  // 可选MLA
+    
+    // 流形投影
+    std::shared_ptr<Linear> manifold_proj1;
+    std::shared_ptr<LayerNorm> manifold_norm;
+    std::shared_ptr<GELU> manifold_gelu;
+    std::shared_ptr<Linear> manifold_proj2;
+    
+    // 输出融合 (低秩因式分解: 15000→256→5000)
+    std::shared_ptr<Linear> output_fusion_down;
+    std::shared_ptr<LayerNorm> output_fusion_bottleneck_norm;
+    std::shared_ptr<Linear> output_fusion_up;
+    std::shared_ptr<LayerNorm> output_fusion_norm;
+    
+    // 训练模式
+    bool training_mode;
+    
+    NeuroFlowModel(const Config& cfg) : config(cfg), training_mode(false) {
+        // 输入投影
+        input_proj_linear = std::make_shared<Linear>(config.input_dim, config.hidden_dim);
+        input_proj_norm = std::make_shared<LayerNorm>(config.hidden_dim);
+        input_proj_gelu = std::make_shared<GELU>();
+        
+        // ECN
+        ecn = std::make_unique<ExecutiveControlNetwork>(
+            config.hidden_dim, config.hidden_dim, config.hidden_dim, config.num_layers);
+        
+        // DMN
+        dmn = std::make_unique<DefaultModeNetwork>(
+            config.memory_dim, config.hidden_dim / 2, config.num_associations);
+        
+        // SN
+        sn = std::make_unique<SalienceNetwork>(
+            config.hidden_dim, config.hidden_dim / 2);
+        
+        // 记忆
+        memory = std::make_unique<MemoryConsolidationModule>(
+            config.hidden_dim, config.memory_slots, config.memory_dim);
+        
+        // MLA (可选)
+        if (config.use_mla) {
+            mla_cache = std::make_unique<LatentKVCache>(
+                config.hidden_dim, 8, config.mla_latent_dim, 4096);
+        }
+        
+        // 流形投影
+        size_t manifold_in = config.hidden_dim + config.hidden_dim / 2;
+        manifold_proj1 = std::make_shared<Linear>(manifold_in, config.hidden_dim);
+        manifold_norm = std::make_shared<LayerNorm>(config.hidden_dim);
+        manifold_gelu = std::make_shared<GELU>();
+        manifold_proj2 = std::make_shared<Linear>(config.hidden_dim, 32);
+        
+        // 输出融合 (低秩因式分解: hidden_dim*3 -> bn -> hidden_dim)
+        size_t fusion_in = config.hidden_dim * 3;
+        size_t bn = config.fusion_bottleneck_dim;
+        output_fusion_down = std::make_shared<Linear>(fusion_in, bn);
+        output_fusion_bottleneck_norm = std::make_shared<LayerNorm>(bn);
+        output_fusion_up = std::make_shared<Linear>(bn, config.hidden_dim);
+        output_fusion_norm = std::make_shared<LayerNorm>(config.hidden_dim);
+        
+        // 量化
+        if (config.use_quantization) {
+            quantize();
+        }
+    }
+    
+    // 默认构造
+    NeuroFlowModel() : NeuroFlowModel(Config()) {}
+    
+    // 输出结构
+    struct Output {
+        Tensor output;           // 最终输出
+        Tensor decision;         // ECN决策
+        Tensor value;            // OFC价值
+        Tensor saliency;         // SN显著性
+        Tensor gates;            // ECN/DMN门控权重 (2-class)
+        Tensor ecn_gate;         // ECN门控
+        Tensor dmn_gate;         // DMN门控
+        Tensor anomaly;          // 异常评分
+        Tensor mem_attention;    // 记忆注意力
+        Tensor retrieved_mem;    // 检索记忆
+        Tensor manifold;         // 流形表征 (可选)
+    };
+    
+    // 前向传播
+    Output forward(const Tensor& x, const Tensor* memory_input = nullptr,
+                   bool consolidate = false, bool return_manifold = false) {
+        Output out;
+        size_t batch = x.shape_[0];
+        
+        // 输入投影
+        Tensor h = input_proj_linear->forward(x);
+        h = input_proj_norm->forward(h);
+        h = input_proj_gelu->forward(h);
+        
+        // SN: 显著性检测 + 门控
+        auto sn_out = sn->forward(h);
+        out.saliency = sn_out.saliency;
+        out.gates = sn_out.gates;
+        out.anomaly = sn_out.anomaly;
+        
+        // 提取门控权重
+        float* gates = out.gates.as_fp32();
+        Tensor ecn_gate({batch, 1}, QuantType::FP32);
+        Tensor dmn_gate({batch, 1}, QuantType::FP32);
+        for (size_t i = 0; i < batch; ++i) {
+            ecn_gate.as_fp32()[i] = gates[i * 2];
+            dmn_gate.as_fp32()[i] = gates[i * 2 + 1];
+        }
+        out.ecn_gate = ecn_gate;
+        out.dmn_gate = dmn_gate;
+        
+        // ECN: 执行推理
+        auto ecn_out = ecn->forward(h);
+        out.decision = ecn_out.decision;
+        out.value = ecn_out.value;
+        
+        // DMN: 默认模式网络
+        Tensor mem_seed;
+        if (memory_input) {
+            mem_seed = *memory_input;
+        } else {
+            mem_seed = memory->encode(h);
+        }
+        auto dmn_out = dmn->forward(mem_seed);
+        
+        // 记忆检索
+        auto mem_out = memory->forward(h);
+        out.retrieved_mem = mem_out.retrieved;
+        out.mem_attention = mem_out.attention;
+        
+        // 记忆巩固 (可选)
+        if (consolidate) {
+            memory->consolidate(h);
+        }
+        
+        // 门控加权 - 创建新的张量，避免修改共享数据
+        Tensor ecn_weighted({batch, config.hidden_dim}, QuantType::FP32);
+        Tensor dmn_weighted_full({batch, dmn_out.vision.shape_[1]}, QuantType::FP32);
+        
+        float* ew = ecn_weighted.as_fp32();
+        float* dw = dmn_weighted_full.as_fp32();
+        float* ed = out.decision.as_fp32();
+        float* dv = dmn_out.vision.as_fp32();
+        float* eg = ecn_gate.as_fp32();
+        float* dg = dmn_gate.as_fp32();
+        
+        // ECN加权
+        for (size_t i = 0; i < batch; ++i) {
+            for (size_t j = 0; j < config.hidden_dim; ++j) {
+                ew[i * config.hidden_dim + j] = ed[i * config.hidden_dim + j] * eg[i];
+            }
+        }
+        
+        // DMN加权
+        for (size_t i = 0; i < batch; ++i) {
+            for (size_t j = 0; j < dmn_out.vision.shape_[1]; ++j) {
+                dw[i * dmn_out.vision.shape_[1] + j] = dv[i * dmn_out.vision.shape_[1] + j] * dg[i];
+            }
+        }
+        
+        // 只取hidden_dim部分
+        Tensor dmn_weighted({batch, config.hidden_dim}, QuantType::FP32);
+        float* dwf = dmn_weighted.as_fp32();
+        for (size_t i = 0; i < batch; ++i) {
+            for (size_t j = 0; j < config.hidden_dim; ++j) {
+                if (j < dmn_out.vision.shape_[1]) {
+                    dwf[i * config.hidden_dim + j] = dw[i * dmn_out.vision.shape_[1] + j];
+                } else {
+                    dwf[i * config.hidden_dim + j] = 0.0f;
+                }
+            }
+        }
+        
+        // 融合: ECN + DMN + Memory
+        std::vector<Tensor> to_concat;
+        to_concat.push_back(ecn_weighted);
+        to_concat.push_back(dmn_weighted);
+        
+        Tensor mem_for_fusion = out.retrieved_mem.clone();
+        if (mem_for_fusion.shape_[1] > config.hidden_dim) {
+            // 截取 (不是reshape)
+            Tensor truncated({batch, config.hidden_dim}, QuantType::FP32);
+            float* tw = truncated.as_fp32();
+            float* mw = mem_for_fusion.as_fp32();
+            for (size_t i = 0; i < batch; ++i) {
+                for (size_t j = 0; j < config.hidden_dim; ++j) {
+                    tw[i * config.hidden_dim + j] = mw[i * mem_for_fusion.shape_[1] + j];
+                }
+            }
+            mem_for_fusion = truncated;
+        } else if (mem_for_fusion.shape_[1] < config.hidden_dim) {
+            // 补零
+            Tensor padded({batch, config.hidden_dim}, QuantType::FP32);
+            float* p = padded.as_fp32();
+            float* m = mem_for_fusion.as_fp32();
+            memset(p, 0, padded.data_size_);
+            for (size_t i = 0; i < batch; ++i) {
+                for (size_t j = 0; j < mem_for_fusion.shape_[1]; ++j) {
+                    p[i * config.hidden_dim + j] = m[i * mem_for_fusion.shape_[1] + j];
+                }
+            }
+            mem_for_fusion = padded;
+        }
+        to_concat.push_back(mem_for_fusion);
+        
+        Tensor combined = TensorOps::concat(to_concat, 1);
+        Tensor fused_bn = output_fusion_down->forward(combined);
+        fused_bn = output_fusion_bottleneck_norm->forward(fused_bn);
+        TensorOps::relu(fused_bn);
+        out.output = output_fusion_up->forward(fused_bn);
+        out.output = output_fusion_norm->forward(out.output);
+        
+        // 流形 (可选)
+        if (return_manifold) {
+            Tensor manifold_in({batch, config.hidden_dim + config.hidden_dim / 2}, QuantType::FP32);
+            float* mi = manifold_in.as_fp32();
+            float* eh = ecn_out.hidden_states.back().as_fp32();
+            float* dl = dmn_out.latent.as_fp32();
+            
+            for (size_t i = 0; i < batch; ++i) {
+                for (size_t j = 0; j < config.hidden_dim; ++j) {
+                    mi[i * manifold_in.shape_[1] + j] = eh[i * config.hidden_dim + j];
+                }
+                for (size_t j = 0; j < config.hidden_dim / 2; ++j) {
+                    mi[i * manifold_in.shape_[1] + config.hidden_dim + j] = dl[i * config.hidden_dim / 2 + j];
+                }
+            }
+            
+            Tensor m = manifold_proj1->forward(manifold_in);
+            m = manifold_norm->forward(m);
+            m = manifold_gelu->forward(m);
+            out.manifold = manifold_proj2->forward(m);
+        }
+        
+        return out;
+    }
+    
+    // 神经流形轨迹
+    std::vector<Tensor> get_manifold_trajectory(const Tensor& x, size_t steps = 10) {
+        std::vector<Tensor> trajectory;
+        Tensor current = x.clone();
+        
+        for (size_t s = 0; s < steps; ++s) {
+            auto out = forward(current, nullptr, false, true);
+            trajectory.push_back(out.manifold.clone());
+            
+            // 残差更新
+            float* c = current.as_fp32();
+            float* o = out.output.as_fp32();
+            for (size_t i = 0; i < current.shape_[0]; ++i) {
+                size_t min_dim = std::min(current.shape_[1], out.output.shape_[1]);
+                for (size_t j = 0; j < min_dim; ++j) {
+                    c[i * current.shape_[1] + j] += 0.1f * o[i * out.output.shape_[1] + j];
+                }
+            }
+        }
+        
+        return trajectory;
+    }
+    
+    // 设置训练模式
+    void set_training(bool t) {
+        training_mode = t;
+        ecn->set_training(t);
+    }
+    
+    // 量化
+    void quantize() {
+        input_proj_linear->quantize();
+        manifold_proj1->quantize();
+        manifold_proj2->quantize();
+        output_fusion_down->quantize();
+        output_fusion_up->quantize();
+        ecn->quantize();
+        dmn->quantize();
+        sn->quantize();
+        memory->encode_proj->quantize();
+        memory->retrieve_proj->quantize();
+        memory->query_proj->quantize();
+    }
+    
+    // 获取模型统计
+    struct Stats {
+        size_t total_params;
+        size_t memory_bytes;
+        float quantization_ratio;
+    };
+    
+    Stats get_stats() {
+        Stats s;
+        s.total_params = 0;
+        s.memory_bytes = 0;
+        s.quantization_ratio = 0.0f;
+        
+        // 简化统计
+        size_t fp32_layers = 0;
+        size_t quant_layers = 0;
+        
+        // 统计各层
+        auto count_linear = [&](std::shared_ptr<Linear>& l) {
+            s.total_params += l->weight.numel();
+            if (l->bias.data_) s.total_params += l->bias.numel();
+            s.memory_bytes += l->weight.data_size_ + l->bias.data_size_;
+            if (l->quantized) quant_layers++;
+            else fp32_layers++;
+        };
+        
+        count_linear(input_proj_linear);
+        count_linear(manifold_proj1);
+        count_linear(manifold_proj2);
+        count_linear(output_fusion_down);
+        count_linear(output_fusion_up);
+        
+        for (auto& l : ecn->dlpfc_linear) count_linear(l);
+        count_linear(ecn->ofc1);
+        count_linear(ecn->ofc2);
+        count_linear(ecn->vmpfc1);
+        count_linear(ecn->vmpfc2);
+        
+        count_linear(dmn->mem_encoder1);
+        count_linear(dmn->mem_encoder2);
+        count_linear(dmn->future_proj1);
+        for (auto& [h1, h2] : dmn->association_heads) {
+            count_linear(h1);
+            count_linear(h2);
+        }
+        
+        count_linear(sn->saliency1);
+        count_linear(sn->saliency2);
+        count_linear(sn->saliency3);
+        count_linear(sn->gate1);
+        count_linear(sn->gate2);
+        count_linear(sn->anomaly1);
+        count_linear(sn->anomaly2);
+        
+        count_linear(memory->encode_proj);
+        count_linear(memory->retrieve_proj);
+        count_linear(memory->query_proj);
+        
+        s.memory_bytes += memory->memory_bank.data_size_;
+        s.total_params += memory->memory_bank.numel();
+        
+        if (fp32_layers + quant_layers > 0) {
+            s.quantization_ratio = static_cast<float>(quant_layers) / (fp32_layers + quant_layers);
+        }
+        
+        return s;
+    }
+    
+    // ========== 序列化 (NFv1 格式) ==========
+    void save(const std::string& path) const {
+        std::ofstream ofs(path, std::ios::binary);
+        if (!ofs) throw std::runtime_error("Cannot open file for save: " + path);
+        
+        // Magic header
+        ofs.write("NFv1", 4);
+        
+        // Helper: serialize a named tensor
+        auto save_tensor = [&](const std::string& name, const Tensor& t) {
+            uint32_t name_len = name.size();
+            ofs.write(reinterpret_cast<const char*>(&name_len), 4);
+            ofs.write(name.data(), name_len);
+            
+            uint32_t ndim = t.shape_.size();
+            ofs.write(reinterpret_cast<const char*>(&ndim), 4);
+            for (auto d : t.shape_) {
+                uint32_t dim = d;
+                ofs.write(reinterpret_cast<const char*>(&dim), 4);
+            }
+            
+            uint32_t dsize = t.data_size_;
+            ofs.write(reinterpret_cast<const char*>(&dsize), 4);
+            ofs.write(reinterpret_cast<const char*>(t.data_.get()), dsize);
+        };
+        
+        // Helper: save Linear layer (shared_ptr)
+        auto save_linear = [&](const std::string& prefix, const std::shared_ptr<Linear>& layer) {
+            save_tensor(prefix + ".weight", layer->weight);
+            if (layer->bias.data_) save_tensor(prefix + ".bias", layer->bias);
+        };
+        
+        // === 输入投影 ===
+        save_linear("input_proj", input_proj_linear);
+        save_tensor("input_proj_norm.weight", input_proj_norm->weight);
+        save_tensor("input_proj_norm.bias", input_proj_norm->bias);
+        
+        // === ECN ===
+        for (size_t i = 0; i < ecn->dlpfc_linear.size(); ++i)
+            save_linear("ecn.dlpfc" + std::to_string(i), ecn->dlpfc_linear[i]);
+        save_linear("ecn.ofc1", ecn->ofc1);
+        save_linear("ecn.ofc2", ecn->ofc2);
+        save_linear("ecn.vmpfc1", ecn->vmpfc1);
+        save_linear("ecn.vmpfc2", ecn->vmpfc2);
+        
+        // === DMN ===
+        save_linear("dmn.mem_encoder1", dmn->mem_encoder1);
+        save_linear("dmn.mem_encoder2", dmn->mem_encoder2);
+        save_linear("dmn.future_proj1", dmn->future_proj1);
+        int head_idx = 0;
+        for (auto& [h1, h2] : dmn->association_heads) {
+            save_linear("dmn.head" + std::to_string(head_idx) + ".1", h1);
+            save_linear("dmn.head" + std::to_string(head_idx) + ".2", h2);
+            head_idx++;
+        }
+        
+        // === SN ===
+        save_linear("sn.saliency1", sn->saliency1);
+        save_linear("sn.saliency2", sn->saliency2);
+        save_linear("sn.saliency3", sn->saliency3);
+        save_linear("sn.gate1", sn->gate1);
+        save_linear("sn.gate2", sn->gate2);
+        save_linear("sn.anomaly1", sn->anomaly1);
+        save_linear("sn.anomaly2", sn->anomaly2);
+        
+        // === 记忆 ===
+        save_linear("memory.encode", memory->encode_proj);
+        save_linear("memory.retrieve", memory->retrieve_proj);
+        save_linear("memory.query_proj", memory->query_proj);
+        save_tensor("memory.bank", memory->memory_bank);
+        
+        // === 流形投影 ===
+        save_linear("manifold.proj1", manifold_proj1);
+        save_tensor("manifold.norm.weight", manifold_norm->weight);
+        save_tensor("manifold.norm.bias", manifold_norm->bias);
+        save_linear("manifold.proj2", manifold_proj2);
+        
+        // === 输出融合 ===
+        save_linear("output_fusion.down", output_fusion_down);
+        save_tensor("output_fusion.bn_norm.weight", output_fusion_bottleneck_norm->weight);
+        save_tensor("output_fusion.bn_norm.bias", output_fusion_bottleneck_norm->bias);
+        save_linear("output_fusion.up", output_fusion_up);
+        save_tensor("output_fusion.norm.weight", output_fusion_norm->weight);
+        save_tensor("output_fusion.norm.bias", output_fusion_norm->bias);
+        
+        // End marker
+        uint32_t zero = 0;
+        ofs.write(reinterpret_cast<const char*>(&zero), 4);
+        ofs.close();
+    }
+    
+    void load(const std::string& path) {
+        std::ifstream ifs(path, std::ios::binary);
+        if (!ifs) throw std::runtime_error("Cannot open file for load: " + path);
+        
+        // Magic header
+        char magic[5] = {0};
+        ifs.read(magic, 4);
+        if (std::string(magic) != "NFv1") throw std::runtime_error("Invalid model file");
+        
+        // Helper: set tensor data from stream
+        auto load_tensor_data = [&](Tensor& t) {
+            uint32_t ndim, dsize;
+            ifs.read(reinterpret_cast<char*>(&ndim), 4);
+            std::vector<size_t> shape(ndim);
+            for (uint32_t i = 0; i < ndim; ++i) {
+                uint32_t dim;
+                ifs.read(reinterpret_cast<char*>(&dim), 4);
+                shape[i] = dim;
+            }
+            ifs.read(reinterpret_cast<char*>(&dsize), 4);
+            if (t.data_size_ != dsize) {
+                t.data_ = std::shared_ptr<uint8_t>(new uint8_t[dsize], std::default_delete<uint8_t[]>());
+                t.shape_ = shape;
+                t.data_size_ = dsize;
+            }
+            ifs.read(reinterpret_cast<char*>(t.data_.get()), dsize);
+        };
+        
+        // Helper: load named Linear layer
+        auto load_linear = [&](const std::shared_ptr<Linear>& layer, const std::string& suffix) {
+            if (suffix == ".weight") load_tensor_data(layer->weight);
+            else if (suffix == ".bias") { load_tensor_data(layer->bias); }
+        };
+        
+        while (ifs.good()) {
+            uint32_t name_len;
+            ifs.read(reinterpret_cast<char*>(&name_len), 4);
+            if (name_len == 0) break;  // end marker or EOF
+            
+            std::string name(name_len, '\0');
+            ifs.read(&name[0], name_len);
+            
+            // === 输入投影 ===
+            if (name == "input_proj.weight") load_tensor_data(input_proj_linear->weight);
+            else if (name == "input_proj.bias") load_tensor_data(input_proj_linear->bias);
+            else if (name == "input_proj_norm.weight") load_tensor_data(input_proj_norm->weight);
+            else if (name == "input_proj_norm.bias") load_tensor_data(input_proj_norm->bias);
+            
+            // === ECN ===
+            else if (name.rfind("ecn.dlpfc", 0) == 0) {
+                std::string suffix = name.substr(name.find('.', 4));  // e.g. ".weight"
+                int idx = std::stoi(name.substr(9, name.find('.', 9) - 9));
+                load_linear(ecn->dlpfc_linear[idx], suffix);
+            }
+            else if (name == "ecn.ofc1.weight") load_tensor_data(ecn->ofc1->weight);
+            else if (name == "ecn.ofc1.bias") load_tensor_data(ecn->ofc1->bias);
+            else if (name == "ecn.ofc2.weight") load_tensor_data(ecn->ofc2->weight);
+            else if (name == "ecn.ofc2.bias") load_tensor_data(ecn->ofc2->bias);
+            else if (name == "ecn.vmpfc1.weight") load_tensor_data(ecn->vmpfc1->weight);
+            else if (name == "ecn.vmpfc1.bias") load_tensor_data(ecn->vmpfc1->bias);
+            else if (name == "ecn.vmpfc2.weight") load_tensor_data(ecn->vmpfc2->weight);
+            else if (name == "ecn.vmpfc2.bias") load_tensor_data(ecn->vmpfc2->bias);
+            
+            // === DMN ===
+            else if (name == "dmn.mem_encoder1.weight") load_tensor_data(dmn->mem_encoder1->weight);
+            else if (name == "dmn.mem_encoder1.bias") load_tensor_data(dmn->mem_encoder1->bias);
+            else if (name == "dmn.mem_encoder2.weight") load_tensor_data(dmn->mem_encoder2->weight);
+            else if (name == "dmn.mem_encoder2.bias") load_tensor_data(dmn->mem_encoder2->bias);
+            else if (name == "dmn.future_proj1.weight") load_tensor_data(dmn->future_proj1->weight);
+            else if (name == "dmn.future_proj1.bias") load_tensor_data(dmn->future_proj1->bias);
+            else if (name.rfind("dmn.head", 0) == 0) {
+                // dmn.head<N>.<1|2>.<weight|bias>
+                size_t dot1 = name.find('.', 8);  // after "dmn.head"
+                int h = std::stoi(name.substr(8, dot1 - 8));
+                size_t dot2 = name.find('.', dot1 + 1);
+                int which = std::stoi(name.substr(dot1 + 1, dot2 - dot1 - 1));
+                std::string suffix = name.substr(name.rfind('.'));
+                auto& layer = (which == 1) ? dmn->association_heads[h].first : dmn->association_heads[h].second;
+                load_linear(layer, suffix);
+            }
+            
+            // === SN ===
+            else if (name == "sn.saliency1.weight") load_tensor_data(sn->saliency1->weight);
+            else if (name == "sn.saliency1.bias") load_tensor_data(sn->saliency1->bias);
+            else if (name == "sn.saliency2.weight") load_tensor_data(sn->saliency2->weight);
+            else if (name == "sn.saliency2.bias") load_tensor_data(sn->saliency2->bias);
+            else if (name == "sn.saliency3.weight") load_tensor_data(sn->saliency3->weight);
+            else if (name == "sn.saliency3.bias") load_tensor_data(sn->saliency3->bias);
+            else if (name == "sn.gate1.weight") load_tensor_data(sn->gate1->weight);
+            else if (name == "sn.gate1.bias") load_tensor_data(sn->gate1->bias);
+            else if (name == "sn.gate2.weight") load_tensor_data(sn->gate2->weight);
+            else if (name == "sn.gate2.bias") load_tensor_data(sn->gate2->bias);
+            else if (name == "sn.anomaly1.weight") load_tensor_data(sn->anomaly1->weight);
+            else if (name == "sn.anomaly1.bias") load_tensor_data(sn->anomaly1->bias);
+            else if (name == "sn.anomaly2.weight") load_tensor_data(sn->anomaly2->weight);
+            else if (name == "sn.anomaly2.bias") load_tensor_data(sn->anomaly2->bias);
+            
+            // === 记忆 ===
+            else if (name == "memory.encode.weight") load_tensor_data(memory->encode_proj->weight);
+            else if (name == "memory.encode.bias") load_tensor_data(memory->encode_proj->bias);
+            else if (name == "memory.retrieve.weight") load_tensor_data(memory->retrieve_proj->weight);
+            else if (name == "memory.retrieve.bias") load_tensor_data(memory->retrieve_proj->bias);
+            else if (name == "memory.query_proj.weight") load_tensor_data(memory->query_proj->weight);
+            else if (name == "memory.query_proj.bias") load_tensor_data(memory->query_proj->bias);
+            else if (name == "memory.bank") load_tensor_data(memory->memory_bank);
+            
+            // === 流形投影 ===
+            else if (name == "manifold.proj1.weight") load_tensor_data(manifold_proj1->weight);
+            else if (name == "manifold.proj1.bias") load_tensor_data(manifold_proj1->bias);
+            else if (name == "manifold.norm.weight") load_tensor_data(manifold_norm->weight);
+            else if (name == "manifold.norm.bias") load_tensor_data(manifold_norm->bias);
+            else if (name == "manifold.proj2.weight") load_tensor_data(manifold_proj2->weight);
+            else if (name == "manifold.proj2.bias") load_tensor_data(manifold_proj2->bias);
+            
+            // === 输出融合 ===
+            else if (name == "output_fusion.down.weight") load_tensor_data(output_fusion_down->weight);
+            else if (name == "output_fusion.down.bias") load_tensor_data(output_fusion_down->bias);
+            else if (name == "output_fusion.bn_norm.weight") load_tensor_data(output_fusion_bottleneck_norm->weight);
+            else if (name == "output_fusion.bn_norm.bias") load_tensor_data(output_fusion_bottleneck_norm->bias);
+            else if (name == "output_fusion.up.weight") load_tensor_data(output_fusion_up->weight);
+            else if (name == "output_fusion.up.bias") load_tensor_data(output_fusion_up->bias);
+            else if (name == "output_fusion.norm.weight") load_tensor_data(output_fusion_norm->weight);
+            else if (name == "output_fusion.norm.bias") load_tensor_data(output_fusion_norm->bias);
+            
+            // Unknown layer — skip
+            else {
+                uint32_t ndim, dsize;
+                ifs.read(reinterpret_cast<char*>(&ndim), 4);
+                ifs.seekg(ndim * 4, std::ios::cur);
+                ifs.read(reinterpret_cast<char*>(&dsize), 4);
+                ifs.seekg(dsize, std::ios::cur);
+            }
+        }
+        ifs.close();
+    }
+};
+
+/**
+ * NeuroFlowLite - 超轻量版
+ * 适合边缘设备部署
+ */
+class NeuroFlowLite : public NeuroFlowModel {
+public:
+    NeuroFlowLite(size_t input_dim = 512) : NeuroFlowModel() {
+        Config cfg;
+        cfg.input_dim = input_dim;
+        cfg.hidden_dim = 128;
+        cfg.output_dim = 10;
+        cfg.memory_dim = 64;
+        cfg.memory_slots = 32;
+        cfg.num_layers = 1;
+        cfg.num_associations = 4;
+        cfg.use_quantization = true;
+        cfg.use_mla = true;
+        cfg.mla_latent_dim = 32;
+        
+        // 需要重新初始化...
+    }
+};
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_MODEL_HPP

include/neuroflow/multimodal.hpp

@@ -0,0 +1,441 @@
+#ifndef NEUROFLOW_MULTIMODAL_HPP
+#define NEUROFLOW_MULTIMODAL_HPP
+
+/**
+ * NeuroFlow 多模态模块
+ * 
+ * Vision-Language能力：
+ * 1. VisionEncoder - 轻量ViT风格图像编码
+ * 2. CrossModalFusion - 文本-图像对齐融合
+ * 3. MultiModalAttention - 跨模态注意力
+ * 
+ * 与类脑模块整合：
+ * - ECN处理多模态推理决策
+ * - DMN处理跨模态联想记忆
+ * - SN处理多模态显著性分配
+ */
+
+#include <cmath>
+#include <memory>
+#include <vector>
+#include "networks.hpp"
+#include "tensor.hpp"
+
+namespace neuroflow {
+
+/**
+ * PatchEmbedding - 图像Patch嵌入
+ * 
+ * 将图像分割成patch并嵌入到向量空间
+ * ViT风格，但轻量化实现
+ */
+class PatchEmbedding {
+public:
+    size_t patch_size;      // patch大小 (如16x16)
+    size_t image_size;      // 图像大小 (如224x224)
+    size_t in_channels;     // 输入通道 (如3 for RGB)
+    size_t embed_dim;       // 嵌入维度
+    size_t num_patches;     // patch数量
+    
+    std::shared_ptr<Linear> proj;  // 投影层
+    Tensor pos_embedding;          // 位置编码
+    
+    PatchEmbedding(size_t img_size = 224, size_t patch = 16, 
+                   size_t channels = 3, size_t embed = 256)
+        : image_size(img_size), patch_size(patch), 
+          in_channels(channels), embed_dim(embed) {
+        
+        // 边界检查：确保 image_size >= patch 且能整除
+        if (img_size < patch || img_size % patch != 0) {
+            throw std::invalid_argument("image_size must be >= patch_size and divisible by patch_size");
+        }
+        num_patches = (img_size / patch) * (img_size / patch);
+        
+        // 投影: patch_size*patch_size*channels -> embed_dim
+        size_t patch_dim = patch * patch * channels;
+        proj = std::make_shared<Linear>(patch_dim, embed_dim);
+        
+        // 位置编码 (可学习)
+        pos_embedding = Tensor({num_patches, embed_dim}, QuantType::FP32);
+        float* pe = pos_embedding.as_fp32();
+        // 使用正弦位置编码初始化
+        for (size_t i = 0; i < num_patches; ++i) {
+            for (size_t j = 0; j < embed_dim; ++j) {
+                if (j % 2 == 0) {
+                    pe[i * embed_dim + j] = std::sin(i / std::pow(10000, j / static_cast<float>(embed_dim)));
+                } else {
+                    pe[i * embed_dim + j] = std::cos(i / std::pow(10000, (j-1) / static_cast<float>(embed_dim)));
+                }
+            }
+        }
+    }
+    
+    // 从图像数据提取patch并嵌入
+    Tensor forward(const Tensor& image) {
+        // image: {batch, channels, height, width} 或 {batch, height, width, channels}
+        size_t batch = image.shape_[0];
+        
+        // 假设输入是 {batch, channels, height, width}
+        // 简化处理：将每个patch展平后投影
+        
+        Tensor embedded({batch * num_patches, embed_dim}, QuantType::FP32);
+        float* emb = embedded.as_fp32();
+        const float* img = image.as_fp32();
+        float* pe = pos_embedding.as_fp32();
+        
+        size_t patch_pixels = patch_size * patch_size * in_channels;
+        size_t patches_per_row = image_size / patch_size;
+        
+        // 逐patch提取并投影
+        for (size_t b = 0; b < batch; ++b) {
+            for (size_t pi = 0; pi < patches_per_row; ++pi) {
+                for (size_t pj = 0; pj < patches_per_row; ++pj) {
+                    size_t patch_idx = pi * patches_per_row + pj;
+                    
+                    // 提取patch数据 (简化版)
+                    // 实际应该从image中提取对应区域的像素
+                    // 这里简化为直接使用随机值模拟
+                    
+                    // 添加位置编码
+                    for (size_t d = 0; d < embed_dim; ++d) {
+                        emb[(b * num_patches + patch_idx) * embed_dim + d] = 
+                            pe[patch_idx * embed_dim + d]; // 初始化为位置编码
+                    }
+                }
+            }
+        }
+        
+        // 投影 (简化：直接使用嵌入)
+        // 实际应该调用 proj->forward(patch_flat)
+        
+        return embedded.reshape({batch, num_patches, embed_dim});
+    }
+};
+
+/**
+ * VisionEncoder - 轻量ViT风格图像编码器
+ * 
+ * 特点：
+ * - Patch embedding
+ * - 简化Transformer层
+ * - 输出图像特征向量
+ */
+class VisionEncoder {
+public:
+    size_t embed_dim;
+    size_t num_heads;
+    size_t num_layers;
+    size_t image_size;
+    size_t patch_size;
+    
+    std::shared_ptr<PatchEmbedding> patch_embed;
+    
+    // 简化Transformer层
+    std::vector<std::shared_ptr<Linear>> self_attn_qkv;
+    std::vector<std::shared_ptr<Linear>> self_attn_proj;
+    std::vector<std::shared_ptr<LayerNorm>> attn_norm;
+    std::vector<std::shared_ptr<Linear>> mlp_fc1;
+    std::vector<std::shared_ptr<Linear>> mlp_fc2;
+    std::vector<std::shared_ptr<LayerNorm>> mlp_norm;
+    
+    // 输出投影
+    std::shared_ptr<Linear> output_proj;
+    
+    VisionEncoder(size_t img_size = 224, size_t patch = 16,
+                  size_t embed = 256, size_t heads = 8, size_t layers = 4)
+        : image_size(img_size), patch_size(patch), 
+          embed_dim(embed), num_heads(heads), num_layers(layers) {
+        
+        // Patch embedding
+        patch_embed = std::make_shared<PatchEmbedding>(img_size, patch, 3, embed);
+        
+        // 边界检查：确保 embed >= heads
+        if (embed < heads || embed % heads != 0) {
+            throw std::invalid_argument("embed_dim must be >= num_heads and divisible by num_heads");
+        }
+        // Transformer层 (简化版)
+        size_t head_dim = embed / heads;
+        
+        for (size_t i = 0; i < layers; ++i) {
+            // Self-attention Q, K, V
+            self_attn_qkv.push_back(std::make_shared<Linear>(embed, embed * 3, false));
+            self_attn_proj.push_back(std::make_shared<Linear>(embed, embed));
+            attn_norm.push_back(std::make_shared<LayerNorm>(embed));
+            
+            // MLP
+            mlp_fc1.push_back(std::make_shared<Linear>(embed, embed * 4));
+            mlp_fc2.push_back(std::make_shared<Linear>(embed * 4, embed));
+            mlp_norm.push_back(std::make_shared<LayerNorm>(embed));
+        }
+        
+        // 输出投影 (将patch序列压缩为单一特征向量)
+        output_proj = std::make_shared<Linear>(embed, embed);
+    }
+    
+    // 前向传播
+    Tensor forward(const Tensor& image) {
+        // Patch embedding
+        Tensor x = patch_embed->forward(image);
+        size_t batch = x.shape_[0];
+        size_t num_patches = x.shape_[1];
+        
+        // Transformer层处理
+        for (size_t i = 0; i < num_layers; ++i) {
+            // Self-attention (简化实现)
+            Tensor normed = attn_norm[i]->forward(x.reshape({batch * num_patches, embed_dim}));
+            normed = normed.reshape({batch, num_patches, embed_dim});
+            
+            // QKV projection
+            Tensor qkv = self_attn_qkv[i]->forward(normed.reshape({batch * num_patches, embed_dim}));
+            // 简化：直接使用normed作为attention输出
+            
+            Tensor attn_out({batch * num_patches, embed_dim}, QuantType::FP32);
+            float* ao = attn_out.as_fp32();
+            float* n = normed.as_fp32();
+            for (size_t j = 0; j < batch * num_patches * embed_dim; ++j) {
+                ao[j] = n[j]; // 简化：identity
+            }
+            
+            attn_out = self_attn_proj[i]->forward(attn_out);
+            
+            // 残差
+            float* x_data = x.as_fp32();
+            for (size_t j = 0; j < x.numel(); ++j) {
+                x_data[j] += ao[j];
+            }
+            
+            // MLP
+            Tensor mlp_in = mlp_norm[i]->forward(x.reshape({batch * num_patches, embed_dim}));
+            Tensor mlp_hidden = mlp_fc1[i]->forward(mlp_in);
+            TensorOps::gelu(mlp_hidden);
+            Tensor mlp_out = mlp_fc2[i]->forward(mlp_hidden);
+            
+            // 残差
+            for (size_t j = 0; j < x.numel(); ++j) {
+                x_data[j] += mlp_out.as_fp32()[j];
+            }
+        }
+        
+        // 全局平均池化 + 输出投影
+        Tensor global_feat({batch, embed_dim}, QuantType::FP32);
+        float* gf = global_feat.as_fp32();
+        float* x_data = x.as_fp32();
+        
+        for (size_t b = 0; b < batch; ++b) {
+            for (size_t d = 0; d < embed_dim; ++d) {
+                float sum = 0;
+                for (size_t p = 0; p < num_patches; ++p) {
+                    sum += x_data[(b * num_patches + p) * embed_dim + d];
+                }
+                gf[b * embed_dim + d] = sum / num_patches;
+            }
+        }
+        
+        return output_proj->forward(global_feat);
+    }
+    
+    void quantize() {
+        patch_embed->proj->quantize();
+        for (auto& l : self_attn_qkv) l->quantize();
+        for (auto& l : self_attn_proj) l->quantize();
+        for (auto& l : mlp_fc1) l->quantize();
+        for (auto& l : mlp_fc2) l->quantize();
+        output_proj->quantize();
+    }
+};
+
+/**
+ * CrossModalFusion - 跨模态融合层
+ * 
+ * 将文本特征和图像特征对齐融合
+ * 类似CLIP的对比学习风格
+ */
+class CrossModalFusion {
+public:
+    size_t text_dim;
+    size_t image_dim;
+    size_t fusion_dim;
+    
+    // 文本投影到公共空间
+    std::shared_ptr<Linear> text_proj;
+    
+    // 图像投影到公共空间
+    std::shared_ptr<Linear> image_proj;
+    
+    // 融合层
+    std::shared_ptr<Linear> fusion_layer;
+    std::shared_ptr<LayerNorm> fusion_norm;
+    
+    CrossModalFusion(size_t text_d, size_t image_d, size_t fusion_d)
+        : text_dim(text_d), image_dim(image_d), fusion_dim(fusion_d) {
+        
+        text_proj = std::make_shared<Linear>(text_d, fusion_d);
+        image_proj = std::make_shared<Linear>(image_d, fusion_d);
+        fusion_layer = std::make_shared<Linear>(fusion_d * 2, fusion_d);
+        fusion_norm = std::make_shared<LayerNorm>(fusion_d);
+    }
+    
+    struct Output {
+        Tensor fused;           // 融合特征
+        Tensor text_feat;       // 文本特征 (对齐后)
+        Tensor image_feat;      // 图像特征 (对齐后)
+        Tensor similarity;      // 文本-图像相似度分数
+    };
+    
+    // 前向传播
+    Output forward(const Tensor& text_features, const Tensor& image_features) {
+        Output out;
+        size_t batch = text_features.shape_[0];
+        
+        // 投影到公共空间
+        out.text_feat = text_proj->forward(text_features);
+        out.image_feat = image_proj->forward(image_features);
+        
+        // L2归一化 (用于相似度计算)
+        float* tf = out.text_feat.as_fp32();
+        float* if_ = out.image_feat.as_fp32();
+        
+        for (size_t b = 0; b < batch; ++b) {
+            // 文本归一化
+            float t_norm = 0;
+            for (size_t d = 0; d < fusion_dim; ++d) {
+                t_norm += tf[b * fusion_dim + d] * tf[b * fusion_dim + d];
+            }
+            t_norm = std::sqrt(t_norm) + 1e-8f;
+            for (size_t d = 0; d < fusion_dim; ++d) {
+                tf[b * fusion_dim + d] /= t_norm;
+            }
+            
+            // 图像归一化
+            float i_norm = 0;
+            for (size_t d = 0; d < fusion_dim; ++d) {
+                i_norm += if_[b * fusion_dim + d] * if_[b * fusion_dim + d];
+            }
+            i_norm = std::sqrt(i_norm) + 1e-8f;
+            for (size_t d = 0; d < fusion_dim; ++d) {
+                if_[b * fusion_dim + d] /= i_norm;
+            }
+        }
+        
+        // 计算相似度 (余弦相似度)
+        out.similarity = Tensor({batch, 1}, QuantType::FP32);
+        float* sim = out.similarity.as_fp32();
+        
+        for (size_t b = 0; b < batch; ++b) {
+            float dot = 0;
+            for (size_t d = 0; d < fusion_dim; ++d) {
+                dot += tf[b * fusion_dim + d] * if_[b * fusion_dim + d];
+            }
+            sim[b] = dot; // 归一化后的余弦相似度
+        }
+        
+        // 融合特征: concat(text_feat, image_feat) -> fusion
+        Tensor concat_feat({batch, fusion_dim * 2}, QuantType::FP32);
+        float* cf = concat_feat.as_fp32();
+        
+        for (size_t b = 0; b < batch; ++b) {
+            for (size_t d = 0; d < fusion_dim; ++d) {
+                cf[b * fusion_dim * 2 + d] = tf[b * fusion_dim + d];
+                cf[b * fusion_dim * 2 + fusion_dim + d] = if_[b * fusion_dim + d];
+            }
+        }
+        
+        out.fused = fusion_layer->forward(concat_feat);
+        out.fused = fusion_norm->forward(out.fused);
+        
+        return out;
+    }
+    
+    void quantize() {
+        text_proj->quantize();
+        image_proj->quantize();
+        fusion_layer->quantize();
+    }
+};
+
+/**
+ * MultiModalAttention - 跨模态注意力
+ * 
+ * 让文本关注图像区域，图像关注文本token
+ * 类似LLaVA的cross-attention机制
+ */
+class MultiModalAttention {
+public:
+    size_t text_dim;
+    size_t image_dim;
+    size_t num_heads;
+    size_t head_dim;
+    
+    // Text -> Image cross-attention
+    std::shared_ptr<Linear> text_query;
+    std::shared_ptr<Linear> image_key;
+    std::shared_ptr<Linear> image_value;
+    std::shared_ptr<Linear> text_output;
+    
+    // Image -> Text cross-attention
+    std::shared_ptr<Linear> image_query;
+    std::shared_ptr<Linear> text_key;
+    std::shared_ptr<Linear> text_value;
+    std::shared_ptr<Linear> image_output;
+    
+    MultiModalAttention(size_t text_d, size_t image_d, size_t heads = 8)
+        : text_dim(text_d), image_dim(image_d), num_heads(heads),
+          head_dim(std::min(text_d, image_d) / heads) {
+        
+        // Text -> Image
+        text_query = std::make_shared<Linear>(text_d, num_heads * head_dim, false);
+        image_key = std::make_shared<Linear>(image_d, num_heads * head_dim, false);
+        image_value = std::make_shared<Linear>(image_d, num_heads * head_dim, false);
+        text_output = std::make_shared<Linear>(num_heads * head_dim, text_d);
+        
+        // Image -> Text
+        image_query = std::make_shared<Linear>(image_d, num_heads * head_dim, false);
+        text_key = std::make_shared<Linear>(text_d, num_heads * head_dim, false);
+        text_value = std::make_shared<Linear>(text_d, num_heads * head_dim, false);
+        image_output = std::make_shared<Linear>(num_heads * head_dim, image_d);
+    }
+    
+    // Text attends to Image
+    Tensor text_attend_image(const Tensor& text, const Tensor& image) {
+        size_t batch = text.shape_[0];
+        
+        // 简化实现：直接处理二维特征
+        // 不尝试 reshape 到三维
+        
+        Tensor query = text_query->forward(text);
+        Tensor value = image_value->forward(image);
+        
+        size_t out_dim = num_heads * head_dim;
+        
+        // 简化的attention输出
+        Tensor output({batch, out_dim}, QuantType::FP32);
+        float* q = query.as_fp32();
+        float* v = value.as_fp32();
+        float* o = output.as_fp32();
+        
+        // 简化：加权平均
+        for (size_t b = 0; b < batch; ++b) {
+            for (size_t d = 0; d < out_dim; ++d) {
+                o[b * out_dim + d] = q[b * out_dim + d] * 0.5f 
+                                  + v[b * out_dim + d] * 0.5f;
+            }
+        }
+        
+        return text_output->forward(output);
+    }
+    
+    void quantize() {
+        text_query->quantize();
+        image_key->quantize();
+        image_value->quantize();
+        text_output->quantize();
+        image_query->quantize();
+        text_key->quantize();
+        text_value->quantize();
+        image_output->quantize();
+    }
+};
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_MULTIMODAL_HPP

include/neuroflow/multimodal_model.hpp

@@ -0,0 +1,702 @@
+#ifndef NEUROFLOW_MULTIMODAL_MODEL_HPP
+#define NEUROFLOW_MULTIMODAL_MODEL_HPP
+
+#include <fstream>
+#include <string>
+
+/**
+ * NeuroFlowMultiModal - 多模态类脑模块化神经网络
+ * 
+ * 整合：
+ * 1. Vision Encoder (图像编码)
+ * 2. Cross-Modal Fusion (文本-图像融合)
+ * 3. ExecutiveControlNetwork (ECN) - 多模态推理决策
+ * 4. DefaultModeNetwork (DMN) - 跨模态联想记忆
+ * 5. SalienceNetwork (SN) - 多模态显著性分配
+ * 6. Memory Module - 长记忆存储
+ */
+
+#include "tensor.hpp"
+#include "networks.hpp"
+#include "memory.hpp"
+#include "multimodal.hpp"
+#include <vector>
+#include <memory>
+#include <unordered_map>
+
+namespace neuroflow {
+
+/**
+ * NeuroFlowMultiModal - 多模态类脑神经网络
+ */
+class NeuroFlowMultiModal {
+public:
+    // 配置
+    struct Config {
+        size_t text_dim = 512;        // 文本特征维度
+        size_t image_size = 224;      // 图像大小
+        size_t patch_size = 16;       // ViT patch大小
+        size_t vision_dim = 256;      // 视觉编码维度
+        size_t fusion_dim = 256;      // 融合维度
+        size_t hidden_dim = 256;      // 隐藏维度
+        size_t output_dim = 10;       // 输出维度
+        size_t memory_dim = 128;      // 记忆维度
+        size_t memory_slots = 64;     // 记忆槽数量
+        size_t num_layers = 2;        // ECN层数
+        size_t num_associations = 8;  // DMN联想头数量
+        size_t vision_layers = 4;     // Vision Encoder层数
+        size_t vision_heads = 8;      // Vision注意力头数
+        bool use_quantization = false;
+        bool use_mla = false;
+        size_t mla_latent_dim = 32;
+    };
+    
+    Config config;
+    
+    // ========== 多模态组件 ==========
+    std::unique_ptr<VisionEncoder> vision_encoder;
+    std::unique_ptr<CrossModalFusion> cross_modal_fusion;
+    std::unique_ptr<MultiModalAttention> multimodal_attention;
+    
+    // ========== 类脑模块 ==========
+    std::unique_ptr<ExecutiveControlNetwork> ecn;
+    std::unique_ptr<DefaultModeNetwork> dmn;
+    std::unique_ptr<SalienceNetwork> sn;
+    std::unique_ptr<MemoryConsolidationModule> memory;
+    std::unique_ptr<LatentKVCache> mla_cache;
+    
+    // ========== 输入投影 ==========
+    std::shared_ptr<Linear> text_proj;
+    std::shared_ptr<LayerNorm> text_norm;
+    
+    // ========== 融合后处理 ==========
+    std::shared_ptr<Linear> multimodal_proj;
+    std::shared_ptr<LayerNorm> multimodal_norm;
+    
+    // ========== 输出层 ==========
+    std::shared_ptr<Linear> output_layer;
+    std::shared_ptr<LayerNorm> output_norm;
+    
+    // ========== 流形投影 ==========
+    std::shared_ptr<Linear> manifold_proj1;
+    std::shared_ptr<LayerNorm> manifold_norm;
+    std::shared_ptr<Linear> manifold_proj2;
+    
+    bool training_mode;
+    
+    NeuroFlowMultiModal(const Config& cfg) : config(cfg), training_mode(false) {
+        
+        // 多模态组件
+        vision_encoder = std::make_unique<VisionEncoder>(
+            config.image_size, config.patch_size, 
+            config.vision_dim, config.vision_heads, config.vision_layers);
+        
+        cross_modal_fusion = std::make_unique<CrossModalFusion>(
+            config.text_dim, config.vision_dim, config.fusion_dim);
+        
+        multimodal_attention = std::make_unique<MultiModalAttention>(
+            config.fusion_dim, config.fusion_dim, 8);
+        
+        // 文本投影
+        text_proj = std::make_shared<Linear>(config.text_dim, config.hidden_dim);
+        text_norm = std::make_shared<LayerNorm>(config.hidden_dim);
+        
+        // 多模态融合投影
+        multimodal_proj = std::make_shared<Linear>(config.fusion_dim, config.hidden_dim);
+        multimodal_norm = std::make_shared<LayerNorm>(config.hidden_dim);
+        
+        // 类脑模块 (基于融合后的特征)
+        ecn = std::make_unique<ExecutiveControlNetwork>(
+            config.hidden_dim, config.hidden_dim, config.output_dim, config.num_layers);
+        
+        dmn = std::make_unique<DefaultModeNetwork>(
+            config.memory_dim, config.hidden_dim / 2, config.num_associations);
+        
+        sn = std::make_unique<SalienceNetwork>(
+            config.hidden_dim, config.hidden_dim / 2);
+        
+        memory = std::make_unique<MemoryConsolidationModule>(
+            config.hidden_dim, config.memory_slots, config.memory_dim);
+        
+        if (config.use_mla) {
+            mla_cache = std::make_unique<LatentKVCache>(
+                config.hidden_dim, 8, config.mla_latent_dim, 4096);
+        }
+        
+        // 输出层
+        output_layer = std::make_shared<Linear>(config.hidden_dim, config.output_dim);
+        output_norm = std::make_shared<LayerNorm>(config.output_dim);
+        
+        // 流形投影
+        manifold_proj1 = std::make_shared<Linear>(config.hidden_dim, config.hidden_dim);
+        manifold_norm = std::make_shared<LayerNorm>(config.hidden_dim);
+        manifold_proj2 = std::make_shared<Linear>(config.hidden_dim, 32);
+        
+        if (config.use_quantization) {
+            quantize();
+        }
+    }
+    
+    NeuroFlowMultiModal() : NeuroFlowMultiModal(Config()) {}
+    
+    // ========== 输出结构 ==========
+    struct Output {
+        Tensor output;              // 最终输出
+        Tensor decision;            // ECN决策
+        Tensor value;               // OFC价值
+        Tensor saliency;            // SN显著性
+        Tensor text_image_sim;      // 文本-图像相似度
+        Tensor gates;               // ECN/DMN门控
+        Tensor anomaly;             // 异常评分
+        Tensor retrieved_mem;       // 检索记忆
+        Tensor manifold;            // 流形表征
+        Tensor vision_feat;         // 视觉特征
+        Tensor text_feat;           // 文本特征 (对齐后)
+        Tensor fused_feat;          // 融合特征
+    };
+    
+    // ========== 纯文本模式 ==========
+    Output forward_text(const Tensor& text_input) {
+        Output out;
+        size_t batch = text_input.shape_[0];
+        
+        // 文本投影
+        Tensor h = text_proj->forward(text_input);
+        h = text_norm->forward(h);
+        
+        // 类脑处理 (纯文本模式)
+        auto sn_out = sn->forward(h);
+        auto ecn_out = ecn->forward(h);
+        
+        out.saliency = sn_out.saliency;
+        out.gates = sn_out.gates;
+        out.decision = ecn_out.decision;
+        out.value = ecn_out.value;
+        
+        // 记忆
+        auto mem_out = memory->forward(h);
+        out.retrieved_mem = mem_out.retrieved;
+        
+        // 输出
+        out.output = output_layer->forward(h);
+        out.output = output_norm->forward(out.output);
+        
+        return out;
+    }
+    
+    // ========== 多模态模式 (文本+图像) ==========
+    Output forward_multimodal(const Tensor& text_input, const Tensor& image_input,
+                              bool consolidate = false, bool return_manifold = false) {
+        Output out;
+        size_t batch = text_input.shape_[0];
+        
+        // 1. Vision Encoder: 图像编码
+        Tensor vision_feat = vision_encoder->forward(image_input);
+        out.vision_feat = vision_feat.clone();
+        
+        // 2. Cross-Modal Fusion: 文本-图像对齐融合
+        auto fusion_out = cross_modal_fusion->forward(text_input, vision_feat);
+        out.text_feat = fusion_out.text_feat.clone();
+        out.text_image_sim = fusion_out.similarity.clone();
+        out.fused_feat = fusion_out.fused.clone();
+        
+        // 3. 多模态融合投影
+        Tensor multimodal_h = multimodal_proj->forward(fusion_out.fused);
+        multimodal_h = multimodal_norm->forward(multimodal_h);
+        
+        // 4. Cross-Modal Attention (可选增强)
+        Tensor text_enhanced = multimodal_attention->text_attend_image(
+            text_proj->forward(text_input), vision_feat);
+        
+        // 残差融合
+        float* mh = multimodal_h.as_fp32();
+        float* te = text_enhanced.as_fp32();
+        size_t min_dim = std::min(multimodal_h.shape_[1], text_enhanced.shape_[1]);
+        for (size_t b = 0; b < batch; ++b) {
+            for (size_t d = 0; d < min_dim; ++d) {
+                mh[b * multimodal_h.shape_[1] + d] += 0.3f * te[b * text_enhanced.shape_[1] + d];
+            }
+        }
+        
+        // 5. Salience Network: 多模态显著性检测
+        auto sn_out = sn->forward(multimodal_h);
+        out.saliency = sn_out.saliency;
+        out.gates = sn_out.gates;
+        out.anomaly = sn_out.anomaly;
+        
+        // 提取门控
+        float* gates = out.gates.as_fp32();
+        Tensor ecn_gate({batch, 1}, QuantType::FP32);
+        Tensor dmn_gate({batch, 1}, QuantType::FP32);
+        for (size_t i = 0; i < batch; ++i) {
+            ecn_gate.as_fp32()[i] = gates[i * 2];
+            dmn_gate.as_fp32()[i] = gates[i * 2 + 1];
+        }
+        
+        // 6. ECN: 多模态推理决策 (模拟前额叶整合视觉信息做决策)
+        auto ecn_out = ecn->forward(multimodal_h);
+        out.decision = ecn_out.decision;
+        out.value = ecn_out.value;
+        
+        // 7. DMN: 跨模态联想记忆 (模拟后扣带回将视觉与记忆关联)
+        Tensor mem_seed = memory->encode(multimodal_h);
+        auto dmn_out = dmn->forward(mem_seed);
+        
+        // 8. Memory Retrieval
+        auto mem_out = memory->forward(multimodal_h);
+        out.retrieved_mem = mem_out.retrieved;
+        
+        if (consolidate) {
+            memory->consolidate(multimodal_h);
+        }
+        
+        // 9. 门控加权融合
+        Tensor ecn_weighted = out.decision.clone();
+        Tensor dmn_weighted = dmn_out.vision.reshape({batch, dmn_out.vision.shape_[1]});
+        
+        float* eg = ecn_gate.as_fp32();
+        float* dg = dmn_gate.as_fp32();
+        float* ew = ecn_weighted.as_fp32();
+        float* dw = dmn_weighted.as_fp32();
+        
+        for (size_t i = 0; i < batch; ++i) {
+            for (size_t j = 0; j < config.output_dim && j < ecn_weighted.shape_[1]; ++j) {
+                ew[i * ecn_weighted.shape_[1] + j] *= eg[i];
+            }
+            for (size_t j = 0; j < dmn_weighted.shape_[1]; ++j) {
+                dw[i * dmn_weighted.shape_[1] + j] *= dg[i];
+            }
+        }
+        
+        // 10. 最终融合输出
+        // 综合ECN决策 + DMN联想 + Memory + 融合特征
+        Tensor combined({batch, config.hidden_dim}, QuantType::FP32);
+        float* c = combined.as_fp32();
+        
+        // 加权组合
+        for (size_t b = 0; b < batch; ++b) {
+            for (size_t d = 0; d < config.hidden_dim; ++d) {
+                float val = 0;
+                if (d < out.decision.shape_[1]) {
+                    val += 0.3f * ew[b * out.decision.shape_[1] + d];
+                }
+                if (d < dmn_weighted.shape_[1]) {
+                    val += 0.2f * dw[b * dmn_weighted.shape_[1] + d];
+                }
+                if (d < out.retrieved_mem.shape_[1]) {
+                    val += 0.2f * out.retrieved_mem.as_fp32()[b * out.retrieved_mem.shape_[1] + d];
+                }
+                if (d < multimodal_h.shape_[1]) {
+                    val += 0.3f * mh[b * multimodal_h.shape_[1] + d];
+                }
+                c[b * config.hidden_dim + d] = val;
+            }
+        }
+        
+        out.output = output_layer->forward(combined);
+        out.output = output_norm->forward(out.output);
+        
+        // 11. 流形 (可选)
+        if (return_manifold) {
+            Tensor manifold_in({batch, config.hidden_dim}, QuantType::FP32);
+            float* mi = manifold_in.as_fp32();
+            float* eh = ecn_out.hidden_states.back().as_fp32();
+            
+            for (size_t i = 0; i < batch; ++i) {
+                for (size_t j = 0; j < config.hidden_dim && j < ecn_out.hidden_states.back().shape_[1]; ++j) {
+                    mi[i * config.hidden_dim + j] = eh[i * ecn_out.hidden_states.back().shape_[1] + j];
+                }
+                for (size_t j = ecn_out.hidden_states.back().shape_[1]; j < config.hidden_dim; ++j) {
+                    mi[i * config.hidden_dim + j] = 0;
+                }
+            }
+            
+            Tensor m = manifold_proj1->forward(manifold_in);
+            m = manifold_norm->forward(m);
+            TensorOps::gelu(m);
+            out.manifold = manifold_proj2->forward(m);
+        }
+        
+        return out;
+    }
+    
+    // ========== 通用forward接口 ==========
+    Output forward(const Tensor& text_input, const Tensor* image_input = nullptr,
+                   bool consolidate = false, bool return_manifold = false) {
+        if (image_input) {
+            return forward_multimodal(text_input, *image_input, consolidate, return_manifold);
+        } else {
+            return forward_text(text_input);
+        }
+    }
+    
+    // ========== 图像理解模式 (无文本，纯视觉推理) ==========
+    Output forward_image_only(const Tensor& image_input) {
+        Output out;
+        size_t batch = image_input.shape_[0];
+        
+        // Vision Encoder
+        Tensor vision_feat = vision_encoder->forward(image_input);
+        out.vision_feat = vision_feat.clone();
+        
+        // 直接投影到隐藏层
+        Tensor h = multimodal_proj->forward(vision_feat);
+        h = multimodal_norm->forward(h);
+        
+        // 类脑处理 (纯视觉决策)
+        auto sn_out = sn->forward(h);
+        auto ecn_out = ecn->forward(h);
+        
+        out.saliency = sn_out.saliency;
+        out.gates = sn_out.gates;
+        out.decision = ecn_out.decision;
+        out.value = ecn_out.value;
+        
+        // 记忆
+        auto mem_out = memory->forward(h);
+        out.retrieved_mem = mem_out.retrieved;
+        
+        // 输出
+        out.output = output_layer->forward(h);
+        out.output = output_norm->forward(out.output);
+        
+        return out;
+    }
+    
+    // ========== 设置训练模式 ==========
+    void set_training(bool t) {
+        training_mode = t;
+        ecn->set_training(t);
+    }
+    
+    // ========== 量化 ==========
+    void quantize() {
+        vision_encoder->quantize();
+        cross_modal_fusion->quantize();
+        multimodal_attention->quantize();
+        text_proj->quantize();
+        multimodal_proj->quantize();
+        output_layer->quantize();
+        manifold_proj1->quantize();
+        manifold_proj2->quantize();
+        ecn->quantize();
+        dmn->quantize();
+        sn->quantize();
+        memory->encode_proj->quantize();
+        memory->retrieve_proj->quantize();
+        memory->query_proj->quantize();
+    }
+    
+    // ========== 获取模型统计 ==========
+    struct Stats {
+        size_t total_params;
+        size_t vision_params;
+        size_t fusion_params;
+        size_t brain_params;
+        size_t memory_bytes;
+        float quantization_ratio;
+    };
+    
+    Stats get_stats() {
+        Stats s;
+        s.total_params = 0;
+        s.vision_params = 0;
+        s.fusion_params = 0;
+        s.brain_params = 0;
+        s.memory_bytes = 0;
+        s.quantization_ratio = 0.0f;
+        
+        auto count_linear = [&](std::shared_ptr<Linear>& l, size_t& category) {
+            s.total_params += l->weight.numel();
+            if (l->bias.data_) s.total_params += l->bias.numel();
+            category += l->weight.numel();
+            s.memory_bytes += l->weight.data_size_ + l->bias.data_size_;
+        };
+        
+        // Vision Encoder
+        count_linear(vision_encoder->patch_embed->proj, s.vision_params);
+        for (auto& l : vision_encoder->self_attn_qkv) count_linear(l, s.vision_params);
+        for (auto& l : vision_encoder->self_attn_proj) count_linear(l, s.vision_params);
+        for (auto& l : vision_encoder->mlp_fc1) count_linear(l, s.vision_params);
+        for (auto& l : vision_encoder->mlp_fc2) count_linear(l, s.vision_params);
+        count_linear(vision_encoder->output_proj, s.vision_params);
+        
+        // Cross-Modal Fusion
+        count_linear(cross_modal_fusion->text_proj, s.fusion_params);
+        count_linear(cross_modal_fusion->image_proj, s.fusion_params);
+        count_linear(cross_modal_fusion->fusion_layer, s.fusion_params);
+        
+        // Multi-Modal Attention
+        count_linear(multimodal_attention->text_query, s.fusion_params);
+        count_linear(multimodal_attention->image_key, s.fusion_params);
+        count_linear(multimodal_attention->image_value, s.fusion_params);
+        count_linear(multimodal_attention->text_output, s.fusion_params);
+        count_linear(multimodal_attention->image_query, s.fusion_params);
+        count_linear(multimodal_attention->text_key, s.fusion_params);
+        count_linear(multimodal_attention->text_value, s.fusion_params);
+        count_linear(multimodal_attention->image_output, s.fusion_params);
+        
+        // 类脑模块
+        count_linear(text_proj, s.brain_params);
+        count_linear(multimodal_proj, s.brain_params);
+        count_linear(output_layer, s.brain_params);
+        count_linear(manifold_proj1, s.brain_params);
+        count_linear(manifold_proj2, s.brain_params);
+        
+        for (auto& l : ecn->dlpfc_linear) count_linear(l, s.brain_params);
+        count_linear(ecn->ofc1, s.brain_params);
+        count_linear(ecn->ofc2, s.brain_params);
+        count_linear(ecn->vmpfc1, s.brain_params);
+        count_linear(ecn->vmpfc2, s.brain_params);
+        
+        count_linear(dmn->mem_encoder1, s.brain_params);
+        count_linear(dmn->mem_encoder2, s.brain_params);
+        count_linear(dmn->future_proj1, s.brain_params);
+        for (auto& [h1, h2] : dmn->association_heads) {
+            count_linear(h1, s.brain_params);
+            count_linear(h2, s.brain_params);
+        }
+        
+        count_linear(sn->saliency1, s.brain_params);
+        count_linear(sn->saliency2, s.brain_params);
+        count_linear(sn->saliency3, s.brain_params);
+        count_linear(sn->gate1, s.brain_params);
+        count_linear(sn->gate2, s.brain_params);
+        count_linear(sn->anomaly1, s.brain_params);
+        count_linear(sn->anomaly2, s.brain_params);
+        
+        count_linear(memory->encode_proj, s.brain_params);
+        count_linear(memory->retrieve_proj, s.brain_params);
+        count_linear(memory->query_proj, s.brain_params);
+        
+        s.memory_bytes += memory->memory_bank.data_size_;
+        s.total_params += memory->memory_bank.numel();
+        
+        return s;
+    }
+    
+    // ========== 序列化 ==========
+    void save(const std::string& path) {
+        std::ofstream ofs(path, std::ios::binary);
+        if (!ofs) throw std::runtime_error("Cannot open file for save: " + path);
+        
+        // Magic header
+        ofs.write("NFv1", 4);
+        
+        // Helper: serialize a named tensor
+        auto save_tensor = [&](const std::string& name, const Tensor& t) {
+            uint32_t name_len = name.size();
+            ofs.write(reinterpret_cast<const char*>(&name_len), 4);
+            ofs.write(name.data(), name_len);
+            
+            uint32_t ndim = t.shape_.size();
+            ofs.write(reinterpret_cast<const char*>(&ndim), 4);
+            for (auto d : t.shape_) {
+                uint32_t dim = d;
+                ofs.write(reinterpret_cast<const char*>(&dim), 4);
+            }
+            
+            uint32_t dsize = t.data_size_;
+            ofs.write(reinterpret_cast<const char*>(&dsize), 4);
+            ofs.write(reinterpret_cast<const char*>(t.data_.get()), dsize);
+        };
+        
+        // Helper: save Linear layer (accepts shared_ptr or raw ref)
+        auto save_linear = [&](const std::string& prefix, const std::shared_ptr<Linear>& layer) {
+            save_tensor(prefix + ".weight", layer->weight);
+            if (layer->bias.data_) save_tensor(prefix + ".bias", layer->bias);
+            if (layer->weight_scale.data_) save_tensor(prefix + ".weight_scale", layer->weight_scale);
+        };
+        auto save_linear_raw = [&](const std::string& prefix, const Linear& layer) {
+            save_tensor(prefix + ".weight", layer.weight);
+            if (layer.bias.data_) save_tensor(prefix + ".bias", layer.bias);
+            if (layer.weight_scale.data_) save_tensor(prefix + ".weight_scale", layer.weight_scale);
+        };
+        
+        // Helper: save LayerNorm
+        auto save_ln = [&](const std::string& prefix, const LayerNorm& ln) {
+            save_tensor(prefix + ".weight", ln.weight);
+            save_tensor(prefix + ".bias", ln.bias);
+        };
+        
+        // === 投影层 ===
+        save_linear_raw("text_proj", *text_proj);
+        save_ln("text_norm", *text_norm);
+        save_linear_raw("output_layer", *output_layer);
+        save_ln("output_norm", *output_norm);
+        
+        // === 类脑网络 ===
+        save_linear("ecn.dlpfc0", ecn->dlpfc_linear[0]);
+        save_linear("ecn.dlpfc1", ecn->dlpfc_linear[1]);
+        save_linear("ecn.ofc1", ecn->ofc1);
+        save_linear("ecn.ofc2", ecn->ofc2);
+        save_linear("ecn.vmpfc1", ecn->vmpfc1);
+        save_linear("ecn.vmpfc2", ecn->vmpfc2);
+        
+        save_linear("dmn.mem_encoder1", dmn->mem_encoder1);
+        save_linear("dmn.mem_encoder2", dmn->mem_encoder2);
+        save_linear("dmn.future_proj1", dmn->future_proj1);
+        int head_idx = 0;
+        for (auto& [h1, h2] : dmn->association_heads) {
+            save_linear("dmn.head" + std::to_string(head_idx) + ".1", h1);
+            save_linear("dmn.head" + std::to_string(head_idx) + ".2", h2);
+            head_idx++;
+        }
+        
+        save_linear("sn.saliency1", sn->saliency1);
+        save_linear("sn.saliency2", sn->saliency2);
+        save_linear("sn.saliency3", sn->saliency3);
+        save_linear("sn.gate1", sn->gate1);
+        save_linear("sn.gate2", sn->gate2);
+        save_linear("sn.anomaly1", sn->anomaly1);
+        save_linear("sn.anomaly2", sn->anomaly2);
+        
+        save_linear("memory.encode", memory->encode_proj);
+        save_linear("memory.retrieve", memory->retrieve_proj);
+        save_linear("memory.query_proj", memory->query_proj);
+        save_tensor("memory.bank", memory->memory_bank);
+        
+        // End marker
+        uint32_t zero = 0;
+        ofs.write(reinterpret_cast<const char*>(&zero), 4);
+        ofs.close();
+    }
+    
+    void load(const std::string& path) {
+        std::ifstream ifs(path, std::ios::binary);
+        if (!ifs) throw std::runtime_error("Cannot open file for load: " + path);
+        
+        // Magic header
+        char magic[5] = {0};
+        ifs.read(magic, 4);
+        if (std::string(magic) != "NFv1") throw std::runtime_error("Invalid model file");
+        
+        // Helper: load a tensor and set its data
+        auto load_tensor_data = [&](Tensor& t) {
+            uint32_t ndim;
+            ifs.read(reinterpret_cast<char*>(&ndim), 4);
+            std::vector<size_t> shape(ndim);
+            for (uint32_t i = 0; i < ndim; i++) {
+                uint32_t d;
+                ifs.read(reinterpret_cast<char*>(&d), 4);
+                shape[i] = d;
+            }
+            uint32_t dsize;
+            ifs.read(reinterpret_cast<char*>(&dsize), 4);
+            
+            // Only load if shapes match
+            if (shape == t.shape_ && dsize == t.data_size_) {
+                ifs.read(reinterpret_cast<char*>(t.data_.get()), dsize);
+            } else if (shape != t.shape_) {
+                // Skip mismatched data
+                ifs.seekg(dsize, std::ios::cur);
+            }
+        };
+        
+        // Read tensors
+        while (ifs.good()) {
+            uint32_t name_len;
+            ifs.read(reinterpret_cast<char*>(&name_len), 4);
+            if (name_len == 0 || !ifs) break;
+            
+            std::string name(name_len, '\0');
+            ifs.read(&name[0], name_len);
+            
+            // Match name to tensor
+            if (name == "text_proj.weight") load_tensor_data(text_proj->weight);
+            else if (name == "text_proj.bias") load_tensor_data(text_proj->bias);
+            else if (name == "text_norm.weight") load_tensor_data(text_norm->weight);
+            else if (name == "text_norm.bias") load_tensor_data(text_norm->bias);
+            else if (name == "output_layer.weight") load_tensor_data(output_layer->weight);
+            else if (name == "output_layer.bias") load_tensor_data(output_layer->bias);
+            else if (name == "output_norm.weight") load_tensor_data(output_norm->weight);
+            else if (name == "output_norm.bias") load_tensor_data(output_norm->bias);
+            else if (name == "ecn.dlpfc0.weight") load_tensor_data(ecn->dlpfc_linear[0]->weight);
+            else if (name == "ecn.dlpfc0.bias") load_tensor_data(ecn->dlpfc_linear[0]->bias);
+            else if (name == "ecn.dlpfc1.weight") load_tensor_data(ecn->dlpfc_linear[1]->weight);
+            else if (name == "ecn.dlpfc1.bias") load_tensor_data(ecn->dlpfc_linear[1]->bias);
+            else if (name == "ecn.ofc1.weight") load_tensor_data(ecn->ofc1->weight);
+            else if (name == "ecn.ofc1.bias") load_tensor_data(ecn->ofc1->bias);
+            else if (name == "ecn.ofc2.weight") load_tensor_data(ecn->ofc2->weight);
+            else if (name == "ecn.ofc2.bias") load_tensor_data(ecn->ofc2->bias);
+            else if (name == "ecn.vmpfc1.weight") load_tensor_data(ecn->vmpfc1->weight);
+            else if (name == "ecn.vmpfc1.bias") load_tensor_data(ecn->vmpfc1->bias);
+            else if (name == "ecn.vmpfc2.weight") load_tensor_data(ecn->vmpfc2->weight);
+            else if (name == "ecn.vmpfc2.bias") load_tensor_data(ecn->vmpfc2->bias);
+            else if (name == "dmn.mem_encoder1.weight") load_tensor_data(dmn->mem_encoder1->weight);
+            else if (name == "dmn.mem_encoder1.bias") load_tensor_data(dmn->mem_encoder1->bias);
+            else if (name == "dmn.mem_encoder2.weight") load_tensor_data(dmn->mem_encoder2->weight);
+            else if (name == "dmn.mem_encoder2.bias") load_tensor_data(dmn->mem_encoder2->bias);
+            else if (name == "dmn.future_proj1.weight") load_tensor_data(dmn->future_proj1->weight);
+            else if (name == "dmn.future_proj1.bias") load_tensor_data(dmn->future_proj1->bias);
+            else if (name.find("dmn.head") == 0) {
+                int idx = std::stoi(name.substr(9, name.find('.', 9) - 9));
+                bool is_h1 = (name[name.size()-1] == '1');
+                auto& [h1, h2] = dmn->association_heads[idx];
+                if (is_h1) load_tensor_data(h1->weight);
+                else load_tensor_data(h2->weight);
+            }
+            else if (name == "sn.saliency1.weight") load_tensor_data(sn->saliency1->weight);
+            else if (name == "sn.saliency1.bias") load_tensor_data(sn->saliency1->bias);
+            else if (name == "sn.saliency2.weight") load_tensor_data(sn->saliency2->weight);
+            else if (name == "sn.saliency2.bias") load_tensor_data(sn->saliency2->bias);
+            else if (name == "sn.saliency3.weight") load_tensor_data(sn->saliency3->weight);
+            else if (name == "sn.saliency3.bias") load_tensor_data(sn->saliency3->bias);
+            else if (name == "sn.gate1.weight") load_tensor_data(sn->gate1->weight);
+            else if (name == "sn.gate1.bias") load_tensor_data(sn->gate1->bias);
+            else if (name == "sn.gate2.weight") load_tensor_data(sn->gate2->weight);
+            else if (name == "sn.gate2.bias") load_tensor_data(sn->gate2->bias);
+            else if (name == "sn.anomaly1.weight") load_tensor_data(sn->anomaly1->weight);
+            else if (name == "sn.anomaly1.bias") load_tensor_data(sn->anomaly1->bias);
+            else if (name == "sn.anomaly2.weight") load_tensor_data(sn->anomaly2->weight);
+            else if (name == "sn.anomaly2.bias") load_tensor_data(sn->anomaly2->bias);
+            else if (name == "memory.encode.weight") load_tensor_data(memory->encode_proj->weight);
+            else if (name == "memory.encode.bias") load_tensor_data(memory->encode_proj->bias);
+            else if (name == "memory.retrieve.weight") load_tensor_data(memory->retrieve_proj->weight);
+            else if (name == "memory.retrieve.bias") load_tensor_data(memory->retrieve_proj->bias);
+            else if (name == "memory.query_proj.weight") load_tensor_data(memory->query_proj->weight);
+            else if (name == "memory.query_proj.bias") load_tensor_data(memory->query_proj->bias);
+            else if (name == "memory.bank") load_tensor_data(memory->memory_bank);
+            else {
+                // Unknown tensor: skip
+                uint32_t ndim; ifs.read(reinterpret_cast<char*>(&ndim), 4);
+                for (uint32_t i = 0; i < ndim; i++) { uint32_t d; ifs.read(reinterpret_cast<char*>(&d), 4); }
+                uint32_t dsize; ifs.read(reinterpret_cast<char*>(&dsize), 4);
+                ifs.seekg(dsize, std::ios::cur);
+            }
+        }
+        ifs.close();
+    }
+};
+
+/**
+ * NeuroFlowMultiModalLite - 超轻量多模态版
+ */
+class NeuroFlowMultiModalLite : public NeuroFlowMultiModal {
+public:
+    NeuroFlowMultiModalLite(size_t text_dim = 256, size_t image_size = 112) {
+        Config cfg;
+        cfg.text_dim = text_dim;
+        cfg.image_size = image_size;
+        cfg.patch_size = 8;
+        cfg.vision_dim = 128;
+        cfg.fusion_dim = 128;
+        cfg.hidden_dim = 128;
+        cfg.output_dim = 10;
+        cfg.memory_dim = 64;
+        cfg.memory_slots = 32;
+        cfg.num_layers = 1;
+        cfg.num_associations = 4;
+        cfg.vision_layers = 2;
+        cfg.vision_heads = 4;
+        cfg.use_quantization = true;
+        cfg.use_mla = true;
+        cfg.mla_latent_dim = 32;
+        
+        // 需要重新初始化...
+    }
+};
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_MULTIMODAL_MODEL_HPP

include/neuroflow/networks.hpp

@@ -0,0 +1,466 @@
+#ifndef NEUROFLOW_NETWORKS_HPP
+#define NEUROFLOW_NETWORKS_HPP
+
+/**
+ * NeuroFlow 核心网络模块
+ * 
+ * 1. ExecutiveControlNetwork (ECN) - 执行控制
+ * 2. DefaultModeNetwork (DMN) - 默认模式/联想
+ * 3. SalienceNetwork (SN) - 显著性检测
+ */
+
+#include <cmath>
+#include <memory>
+#include <random>
+#include <thread>
+#include <variant>
+#include <vector>
+#include "tensor.hpp"
+
+#ifdef USE_CUDA
+#include "cuda_kernels.hpp"
+#endif
+
+namespace neuroflow {
+
+/**
+ * Linear层 - 基础线性变换
+ * 支持量化权重
+ */
+class Linear {
+public:
+    Tensor weight;
+    Tensor bias;
+    Tensor weight_scale;  // 量化scale
+    bool quantized;
+    
+    Linear(size_t in_features, size_t out_features, bool use_bias = true, bool quant = false)
+        : quantized(quant) {
+        if (quant) {
+            weight = Tensor({out_features, in_features}, QuantType::INT8);
+            weight_scale = Tensor({out_features}, QuantType::FP32);
+        } else {
+            weight = Tensor({out_features, in_features}, QuantType::FP32);
+            // 初始化权重 ( Xavier )
+            float* w = weight.as_fp32();
+            float scale = std::sqrt(2.0f / (in_features + out_features));
+            size_t n = weight.numel();
+            std::mt19937 init_rng(std::hash<std::thread::id>{}(std::this_thread::get_id()) + in_features * 31 + out_features);
+            std::uniform_real_distribution<float> dist(-scale, scale);
+            for (size_t i = 0; i < n; ++i) {
+                w[i] = dist(init_rng);
+            }
+        }
+        
+        if (use_bias) {
+            bias = Tensor({out_features}, QuantType::FP32);
+            memset(bias.data_.get(), 0, bias.data_size_);
+        }
+    }
+    
+    Tensor forward(const Tensor& input) {
+        Tensor output({input.shape_[0], weight.shape_[0]}, QuantType::FP32);
+        
+        if (quantized) {
+            // 量化矩阵乘法 - INT8权重需要不同处理
+            // 这里简化为普通gemm
+            TensorOps::gemm(input, weight, output, false, true);
+        } else {
+            // weight形状是 [out_features, in_features]
+            // output = input @ weight^T
+            TensorOps::gemm(input, weight, output, false, true);
+        }
+        
+        // 加bias (只有当 bias 存在时)
+        if (bias.data_) {
+#ifdef USE_CUDA
+            if (CudaContext::instance().is_available() && output.is_on_gpu()) {
+                bias.to_gpu();
+                int rows = static_cast<int>(output.shape_[0]);
+                int cols = static_cast<int>(output.shape_[1]);
+                launch_bias_add(output.as_gpu_fp32(), bias.as_gpu_fp32(), rows, cols,
+                                CudaContext::instance().stream());
+                output.gpu_dirty_ = true;
+            } else
+#endif
+            {
+                float* out = output.as_fp32();
+                float* b = bias.as_fp32();
+                for (size_t i = 0; i < output.shape_[0]; ++i) {
+                    for (size_t j = 0; j < output.shape_[1]; ++j) {
+                        out[i * output.shape_[1] + j] += b[j];
+                    }
+                }
+            }
+        }
+        
+        return output;
+    }
+    
+    // 量化权重
+    void quantize() {
+        if (quantized) return;
+        
+        Tensor new_weight({weight.shape_[0], weight.shape_[1]}, QuantType::INT8);
+        Tensor scale({weight.shape_[0]}, QuantType::FP32);
+        
+        TensorOps::quantize_int8(weight, new_weight, scale);
+        
+        weight = new_weight;
+        weight_scale = scale;
+        quantized = true;
+    }
+};
+
+/**
+ * LayerNorm层
+ */
+class LayerNorm {
+public:
+    Tensor weight;
+    Tensor bias;
+    float eps;
+    
+    LayerNorm(size_t dim, float epsilon = 1e-5f) : eps(epsilon) {
+        weight = Tensor({dim}, QuantType::FP32);
+        bias = Tensor({dim}, QuantType::FP32);
+        
+        float* w = weight.as_fp32();
+        float* b = bias.as_fp32();
+        for (size_t i = 0; i < dim; ++i) {
+            w[i] = 1.0f;
+            b[i] = 0.0f;
+        }
+    }
+    
+    Tensor forward(const Tensor& input) {
+        Tensor output = input.clone();
+        TensorOps::layer_norm(output, weight, bias, eps);
+        return output;
+    }
+};
+
+/**
+ * GELU激活层
+ */
+class GELU {
+public:
+    Tensor forward(const Tensor& input) {
+        Tensor output = input.clone();
+        TensorOps::gelu(output);
+        return output;
+    }
+};
+
+/**
+ * Dropout层
+ */
+class Dropout {
+public:
+    float rate;
+    bool training;
+    
+    Dropout(float r = 0.1f) : rate(r), training(false) {}
+    
+    Tensor forward(const Tensor& input) {
+        Tensor output = input.clone();
+        TensorOps::dropout(output, rate, training);
+        return output;
+    }
+    
+    void set_training(bool t) { training = t; }
+};
+
+/**
+ * Sequential容器
+ */
+class Sequential {
+public:
+    using LayerVariant = std::variant<
+        std::shared_ptr<Linear>,
+        std::shared_ptr<LayerNorm>,
+        std::shared_ptr<GELU>,
+        std::shared_ptr<Dropout>
+    >;
+    
+    std::vector<LayerVariant> layers;
+    
+    template<typename T>
+    void add(std::shared_ptr<T> layer) {
+        layers.push_back(layer);
+    }
+    
+    template<typename T>
+    std::shared_ptr<T> get(size_t idx) {
+        return std::get<std::shared_ptr<T>>(layers[idx]);
+    }
+};
+
+/**
+ * ExecutiveControlNetwork (ECN)
+ * 
+ * 模拟背外侧前额叶 (dlPFC)、眶额叶皮层 (OFC)、腹内侧前额叶 (vmPFC)
+ * 功能：逻辑推理、价值评估、决策输出
+ */
+class ExecutiveControlNetwork {
+public:
+    // dlPFC: 多层处理
+    std::vector<std::shared_ptr<Linear>> dlpfc_linear;
+    std::vector<std::shared_ptr<LayerNorm>> dlpfc_norm;
+    std::vector<std::shared_ptr<GELU>> dlpfc_gelu;
+    std::vector<std::shared_ptr<Dropout>> dlpfc_drop;
+    
+    // OFC: 价值评估
+    std::shared_ptr<Linear> ofc1, ofc2;
+    
+    // vmPFC: 决策输出
+    std::shared_ptr<Linear> vmpfc1, vmpfc2;
+    
+    size_t num_layers;
+    size_t hidden_dim;
+    
+    ExecutiveControlNetwork(size_t input_dim, size_t hidden_dim, size_t output_dim, size_t layers = 2)
+        : num_layers(layers), hidden_dim(hidden_dim) {
+        
+        // dlPFC层
+        size_t prev_dim = input_dim;
+        for (size_t i = 0; i < layers; ++i) {
+            dlpfc_linear.push_back(std::make_shared<Linear>(prev_dim, hidden_dim));
+            dlpfc_norm.push_back(std::make_shared<LayerNorm>(hidden_dim));
+            dlpfc_gelu.push_back(std::make_shared<GELU>());
+            dlpfc_drop.push_back(std::make_shared<Dropout>(0.1f));
+            prev_dim = hidden_dim;
+        }
+        
+        // OFC: 价值评估 (hidden -> hidden/2 -> 1)
+        size_t half = hidden_dim / 2;
+        ofc1 = std::make_shared<Linear>(hidden_dim, half);
+        ofc2 = std::make_shared<Linear>(half, 1);
+        
+        // vmPFC: 决策 (hidden -> hidden/2 -> output)
+        vmpfc1 = std::make_shared<Linear>(hidden_dim, half);
+        vmpfc2 = std::make_shared<Linear>(half, output_dim);
+    }
+    
+    struct Output {
+        Tensor decision;   // 决策输出
+        Tensor value;      // 价值评估
+        std::vector<Tensor> hidden_states;  // 中间层激活 (用于流形分析)
+    };
+    
+    Output forward(const Tensor& x) {
+        Output out;
+        Tensor h = x;
+        
+        // dlPFC处理
+        for (size_t i = 0; i < num_layers; ++i) {
+            h = dlpfc_linear[i]->forward(h);
+            h = dlpfc_norm[i]->forward(h);
+            h = dlpfc_gelu[i]->forward(h);
+            h = dlpfc_drop[i]->forward(h);
+            out.hidden_states.push_back(h.clone());
+        }
+        
+        // OFC: 价值评估
+        Tensor v = ofc1->forward(h);
+        TensorOps::gelu(v);
+        out.value = ofc2->forward(v);
+        
+        // vmPFC: 决策
+        Tensor d = vmpfc1->forward(h);
+        TensorOps::gelu(d);
+        out.decision = vmpfc2->forward(d);
+        
+        return out;
+    }
+    
+    void set_training(bool t) {
+        for (auto& drop : dlpfc_drop) drop->set_training(t);
+    }
+    
+    // 量化所有线性层
+    void quantize() {
+        for (auto& l : dlpfc_linear) l->quantize();
+        ofc1->quantize();
+        ofc2->quantize();
+        vmpfc1->quantize();
+        vmpfc2->quantize();
+    }
+};
+
+/**
+ * DefaultModeNetwork (DMN)
+ * 
+ * 模拟后扣带回 (PCC)、内侧前额叶 (mPFC)
+ * 功能：记忆检索、未来规划、创造性联想
+ */
+class DefaultModeNetwork {
+public:
+    size_t memory_dim;
+    size_t latent_dim;
+    size_t num_associations;
+    
+    // 记忆编码
+    std::shared_ptr<Linear> mem_encoder1, mem_encoder2;
+    
+    // 联想头
+    std::vector<std::pair<std::shared_ptr<Linear>, std::shared_ptr<Linear>>> association_heads;
+    
+    // 未来投影
+    std::shared_ptr<Linear> future_proj1;
+    std::shared_ptr<LayerNorm> future_norm;
+    std::shared_ptr<GELU> future_gelu;
+    
+    DefaultModeNetwork(size_t memory_dim, size_t latent_dim, size_t num_assoc = 8)
+        : memory_dim(memory_dim), latent_dim(latent_dim), num_associations(num_assoc) {
+        
+        // 记忆编码器
+        mem_encoder1 = std::make_shared<Linear>(memory_dim, latent_dim * 2);
+        mem_encoder2 = std::make_shared<Linear>(latent_dim * 2, latent_dim);
+        
+        // 联想头
+        for (size_t i = 0; i < num_assoc; ++i) {
+            auto head1 = std::make_shared<Linear>(latent_dim, latent_dim);
+            auto head2 = std::make_shared<Linear>(latent_dim, latent_dim);
+            association_heads.push_back({head1, head2});
+        }
+        
+        // 未来投影
+        future_proj1 = std::make_shared<Linear>(latent_dim * num_assoc, latent_dim * 2);
+        future_norm = std::make_shared<LayerNorm>(latent_dim * 2);
+        future_gelu = std::make_shared<GELU>();
+    }
+    
+    struct Output {
+        Tensor vision;        // 未来愿景
+        std::vector<Tensor> associations;  // 各联想头输出
+        Tensor latent;        // 潜在记忆表征
+    };
+    
+    Output forward(const Tensor& memory_input) {
+        Output out;
+        
+        // 编码记忆
+        Tensor h = mem_encoder1->forward(memory_input);
+        TensorOps::gelu(h);
+        out.latent = mem_encoder2->forward(h);
+        
+        // 各联想头处理
+        for (auto& head : association_heads) {
+            Tensor assoc = head.first->forward(out.latent);
+            TensorOps::gelu(assoc);
+            assoc = head.second->forward(assoc);
+            out.associations.push_back(assoc);
+        }
+        
+        // 合并联想
+        out.vision = TensorOps::concat(out.associations, 1);
+        out.vision = future_proj1->forward(out.vision);
+        out.vision = future_norm->forward(out.vision);
+        out.vision = future_gelu->forward(out.vision);
+        
+        return out;
+    }
+    
+    void quantize() {
+        mem_encoder1->quantize();
+        mem_encoder2->quantize();
+        future_proj1->quantize();
+        for (auto& [h1, h2] : association_heads) {
+            h1->quantize();
+            h2->quantize();
+        }
+    }
+};
+
+/**
+ * SalienceNetwork (SN)
+ * 
+ * 模拟前岛叶 (AI)、前扣带回 (ACC)
+ * 功能：显著性检测、ECN/DMN门控、异常检测
+ */
+class SalienceNetwork {
+public:
+    // 显著性评分
+    std::shared_ptr<Linear> saliency1, saliency2, saliency3;
+    
+    // 门控生成
+    std::shared_ptr<Linear> gate1, gate2;
+    
+    // 异常检测
+    std::shared_ptr<Linear> anomaly1, anomaly2;
+    
+    SalienceNetwork(size_t input_dim, size_t hidden_dim) {
+        // 显著性评分 (sigmoid输出)
+        saliency1 = std::make_shared<Linear>(input_dim, hidden_dim);
+        saliency2 = std::make_shared<Linear>(hidden_dim, hidden_dim / 2);
+        saliency3 = std::make_shared<Linear>(hidden_dim / 2, 1);
+        
+        // 门控 (softmax 2-class)
+        gate1 = std::make_shared<Linear>(input_dim, hidden_dim);
+        gate2 = std::make_shared<Linear>(hidden_dim, 2);
+        
+        // 异常检测
+        anomaly1 = std::make_shared<Linear>(input_dim, hidden_dim);
+        anomaly2 = std::make_shared<Linear>(hidden_dim, 1);
+    }
+    
+    struct Output {
+        Tensor saliency;   // 显著性评分 [0,1]
+        Tensor gates;      // ECN/DMN门控权重
+        Tensor anomaly;    // 异常评分
+    };
+    
+    Output forward(const Tensor& x, const Tensor* baseline = nullptr) {
+        Output out;
+        
+        // 显著性
+        Tensor h = saliency1->forward(x);
+        TensorOps::gelu(h);
+        h = saliency2->forward(h);
+        TensorOps::gelu(h);
+        out.saliency = saliency3->forward(h);
+        // sigmoid
+        float* s = out.saliency.as_fp32();
+        for (size_t i = 0; i < out.saliency.numel(); ++i) {
+            s[i] = 1.0f / (1.0f + std::exp(-s[i]));
+        }
+        
+        // 门控
+        h = gate1->forward(x);
+        TensorOps::gelu(h);
+        out.gates = gate2->forward(h);
+        TensorOps::softmax(out.gates);
+        
+        // 异常
+        if (baseline) {
+            Tensor diff = x.clone();
+
+            float* d = diff.as_fp32();
+            const float* b = baseline->as_fp32();
+            for (size_t i = 0; i < diff.numel(); ++i) d[i] -= b[i];
+            
+            h = anomaly1->forward(diff);
+            TensorOps::gelu(h);
+            out.anomaly = anomaly2->forward(h);
+        } else {
+            out.anomaly = Tensor({x.shape_[0], 1}, QuantType::FP32);
+        }
+        
+        return out;
+    }
+    
+    void quantize() {
+        saliency1->quantize();
+        saliency2->quantize();
+        saliency3->quantize();
+        gate1->quantize();
+        gate2->quantize();
+        anomaly1->quantize();
+        anomaly2->quantize();
+    }
+};
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_NETWORKS_HPP

include/neuroflow/online_learning.hpp

@@ -0,0 +1,477 @@
+/**
+ * NeuroFlow 在线学习模块
+ * 
+ * 支持：
+ * 1. 少样本快速适应
+ * 2. 在线梯度更新
+ * 3. 记忆巩固 (LTP)
+ * 4. 元学习支持
+ */
+
+#pragma once
+
+#ifndef _USE_MATH_DEFINES
+#define _USE_MATH_DEFINES
+#endif
+
+#include <cmath>
+#include <memory>
+#include <vector>
+#include "backprop.hpp"
+#include "memory.hpp"
+#include "model.hpp"
+#include "networks.hpp"
+#include "tensor.hpp"
+
+namespace neuroflow {
+
+/**
+ * 损失函数
+ */
+class LossFunctions {
+public:
+    // 交叉熵损失（分类）
+    static float cross_entropy(const Tensor& pred, const Tensor& target) {
+        size_t batch = pred.shape_[0];
+        size_t classes = pred.shape_[1];
+        
+        const float* p = pred.as_fp32();
+        const float* t = target.as_fp32();
+        
+        float loss = 0.0f;
+        for (size_t b = 0; b < batch; ++b) {
+            // Softmax
+            std::vector<float> probs(classes);
+            float max_val = p[b * classes];
+            for (size_t c = 1; c < classes; ++c) {
+                max_val = std::max(max_val, p[b * classes + c]);
+            }
+            
+            float sum = 0.0f;
+            for (size_t c = 0; c < classes; ++c) {
+                probs[c] = std::exp(p[b * classes + c] - max_val);
+                sum += probs[c];
+            }
+            for (size_t c = 0; c < classes; ++c) {
+                probs[c] /= sum;
+            }
+            
+            // Cross entropy
+            for (size_t c = 0; c < classes; ++c) {
+                if (t[b * classes + c] > 0) {
+                    loss -= t[b * classes + c] * std::log(std::max(probs[c], 1e-7f));
+                }
+            }
+        }
+        
+        return loss / batch;
+    }
+    
+    // MSE损失（回归）
+    static float mse(const Tensor& pred, const Tensor& target) {
+        size_t n = pred.numel();
+        
+        const float* p = pred.as_fp32();
+        const float* t = target.as_fp32();
+        
+        float loss = 0.0f;
+        for (size_t i = 0; i < n; ++i) {
+            float diff = p[i] - t[i];
+            loss += diff * diff;
+        }
+        
+        return loss / n;
+    }
+    
+    // Softmax计算（无损失）
+    static void softmax(Tensor& x) {
+        if (x.shape_.size() != 2) return;
+        
+        size_t batch = x.shape_[0];
+        size_t classes = x.shape_[1];
+        float* p = x.as_fp32();
+        
+        for (size_t b = 0; b < batch; ++b) {
+            float max_val = p[b * classes];
+            for (size_t c = 1; c < classes; ++c) {
+                max_val = std::max(max_val, p[b * classes + c]);
+            }
+            
+            float sum = 0.0f;
+            for (size_t c = 0; c < classes; ++c) {
+                p[b * classes + c] = std::exp(p[b * classes + c] - max_val);
+                sum += p[b * classes + c];
+            }
+            
+            for (size_t c = 0; c < classes; ++c) {
+                p[b * classes + c] /= sum;
+            }
+        }
+    }
+};
+
+/**
+ * 简化版梯度计算器
+ * 
+ * 仅支持关键层的梯度：
+ * - Linear: 权重梯度 + 输入梯度
+ * - LayerNorm: 输入梯度
+ * - GELU: 输入梯度
+ */
+class GradientCalculator {
+public:
+    // Linear层梯度
+    struct LinearGradients {
+        Tensor weight_grad;   // (out, in)
+        Tensor input_grad;    // (batch, in)
+    };
+    
+    static LinearGradients linear_backward(
+        const Tensor& input,      // (batch, in_features)
+        const Tensor& output_grad, // (batch, out_features)
+        const Tensor& weight      // (out, in)
+    ) {
+        LinearGradients grads;
+        
+        size_t batch = input.shape_[0];
+        size_t in_f = input.shape_[1];
+        size_t out_f = output_grad.shape_[1];
+        
+        grads.weight_grad = Tensor({out_f, in_f}, QuantType::FP32);
+        grads.input_grad = Tensor({batch, in_f}, QuantType::FP32);
+        
+        const float* inp = input.as_fp32();
+        const float* og = output_grad.as_fp32();
+        const float* w = weight.as_fp32();
+        float* wg = grads.weight_grad.as_fp32();
+        float* ig = grads.input_grad.as_fp32();
+        
+        // 权重梯度: output_grad.T @ input
+        // weight_grad[i, j] = sum_b(output_grad[b, i] * input[b, j])
+        for (size_t i = 0; i < out_f; ++i) {
+            for (size_t j = 0; j < in_f; ++j) {
+                float sum = 0.0f;
+                for (size_t b = 0; b < batch; ++b) {
+                    sum += og[b * out_f + i] * inp[b * in_f + j];
+                }
+                wg[i * in_f + j] = sum / batch;  // 平均梯度
+            }
+        }
+        
+        // 输入梯度: output_grad @ weight
+        // input_grad[b, j] = sum_i(output_grad[b, i] * weight[i, j])
+        for (size_t b = 0; b < batch; ++b) {
+            for (size_t j = 0; j < in_f; ++j) {
+                float sum = 0.0f;
+                for (size_t i = 0; i < out_f; ++i) {
+                    sum += og[b * out_f + i] * w[i * in_f + j];
+                }
+                ig[b * in_f + j] = sum;
+            }
+        }
+        
+        return grads;
+    }
+    
+    // LayerNorm梯度
+    static Tensor layernorm_backward(
+        const Tensor& input,
+        const Tensor& output_grad,
+        float eps = 1e-5f
+    ) {
+        size_t batch = input.shape_[0];
+        size_t dim = input.shape_[1];
+        
+        Tensor input_grad({batch, dim}, QuantType::FP32);
+        
+        const float* inp = input.as_fp32();
+        const float* og = output_grad.as_fp32();
+        float* ig = input_grad.as_fp32();
+        
+        for (size_t b = 0; b < batch; ++b) {
+            // 计算均值和方差
+            float mean = 0.0f;
+            for (size_t d = 0; d < dim; ++d) {
+                mean += inp[b * dim + d];
+            }
+            mean /= dim;
+            
+            float var = 0.0f;
+            for (size_t d = 0; d < dim; ++d) {
+                float diff = inp[b * dim + d] - mean;
+                var += diff * diff;
+            }
+            var /= dim;
+            float std = std::sqrt(var + eps);
+            
+            // 梯度计算
+            float sum_grad = 0.0f;
+            float sum_grad_x = 0.0f;
+            for (size_t d = 0; d < dim; ++d) {
+                float normalized = (inp[b * dim + d] - mean) / std;
+                sum_grad += og[b * dim + d];
+                sum_grad_x += og[b * dim + d] * normalized;
+            }
+            
+            for (size_t d = 0; d < dim; ++d) {
+                float normalized = (inp[b * dim + d] - mean) / std;
+                ig[b * dim + d] = (og[b * dim + d] - sum_grad / dim - normalized * sum_grad_x / dim) / std;
+            }
+        }
+        
+        return input_grad;
+    }
+    
+    // GELU梯度
+    static Tensor gelu_backward(const Tensor& input, const Tensor& output_grad) {
+        size_t n = input.numel();
+        
+        Tensor input_grad(input.shape_, QuantType::FP32);
+        
+        const float* inp = input.as_fp32();
+        const float* og = output_grad.as_fp32();
+        float* ig = input_grad.as_fp32();
+        
+        for (size_t i = 0; i < n; ++i) {
+            float x = inp[i];
+            float gelu_grad = 0.5f * (1.0f + std::erf(x / std::sqrt(2.0f))) 
+                            + x * std::exp(-x * x / 2.0f) / std::sqrt(2.0f * 3.14159265358979323846f);
+            ig[i] = og[i] * gelu_grad;
+        }
+        
+        return input_grad;
+    }
+};
+
+/**
+ * 优化器
+ */
+class Optimizer {
+public:
+    float lr;
+    float weight_decay;
+    
+    Optimizer(float learning_rate = 0.001f, float wd = 0.0f)
+        : lr(learning_rate), weight_decay(wd) {}
+    
+    // SGD更新
+    void sgd_step(Tensor& param, const Tensor& grad) {
+        if (param.shape_ != grad.shape_) return;
+        
+        float* p = param.as_fp32();
+        const float* g = grad.as_fp32();
+        
+        for (size_t i = 0; i < param.numel(); ++i) {
+            p[i] -= lr * (g[i] + weight_decay * p[i]);
+        }
+    }
+    
+    // Adam状态
+    struct AdamState {
+        Tensor m;  // 一阶矩
+        Tensor v;  // 二阶矩
+        int t = 0;
+    };
+    
+    AdamState create_adam_state(const Tensor& param) {
+        AdamState state;
+        state.m = Tensor(param.shape_, QuantType::FP32);
+        state.v = Tensor(param.shape_, QuantType::FP32);
+        memset(state.m.as_fp32(), 0, state.m.numel() * sizeof(float));
+        memset(state.v.as_fp32(), 0, state.v.numel() * sizeof(float));
+        return state;
+    }
+    
+    // Adam更新
+    void adam_step(Tensor& param, const Tensor& grad, AdamState& state,
+                   float beta1 = 0.9f, float beta2 = 0.999f, float eps = 1e-8f) {
+        if (param.shape_ != grad.shape_) return;
+        
+        state.t++;
+        
+        float* p = param.as_fp32();
+        const float* g = grad.as_fp32();
+        float* m = state.m.as_fp32();
+        float* v = state.v.as_fp32();
+        
+        for (size_t i = 0; i < param.numel(); ++i) {
+            m[i] = beta1 * m[i] + (1 - beta1) * g[i];
+            v[i] = beta2 * v[i] + (1 - beta2) * g[i] * g[i];
+            
+            float m_hat = m[i] / (1 - std::pow(beta1, state.t));
+            float v_hat = v[i] / (1 - std::pow(beta2, state.t));
+            
+            p[i] -= lr * (m_hat / (std::sqrt(v_hat) + eps) + weight_decay * p[i]);
+        }
+    }
+};
+
+/**
+ * 在线学习器
+ * 
+ * 支持少样本快速适应
+ */
+class OnlineLearner {
+public:
+    NeuroFlowModel& model;
+    FullTrainer trainer;
+    
+    OnlineLearner(NeuroFlowModel& m, float lr = 0.01f)
+        : model(m), trainer(m, lr) {}
+    
+    // 单步在线学习
+    struct LearnResult {
+        float initial_loss;
+        float final_loss;
+        float loss_reduction;
+        int steps;
+    };
+    
+    LearnResult learn_step(
+        const Tensor& input,      // (batch, input_dim)
+        const Tensor& target,     // (batch, output_dim) 或 (batch, classes)
+        int num_steps = 5,
+        bool use_memory = true
+    ) {
+        LearnResult result;
+        result.steps = num_steps;
+        
+        // 计算初始损失
+        NeuroFlowModel::Output output = model.forward(input);
+        result.initial_loss = LossFunctions::mse(output.output, target);
+        
+        // 在线学习循环
+        for (int step = 0; step < num_steps; ++step) {
+            auto step_result = trainer.train_step(input, target);
+            (void)step_result;
+        }
+        
+        // 计算最终损失
+        NeuroFlowModel::Output final_output = model.forward(input);
+        result.final_loss = LossFunctions::mse(final_output.output, target);
+        result.loss_reduction = result.initial_loss - result.final_loss;
+        
+        return result;
+    }
+    
+    // 少样本适应
+    LearnResult few_shot_adapt(
+        const std::vector<Tensor>& examples,
+        const std::vector<Tensor>& targets,
+        int num_steps = 10
+    ) {
+        LearnResult result;
+        
+        if (examples.empty()) {
+            result.initial_loss = result.final_loss = 0.0f;
+            return result;
+        }
+        
+        // 合并为batch
+        size_t batch = examples.size();
+        Tensor input({batch, examples[0].shape_[1]}, QuantType::FP32);
+        Tensor target({batch, targets[0].shape_[1]}, QuantType::FP32);
+        
+        float* inp = input.as_fp32();
+        float* tgt = target.as_fp32();
+        
+        for (size_t b = 0; b < batch; ++b) {
+            memcpy(inp + b * examples[0].numel(), 
+                   examples[b].as_fp32(), 
+                   examples[b].numel() * sizeof(float));
+            memcpy(tgt + b * targets[0].numel(), 
+                   targets[b].as_fp32(), 
+                   targets[b].numel() * sizeof(float));
+        }
+        
+        return learn_step(input, target, num_steps, true);
+    }
+    
+    // 元学习：快速适应新任务
+    void meta_learn_step(
+        const Tensor& support_input,
+        const Tensor& support_target,
+        const Tensor& query_input,
+        const Tensor& query_target,
+        int inner_steps = 5
+    ) {
+        // 1. 内循环：在support set上适应
+        LearnResult inner = learn_step(support_input, support_target, inner_steps, true);
+        
+        // 2. 外循环：在query set上验证并更新元参数
+        NeuroFlowModel::Output query_pred = model.forward(query_input);
+        float query_loss = LossFunctions::mse(query_pred.output, query_target);
+        
+        // 更新元学习参数（简化版）
+        // 实际MAML需要二阶梯度
+        model.memory->consolidate(query_input);
+    }
+};
+
+/**
+ * 在线学习测试
+ */
+inline void test_online_learning() {
+    printf("\n=== Online Learning Test ===\n");
+    
+    // 创建模型
+    NeuroFlowModel::Config cfg;
+    cfg.input_dim = 512;
+    cfg.hidden_dim = 256;
+    cfg.output_dim = 10;
+    NeuroFlowModel model(cfg);
+    OnlineLearner learner(model, 0.01f);
+    
+    // 单样本适应
+    Tensor input(std::vector<size_t>{1, 512}, QuantType::FP32);
+    Tensor target(std::vector<size_t>{1, 10}, QuantType::FP32);
+    
+    // 初始化数据
+    float* inp = input.as_fp32();
+    float* tgt = target.as_fp32();
+    for (size_t i = 0; i < 512; ++i) inp[i] = (static_cast<float>(rand()) / RAND_MAX - 0.5f) * 0.1f;
+    for (size_t i = 0; i < 10; ++i) tgt[i] = (i == 3) ? 1.0f : 0.0f;  // 目标类别3
+    
+    // 学习
+    OnlineLearner::LearnResult result = learner.learn_step(input, target, 5, true);
+    
+    printf("  Single sample adaptation:\n");
+    printf("    Initial loss: %.4f\n", result.initial_loss);
+    printf("    Final loss: %.4f\n", result.final_loss);
+    printf("    Loss reduction: %.4f\n", result.loss_reduction);
+    
+    // 少样本适应
+    std::vector<Tensor> few_inputs(5);
+    std::vector<Tensor> few_targets(5);
+    
+    for (size_t i = 0; i < 5; ++i) {
+        few_inputs[i] = Tensor(std::vector<size_t>{1, 512}, QuantType::FP32);
+        few_targets[i] = Tensor(std::vector<size_t>{1, 10}, QuantType::FP32);
+        
+        float* fi = few_inputs[i].as_fp32();
+        float* ft = few_targets[i].as_fp32();
+        for (size_t j = 0; j < 512; ++j) fi[j] = (static_cast<float>(rand()) / RAND_MAX - 0.5f) * 0.1f;
+        for (size_t j = 0; j < 10; ++j) ft[j] = (j == (i % 10)) ? 1.0f : 0.0f;
+    }
+    
+    result = learner.few_shot_adapt(few_inputs, few_targets, 10);
+    
+    printf("  Few-shot adaptation (5 samples):\n");
+    printf("    Initial loss: %.4f\n", result.initial_loss);
+    printf("    Final loss: %.4f\n", result.final_loss);
+    
+    // 记忆巩固效果
+    printf("  Memory consolidation test:\n");
+    float mem_before = model.memory->memory_bank.as_fp32()[0];
+    
+    Tensor batch(std::vector<size_t>{32, cfg.hidden_dim}, QuantType::FP32);
+    model.memory->consolidate(batch);
+    
+    float mem_after = model.memory->memory_bank.as_fp32()[0];
+    printf("    Memory bank change: %.6f\n", std::abs(mem_after - mem_before));
+    printf("    LTP rate: %.4f\n", model.memory->ltp_rate);
+    
+    printf("=== Test Complete ===\n\n");
+}
+
+} // namespace neuroflow

include/neuroflow/rms_norm.hpp

@@ -0,0 +1,50 @@
+#ifndef NEUROFLOW_RMS_NORM_HPP
+#define NEUROFLOW_RMS_NORM_HPP
+
+#include <cstddef>
+#include "tensor.hpp"
+
+#ifdef USE_CUDA
+#include <cuda_runtime.h>
+#endif
+
+namespace neuroflow {
+
+class RMSNorm {
+public:
+    size_t dim_;
+    float eps_;
+    Tensor weight_;
+
+    struct Cache {
+        Tensor input;
+        Tensor rms;
+        Tensor normalized;
+    };
+    Cache cache_;
+
+    RMSNorm(size_t dim, float eps = 1e-5f);
+    Tensor forward(const Tensor& x);
+
+    struct Gradients {
+        Tensor weight_grad;
+        Tensor input_grad;
+    };
+
+    Gradients backward(const Tensor& output_grad);
+};
+
+#ifdef USE_CUDA
+void launch_rms_norm_forward(float* out, const float* input, const float* weight,
+                              float* rms, float* normalized, size_t batch, size_t dim,
+                              float eps, cudaStream_t stream);
+void launch_rms_norm_backward(float* input_grad, float* weight_grad,
+                               const float* output_grad, const float* input,
+                               const float* rms, const float* normalized,
+                               const float* weight, size_t batch, size_t dim,
+                               cudaStream_t stream);
+#endif
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_RMS_NORM_HPP

include/neuroflow/rope.hpp

@@ -0,0 +1,40 @@
+#ifndef NEUROFLOW_ROPE_HPP
+#define NEUROFLOW_ROPE_HPP
+
+#include <cstddef>
+#include <vector>
+#include "tensor.hpp"
+
+#ifdef USE_CUDA
+#include <cuda_runtime.h>
+#endif
+
+namespace neuroflow {
+
+class RoPE {
+public:
+    size_t head_dim_;
+    size_t max_seq_len_;
+    float yarn_scale_factor_ = 1.0f;
+    Tensor freqs_cos_;
+    Tensor freqs_sin_;
+
+    RoPE(size_t head_dim, size_t max_seq_len);
+
+    void apply(Tensor& qkv, size_t seq_len, size_t n_heads, size_t d_model, size_t offset = 0) const;
+    void apply_single(Tensor& x, size_t seq_len, size_t n_heads, size_t offset = 0) const;
+    void set_yarn_scale(float scale_factor);
+};
+
+#ifdef USE_CUDA
+void launch_rope(float* qkv, const float* cos_table, const float* sin_table,
+                 size_t seq_len, size_t n_heads, size_t d_model, size_t head_dim,
+                 size_t offset, cudaStream_t stream);
+void launch_rope_single(float* data, const float* cos_table, const float* sin_table,
+                         size_t seq_len, size_t n_heads, size_t stride, size_t head_dim,
+                         size_t offset, cudaStream_t stream);
+#endif
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_ROPE_HPP

include/neuroflow/sampling.hpp

@@ -0,0 +1,73 @@
+#ifndef NEUROFLOW_SAMPLING_HPP
+#define NEUROFLOW_SAMPLING_HPP
+
+#include <algorithm>
+#include <cmath>
+#include <iostream>
+#include <memory>
+#include <numeric>
+#include <random>
+#include <vector>
+#include "tensor.hpp"
+
+namespace neuroflow {
+
+enum class SamplingStrategyType : uint8_t {
+    GREEDY = 0,
+    TOP_K = 1,
+    TOP_P = 2,
+    TOP_K_TOP_P = 3
+};
+
+struct GenerateConfig {
+    size_t max_new_tokens = 50;
+    float temperature = 1.0f;
+    size_t top_k = 40;
+    float top_p = 0.9f;
+    float repetition_penalty = 1.0f;
+    float punct_penalty = 0.0f;
+    std::vector<size_t> punct_ids;
+    size_t random_seed = 0;
+    SamplingStrategyType strategy = SamplingStrategyType::TOP_K;
+    size_t eos_id = 3;
+};
+
+class SamplingStrategy {
+public:
+    virtual ~SamplingStrategy() = default;
+    virtual Tensor apply(Tensor logits, const GenerateConfig& config,
+                         const std::vector<size_t>& generated) = 0;
+    virtual size_t sample(const Tensor& probs, std::mt19937& rng) const = 0;
+};
+
+class GreedyDecoding : public SamplingStrategy {
+public:
+    Tensor apply(Tensor logits, const GenerateConfig& config,
+                 const std::vector<size_t>& generated) override;
+    size_t sample(const Tensor& probs, std::mt19937& rng) const override;
+};
+
+class TopKSampling : public SamplingStrategy {
+public:
+    Tensor apply(Tensor logits, const GenerateConfig& config,
+                 const std::vector<size_t>& generated) override;
+    size_t sample(const Tensor& probs, std::mt19937& rng) const override;
+};
+
+class TopPSampling : public SamplingStrategy {
+public:
+    Tensor apply(Tensor logits, const GenerateConfig& config,
+                 const std::vector<size_t>& generated) override;
+    size_t sample(const Tensor& probs, std::mt19937& rng) const override;
+};
+
+class TopKTopPSampling : public SamplingStrategy {
+public:
+    Tensor apply(Tensor logits, const GenerateConfig& config,
+                 const std::vector<size_t>& generated) override;
+    size_t sample(const Tensor& probs, std::mt19937& rng) const override;
+};
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_SAMPLING_HPP

include/neuroflow/scheduler.hpp

@@ -0,0 +1,26 @@
+#ifndef NEUROFLOW_SCHEDULER_HPP
+#define NEUROFLOW_SCHEDULER_HPP
+
+#include <cstddef>
+
+namespace neuroflow {
+
+class CosineScheduler {
+public:
+    enum class Phase { WARMUP, COSINE, DONE };
+
+    float lr_max_;
+    float lr_min_;
+    size_t warmup_steps_;
+    size_t total_steps_;
+
+    CosineScheduler(float lr_max, size_t total_steps,
+                    float lr_min_ratio = 0.1f, float warmup_ratio = 0.01f);
+
+    float get_lr(size_t step) const;
+    Phase get_phase(size_t step) const;
+};
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_SCHEDULER_HPP

include/neuroflow/sft.hpp

@@ -0,0 +1,85 @@
+#ifndef NEUROFLOW_SFT_HPP
+#define NEUROFLOW_SFT_HPP
+
+#include <cstddef>
+#include <random>
+#include <string>
+#include <vector>
+
+#include "adamw.hpp"
+#include "alignment_common.hpp"
+#include "causal_lm.hpp"
+#include "scheduler.hpp"
+#include "tokenizer.hpp"
+
+namespace neuroflow {
+
+struct SFTTrainConfig {
+    std::string data_path;
+    std::string ckpt_path;
+    std::string tokenizer_path;
+    std::string output_dir;
+    float learning_rate = 2e-5f;
+    int epochs = 3;
+    size_t max_seq_len = 512;
+    float warmup_ratio = 0.05f;
+    float weight_decay = 0.01f;
+    float grad_clip = 1.0f;
+    float adam_beta1 = 0.9f;
+    float adam_beta2 = 0.999f;
+    float adam_eps = 1e-8f;
+    size_t save_interval = 1000;
+    size_t log_interval = 10;
+    unsigned seed = 42;
+};
+
+class SFTDataLoader {
+public:
+    SFTDataLoader(const std::string& jsonl_path, size_t max_samples = 0);
+
+    bool has_next() const;
+    SFTSample next();
+    void reset();
+    void shuffle(std::mt19937& rng);
+    size_t total_samples() const { return samples_.size(); }
+    size_t invalid_count() const { return invalid_count_; }
+
+private:
+    std::vector<SFTSample> samples_;
+    size_t cursor_ = 0;
+    size_t invalid_count_ = 0;
+};
+
+struct MaskedCEOutput {
+    float loss;
+    Tensor logits_grad;
+    size_t valid_token_count;
+};
+
+MaskedCEOutput masked_cross_entropy(const Tensor& logits, size_t vocab_size,
+                                     const std::vector<size_t>& target_ids,
+                                     const std::vector<float>& loss_mask);
+
+SFTTrainingTensors build_sft_training_tensors(const SFTSample& sample,
+                                                BPETokenizer& tokenizer,
+                                                size_t max_seq_len);
+
+class SFTTrainer {
+public:
+    SFTTrainConfig config;
+
+    SFTTrainer(const SFTTrainConfig& cfg);
+
+    void train();
+    float train_on_sample(const SFTSample& sample);
+
+private:
+    std::unique_ptr<CausalLMHead> model_;
+    std::unique_ptr<BPETokenizer> tokenizer_;
+    std::unique_ptr<AdamW> optimizer_;
+    std::unique_ptr<CosineScheduler> scheduler_;
+};
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_SFT_HPP

include/neuroflow/swiglu.hpp

@@ -0,0 +1,55 @@
+#ifndef NEUROFLOW_SWIGLU_HPP
+#define NEUROFLOW_SWIGLU_HPP
+
+#include <cstddef>
+#include "tensor.hpp"
+#include "model.hpp"
+
+#ifdef USE_CUDA
+#include <cuda_runtime.h>
+#endif
+
+namespace neuroflow {
+
+class SwiGLUFFN {
+public:
+    size_t d_model_;
+    size_t d_ff_;
+    std::shared_ptr<Linear> w_gate_;
+    std::shared_ptr<Linear> w_up_;
+    std::shared_ptr<Linear> w_down_;
+
+    struct Cache {
+        Tensor input;
+        Tensor gate_out;
+        Tensor up_out;
+        Tensor gate_activated;
+        Tensor multiplied;
+    };
+    Cache cache_;
+    bool training_mode_ = false;
+
+    SwiGLUFFN(size_t d_model, size_t d_ff = 0);
+    Tensor forward(const Tensor& x);
+
+    struct Gradients {
+        Tensor w_gate_weight_grad;
+        Tensor w_gate_bias_grad;
+        Tensor w_up_weight_grad;
+        Tensor w_up_bias_grad;
+        Tensor w_down_weight_grad;
+        Tensor w_down_bias_grad;
+        Tensor input_grad;
+    };
+
+    Gradients backward(const Tensor& output_grad);
+};
+
+#ifdef USE_CUDA
+void launch_silu(float* data, size_t n, cudaStream_t stream);
+void launch_elementwise_mul(float* out, const float* a, const float* b, size_t n, cudaStream_t stream);
+#endif
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_SWIGLU_HPP

include/neuroflow/tensor.hpp

@@ -0,0 +1,327 @@
+#ifndef NEUROFLOW_TENSOR_HPP
+#define NEUROFLOW_TENSOR_HPP
+
+/**
+ * NeuroFlow Core Tensor Engine
+ * 
+ * 轻量化高性能张量计算库
+ * 特点：
+ * - SIMD优化 (AVX2/ARM NEON)
+ * - 零拷贝内存管理
+ * - 内存映射支持
+ * - INT8/FP8量化支持
+ */
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <cstring>
+#include <functional>
+#include <memory>
+#include <random>
+#include <stdexcept>
+#include <thread>
+#include <vector>
+
+#ifdef USE_CBLAS
+extern "C" {
+#include <cblas.h>
+}
+#define cblas_sgemm scipy_cblas_sgemm
+#endif
+
+#ifdef USE_CUDA
+#include "cuda_context.hpp"
+#endif
+
+
+#ifdef __AVX2__
+#include <immintrin.h>
+#define HAS_AVX2 1
+#endif
+
+#ifdef __ARM_NEON
+#include <arm_neon.h>
+#define HAS_NEON 1
+#endif
+
+namespace neuroflow {
+
+// 量化类型枚举
+enum class QuantType : uint8_t {
+    FP32 = 0,
+    FP16 = 1,
+    INT8 = 2,
+    INT4 = 3,
+    FP8_E4M3 = 4,  // DeepSeek FP8
+    FP8_E5M2 = 5,
+};
+
+// 内存布局
+enum class MemoryLayout : uint8_t {
+    ROW_MAJOR = 0,
+    COL_MAJOR = 1,
+};
+
+/**
+ * Tensor类 - 核心数据结构
+ * 支持多种量化格式和内存布局
+ */
+class Tensor {
+public:
+    std::vector<size_t> shape_;
+    std::vector<size_t> strides_;
+    QuantType dtype_;
+    MemoryLayout layout_;
+    std::shared_ptr<uint8_t> data_;
+    size_t data_size_;
+    // 语义标记：指示此 Tensor 是否"拥有" data_ 的逻辑所有权。
+    // 注意：data_ 是 shared_ptr，实际内存由引用计数管理，析构函数不检查 owns_data_。
+    // owns_data_ 仅用于 tie_weights() 等场景的所有权文档化，不影响内存释放。
+    bool owns_data_;
+
+#ifdef USE_CUDA
+    void* gpu_data_ = nullptr;
+    bool on_gpu_ = false;
+    bool gpu_dirty_ = false;
+
+    void to_gpu() {
+        if (data_size_ == 0 || !data_) return;
+        auto& ctx = CudaContext::instance();
+        if (!ctx.is_available()) return;
+        if (!gpu_data_) {
+            gpu_data_ = ctx.alloc(data_size_);
+        }
+        ctx.copy_h2d(gpu_data_, data_.get(), data_size_);
+        on_gpu_ = true;
+        gpu_dirty_ = false;
+    }
+
+    void to_cpu() {
+        if (!on_gpu_ || !gpu_data_ || data_size_ == 0) return;
+        auto& ctx = CudaContext::instance();
+        if (!ctx.is_available()) return;
+        if (!data_) {
+            data_ = std::shared_ptr<uint8_t>(new uint8_t[data_size_], std::default_delete<uint8_t[]>());
+        }
+        ctx.copy_d2h(data_.get(), gpu_data_, data_size_);
+        ctx.synchronize();
+        gpu_dirty_ = false;
+    }
+
+    float* as_gpu_fp32() {
+        if (dtype_ != QuantType::FP32) throw std::runtime_error("Not FP32 tensor");
+        return reinterpret_cast<float*>(gpu_data_);
+    }
+
+    const float* as_gpu_fp32() const {
+        if (dtype_ != QuantType::FP32) throw std::runtime_error("Not FP32 tensor");
+        return reinterpret_cast<const float*>(gpu_data_);
+    }
+
+    bool is_on_gpu() const { return on_gpu_; }
+
+    void gpu_free() {
+        if (gpu_data_) {
+            auto& ctx = CudaContext::instance();
+            if (ctx.is_available()) ctx.free(gpu_data_);
+            gpu_data_ = nullptr;
+            on_gpu_ = false;
+            gpu_dirty_ = false;
+        }
+    }
+#endif
+    
+    Tensor() : dtype_(QuantType::FP32), layout_(MemoryLayout::ROW_MAJOR), 
+               data_size_(0), owns_data_(true) {}
+
+    ~Tensor() {
+#ifdef USE_CUDA
+        gpu_free();
+#endif
+    }
+
+    Tensor(const Tensor& other)
+        : shape_(other.shape_), strides_(other.strides_), dtype_(other.dtype_),
+          layout_(other.layout_), data_(other.data_), data_size_(other.data_size_),
+          owns_data_(false)
+#ifdef USE_CUDA
+          , gpu_data_(other.gpu_data_), on_gpu_(other.on_gpu_), gpu_dirty_(other.gpu_dirty_)
+#endif
+    {}
+
+    Tensor& operator=(const Tensor& other) {
+        if (this != &other) {
+#ifdef USE_CUDA
+            gpu_free();
+#endif
+            shape_ = other.shape_;
+            strides_ = other.strides_;
+            dtype_ = other.dtype_;
+            layout_ = other.layout_;
+            data_ = other.data_;
+            data_size_ = other.data_size_;
+            owns_data_ = false;
+#ifdef USE_CUDA
+            gpu_data_ = other.gpu_data_;
+            on_gpu_ = other.on_gpu_;
+            gpu_dirty_ = other.gpu_dirty_;
+#endif
+        }
+        return *this;
+    }
+
+    Tensor(Tensor&& other) noexcept
+        : shape_(std::move(other.shape_)), strides_(std::move(other.strides_)),
+          dtype_(other.dtype_), layout_(other.layout_), data_(std::move(other.data_)),
+          data_size_(other.data_size_), owns_data_(other.owns_data_)
+#ifdef USE_CUDA
+          , gpu_data_(other.gpu_data_), on_gpu_(other.on_gpu_), gpu_dirty_(other.gpu_dirty_)
+#endif
+    {
+        other.data_size_ = 0;
+        other.owns_data_ = false;
+#ifdef USE_CUDA
+        other.gpu_data_ = nullptr;
+        other.on_gpu_ = false;
+        other.gpu_dirty_ = false;
+#endif
+    }
+
+    Tensor& operator=(Tensor&& other) noexcept {
+        if (this != &other) {
+#ifdef USE_CUDA
+            gpu_free();
+#endif
+            shape_ = std::move(other.shape_);
+            strides_ = std::move(other.strides_);
+            dtype_ = other.dtype_;
+            layout_ = other.layout_;
+            data_ = std::move(other.data_);
+            data_size_ = other.data_size_;
+            owns_data_ = other.owns_data_;
+#ifdef USE_CUDA
+            gpu_data_ = other.gpu_data_;
+            on_gpu_ = other.on_gpu_;
+            gpu_dirty_ = other.gpu_dirty_;
+            other.gpu_data_ = nullptr;
+            other.on_gpu_ = false;
+            other.gpu_dirty_ = false;
+#endif
+            other.data_size_ = 0;
+            other.owns_data_ = false;
+        }
+        return *this;
+    }
+    
+    Tensor(const std::vector<size_t>& dims, QuantType type = QuantType::FP32)
+        : shape_(dims), dtype_(type), layout_(MemoryLayout::ROW_MAJOR), owns_data_(true) {
+        strides_.resize(dims.size());
+        size_t total = 1;
+        for (size_t i = dims.size(); i > 0; --i) {
+            strides_[i-1] = total;
+            total *= dims[i-1];
+        }
+        data_size_ = total * get_type_size();
+        data_ = std::shared_ptr<uint8_t>(new uint8_t[data_size_], std::default_delete<uint8_t[]>());
+        memset(data_.get(), 0, data_size_);
+    }
+    
+    // 类型大小
+    size_t get_type_size() const {
+        switch (dtype_) {
+            case QuantType::FP32: return 4;
+            case QuantType::FP16: return 2;
+            case QuantType::INT8: return 1;
+            case QuantType::INT4: return 1; // packed
+            case QuantType::FP8_E4M3: return 1;
+            case QuantType::FP8_E5M2: return 1;
+            default: return 4;
+        }
+    }
+    
+    // 总元素数
+    size_t numel() const {
+        size_t n = 1;
+        for (auto d : shape_) n *= d;
+        return n;
+    }
+    
+    // FP32数据访问
+    float* as_fp32() {
+        if (dtype_ != QuantType::FP32) throw std::runtime_error("Not FP32 tensor");
+        return reinterpret_cast<float*>(data_.get());
+    }
+    
+    const float* as_fp32() const {
+        if (dtype_ != QuantType::FP32) throw std::runtime_error("Not FP32 tensor");
+        return reinterpret_cast<const float*>(data_.get());
+    }
+    
+    // 别名，兼容性
+    const float* as_fp32_const() const {
+        return as_fp32();
+    }
+    
+    // INT8数据访问
+    int8_t* as_int8() {
+        if (dtype_ != QuantType::INT8) throw std::runtime_error("Not INT8 tensor");
+        return reinterpret_cast<int8_t*>(data_.get());
+    }
+    
+    const int8_t* as_int8() const {
+        if (dtype_ != QuantType::INT8) throw std::runtime_error("Not INT8 tensor");
+        return reinterpret_cast<const int8_t*>(data_.get());
+    }
+    
+    // reshape (零拷贝)
+    Tensor reshape(const std::vector<size_t>& new_shape) const {
+        Tensor t;
+        t.shape_ = new_shape;
+        t.dtype_ = dtype_;
+        t.layout_ = layout_;
+        t.data_ = data_;
+        t.data_size_ = data_size_;
+        t.owns_data_ = false;
+#ifdef USE_CUDA
+        t.gpu_data_ = gpu_data_;
+        t.on_gpu_ = on_gpu_;
+        t.gpu_dirty_ = gpu_dirty_;
+#endif
+        
+        size_t total = 1;
+        for (auto d : new_shape) total *= d;
+        if (total != numel()) throw std::runtime_error("Invalid reshape");
+        
+        t.strides_.resize(new_shape.size());
+        size_t acc = 1;
+        for (size_t i = new_shape.size(); i > 0; --i) {
+            t.strides_[i-1] = acc;
+            acc *= new_shape[i-1];
+        }
+        return t;
+    }
+    
+    Tensor clone() const {
+        Tensor t(shape_, dtype_);
+        memcpy(t.data_.get(), data_.get(), data_size_);
+#ifdef USE_CUDA
+        if (on_gpu_ && gpu_data_) {
+            auto& ctx = CudaContext::instance();
+            if (ctx.is_available()) {
+                t.gpu_data_ = ctx.alloc(data_size_);
+                ctx.copy_d2d(t.gpu_data_, gpu_data_, data_size_);
+                t.on_gpu_ = true;
+                t.gpu_dirty_ = gpu_dirty_;
+            }
+        }
+#endif
+        return t;
+    }
+};
+
+} // namespace neuroflow
+
+#include "tensor_ops.hpp"
+
+#endif // NEUROFLOW_TENSOR_HPP

include/neuroflow/tensor_ops.hpp

@@ -0,0 +1,92 @@
+#ifndef NEUROFLOW_TENSOR_OPS_HPP
+#define NEUROFLOW_TENSOR_OPS_HPP
+
+
+namespace neuroflow {
+
+class TensorOps {
+public:
+    static void gemm(const Tensor& A, const Tensor& B, Tensor& C,
+                     bool transA = false, bool transB = false,
+                     float alpha = 1.0f, float beta = 0.0f);
+
+    static void quantized_gemm(const Tensor& A, const Tensor& B, Tensor& C);
+
+    static void layer_norm(Tensor& x, Tensor& weight, Tensor& bias, float eps = 1e-5f);
+
+    static void gelu(Tensor& x);
+
+    static void softmax(Tensor& x, int axis = -1);
+
+    static void dropout(Tensor& x, float rate, bool training = true);
+
+    static void quantize_int8(const Tensor& src, Tensor& dst, Tensor& scale);
+
+    static void dequantize_int8(const Tensor& src, Tensor& dst, const Tensor& scale);
+
+    static void add(Tensor& a, const Tensor& b);
+
+    static void mul(Tensor& a, float scalar);
+
+    static void parallel_gemm(const Tensor& A, const Tensor& B, Tensor& C,
+                              bool transA = false, bool transB = false,
+                              float alpha = 1.0f, float beta = 0.0f,
+                              size_t num_threads = 4);
+
+    static Tensor matmul(const Tensor& A, const Tensor& B);
+
+    static Tensor elementwise_add(const Tensor& A, const Tensor& B);
+    static Tensor elementwise_sub(const Tensor& A, const Tensor& B);
+    static Tensor elementwise_mul(const Tensor& A, const Tensor& B);
+    static Tensor scalar_mul(const Tensor& A, float s);
+    static Tensor broadcast_add(const Tensor& A, const Tensor& B);
+
+    static Tensor transpose2d(const Tensor& A);
+
+    static void fill(Tensor& A, float value);
+    static void copy_data(Tensor& dst, const Tensor& src);
+
+    static float dot(const Tensor& A, const Tensor& B);
+    static float norm2(const Tensor& A);
+
+    static Tensor reduce_sum(const Tensor& A, int axis = -1);
+
+    static Tensor slice(const Tensor& A, size_t dim, size_t start, size_t end);
+    static Tensor pad1d(const Tensor& A, size_t left, size_t right, float value = 0.0f);
+
+    static void apply_inplace(Tensor& A, std::function<float(float)> fn);
+    static Tensor apply(const Tensor& A, std::function<float(float)> fn);
+
+    static Tensor relu(const Tensor& A);
+    static Tensor sigmoid(const Tensor& A);
+    static Tensor tanh_act(const Tensor& A);
+    static Tensor log(const Tensor& A);
+    static Tensor exp(const Tensor& A);
+    static void clip_inplace(Tensor& A, float min_val, float max_val);
+
+    static Tensor concat(const std::vector<Tensor>& tensors, size_t axis = 0);
+
+private:
+    static void gemm_scalar(const float* a, const float* b, float* c,
+                            size_t M, size_t K, size_t N,
+                            bool transA, bool transB,
+                            float alpha, float beta);
+
+#ifdef HAS_AVX2
+    static void gemm_avx2(const float* a, const float* b, float* c,
+                          size_t M, size_t K, size_t N,
+                          bool transA, bool transB,
+                          float alpha, float beta);
+#endif
+
+#ifdef HAS_NEON
+    static void gemm_neon(const float* a, const float* b, float* c,
+                          size_t M, size_t K, size_t N,
+                          bool transA, bool transB,
+                          float alpha, float beta);
+#endif
+};
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_TENSOR_OPS_HPP

include/neuroflow/tokenizer.hpp

@@ -0,0 +1,81 @@
+#ifndef NEUROFLOW_TOKENIZER_HPP
+#define NEUROFLOW_TOKENIZER_HPP
+
+#include <algorithm>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <queue>
+#include <sstream>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace neuroflow {
+
+class Tokenizer {
+public:
+    virtual ~Tokenizer() = default;
+
+    virtual std::vector<size_t> encode(const std::string& text, size_t max_len = 0) = 0;
+    virtual std::string decode(const std::vector<size_t>& ids) = 0;
+
+    size_t vocab_size() const { return vocab_size_; }
+    size_t pad_id() const { return pad_id_; }
+    size_t unk_id() const { return unk_id_; }
+    size_t bos_id() const { return bos_id_; }
+    size_t eos_id() const { return eos_id_; }
+
+protected:
+    size_t vocab_size_ = 128000;
+    size_t pad_id_ = 0;
+    size_t unk_id_ = 1;
+    size_t bos_id_ = 2;
+    size_t eos_id_ = 3;
+    size_t max_input_len_ = 128;
+};
+
+class BPETokenizer : public Tokenizer {
+public:
+    BPETokenizer(const std::string& config_path);
+    BPETokenizer();
+
+    std::vector<size_t> encode(const std::string& text, size_t max_len = 0) override;
+    std::string decode(const std::vector<size_t>& ids) override;
+
+    void add_vocab(const std::string& token, size_t id);
+    void set_vocab_size(size_t vs);
+
+private:
+    std::unordered_map<std::string, size_t> vocab_;
+    std::unordered_map<size_t, std::string> id_to_token_;
+    std::vector<std::pair<std::string, std::string>> merges_;
+    std::unordered_map<std::string, size_t> merge_ranks_;
+
+    std::vector<uint8_t> utf8_to_bytes(const std::string& text);
+    std::string bytes_to_utf8(const std::string& raw);
+    std::string apply_bpe(const std::string& token);
+    void parse_config(const std::string& content);
+};
+
+class WordPieceTokenizer : public Tokenizer {
+public:
+    WordPieceTokenizer();
+    WordPieceTokenizer(const std::string& config_path);
+
+    std::vector<size_t> encode(const std::string& text, size_t max_len = 0) override;
+    std::string decode(const std::vector<size_t>& ids) override;
+
+    void add_vocab(const std::string& token, size_t id);
+    void set_vocab_size(size_t vs);
+
+private:
+    std::unordered_map<std::string, size_t> vocab_;
+    std::unordered_map<size_t, std::string> id_to_token_;
+
+    void parse_vocab(const std::string& content);
+};
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_TOKENIZER_HPP

include/neuroflow/train_lm.hpp

@@ -0,0 +1,68 @@
+#ifndef NEUROFLOW_TRAIN_LM_HPP
+#define NEUROFLOW_TRAIN_LM_HPP
+
+#include <cstddef>
+#include <string>
+#include <vector>
+#include "tensor.hpp"
+#include "causal_lm.hpp"
+#include "adamw.hpp"
+#include "scheduler.hpp"
+
+namespace neuroflow {
+
+struct TrainLMConfig {
+    size_t vocab_size = 128000;
+    size_t d_model = 256;
+    size_t max_seq_len = 128;
+    size_t num_attn_layers = 2;
+    size_t num_attn_heads = 4;
+    size_t causal_window_size = 32;
+    size_t sae_k = 64;
+    size_t ntm_memory_slots = 16;
+    bool weight_tying = true;
+    bool use_rope = true;
+    bool use_bridge = true;
+    bool use_swiglu = true;
+    bool use_qk_norm = true;
+    size_t swiglu_intermediate_size = 0;
+    std::string pooling = "mean";
+
+    float learning_rate = 5e-4f;
+    float adam_beta1 = 0.9f;
+    float adam_beta2 = 0.999f;
+    float adam_eps = 1e-8f;
+    float adam_weight_decay = 0.01f;
+    float grad_clip = 1.0f;
+    float lr_min_ratio = 0.1f;
+    float warmup_ratio = 0.01f;
+
+    size_t epochs = 10;
+    size_t total_steps = 0;
+    size_t log_interval = 10;
+    size_t save_interval = 1000;
+    std::string output_dir = "./checkpoints";
+    std::string data_path;
+};
+
+class TrainLM {
+public:
+    TrainLM(const TrainLMConfig& config);
+    void train(const std::vector<std::vector<size_t>>& dataset);
+
+private:
+    TrainLMConfig cfg_;
+    std::unique_ptr<CausalLMHead> lm_head_;
+    std::unique_ptr<AdamW> optimizer_;
+    std::unique_ptr<CosineScheduler> scheduler_;
+
+    void setup_optimizer();
+    float compute_loss_and_grad(const std::vector<size_t>& input_ids,
+                                const std::vector<size_t>& target_ids,
+                                Tensor& logits_grad);
+    void save_checkpoint(size_t step, float loss);
+};
+
+} // namespace neuroflow
+
+#endif // NEUROFLOW_TRAIN_LM_HPP

output/checkpoint_step1000/model.nfv1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02ed4c5dfe413965bf6a062ed13f5f34a2f164fbb52c863a42faaa60feeed664
+size 451607499

output/checkpoint_step2000/model.nfv1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e50b44a7630fbcdcd02ddcd65f996700940d0e1d291d95978440dd672440fb0e
+size 451607499

output/lm_head_lmh1.nfv1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e9576c6cc392d8d6132c156c9f3d2d6eba0f670ea9981f978c09957d25efad0
+size 267391131

scripts/check_json_format.py

@@ -0,0 +1,23 @@
+import json
+
+path = r'D:\neuroflow-C++\configs\tokenizer_128k.json'
+with open(path, 'r', encoding='utf-8') as f:
+    content = f.read()
+
+print('Lines:', content.count('\n'))
+print('Size:', len(content))
+
+# Find vocab section
+idx = content.find('"vocab"')
+print('vocab key at position:', idx)
+if idx > 0:
+    print('Context:', repr(content[idx:idx+50]))
+
+# The C++ parse_config looks for "\"vocab\":{" or "\"vocab\": {"
+# With indent=2, it's likely "\"vocab\": {"
+pattern1 = '"vocab": {'
+pattern2 = '"vocab":{'
+p1 = content.find(pattern1)
+p2 = content.find(pattern2)
+print(f'Pattern "{pattern1}" found at:', p1)
+print(f'Pattern "{pattern2}" found at:', p2)

scripts/config_generator.py

@@ -0,0 +1,192 @@
+import re
+import json
+import argparse
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+MODEL_CONFIG_FIELDS = [
+    ("input_dim", 512, "int", "输入维度(d_model)", "hidden_dim"),
+    ("hidden_dim", 256, "int", "隐藏层维度", "hidden_dim"),
+    ("output_dim", 10, "int", "输出维度(分类数)", "num_labels"),
+    ("memory_dim", 128, "int", "记忆维度(d_mem)", None),
+    ("memory_slots", 64, "int", "记忆槽数(MEM_SLOTS)", None),
+    ("num_layers", 2, "int", "网络层数", "num_layers"),
+    ("num_associations", 8, "int", "DMN关联头数", None),
+    ("use_quantization", False, "bool", "是否启用量化", None),
+    ("use_mla", False, "bool", "是否启用MLA注意力", "use_mla"),
+    ("mla_latent_dim", 32, "int", "MLA潜在维度", None),
+    ("use_causal_lm", False, "bool", "是否启用因果LM", None),
+    ("vocab_size", 5000, "int", "词表大小(VOCAB_SIZE)", "vocab_size"),
+    ("max_seq_len", 128, "int", "最大序列长度", "max_seq_len"),
+    ("causal_window_size", 32, "int", "因果窗口大小", None),
+    ("sae_k", 64, "int", "SAE稀疏度", None),
+    ("ntm_memory_slots", 16, "int", "NTM记忆槽数", None),
+]
+
+CAUSAL_LM_FIELDS = [
+    ("vocab_size", 5000, "int", "词表大小", "vocab_size"),
+    ("d_model", 256, "int", "模型维度", "hidden_dim"),
+    ("max_seq_len", 128, "int", "最大序列长度", "max_seq_len"),
+    ("causal_window_size", 32, "int", "因果窗口大小", None),
+    ("sae_k", 64, "int", "SAE稀疏度", None),
+    ("ntm_memory_slots", 16, "int", "NTM记忆槽数", None),
+    ("use_mla", True, "bool", "启用MLA", None),
+    ("mla_latent_dim", 32, "int", "MLA潜在维度", None),
+    ("mla_n_heads", 8, "int", "MLA头数", None),
+    ("mla_max_cache_len", 4096, "int", "MLA最大缓存长度", None),
+    ("use_quantization", False, "bool", "启用量化", None),
+]
+
+GENERATE_FIELDS = [
+    ("max_new_tokens", 50, "int", "最大生成token数", None),
+    ("temperature", 1.0, "float", "采样温度(0.0,2.0]", None),
+    ("top_k", 40, "int", "Top-K采样K值", None),
+    ("top_p", 0.9, "float", "Top-P采样P值[0.0,1.0]", None),
+    ("repetition_penalty", 1.0, "float", "重复惩罚[1.0,2.0]", None),
+    ("punct_penalty", 0.0, "float", "标点惩罚", None),
+    ("random_seed", 0, "int", "随机种子(0=随机)", None),
+    ("strategy", "top_k", "str", "采样策略(greedy/top_k/top_p/top_k_top_p)", None),
+    ("eos_id", 3, "int", "EOS token ID", None),
+]
+
+
+def parse_config_from_hpp(hpp_path, struct_name):
+    fields = []
+    try:
+        with open(hpp_path, "r", encoding="utf-8", errors="ignore") as f:
+            content = f.read()
+        pattern = rf'struct\s+{struct_name}\s*\{{([^}}]+)\}}'
+        match = re.search(pattern, content, re.DOTALL)
+        if not match:
+            logger.warning(f"未找到结构体 {struct_name}，使用默认字段")
+            return None
+        body = match.group(1)
+        for line in body.strip().split("\n"):
+            line = line.strip()
+            if not line or line.startswith("//"):
+                continue
+            m = re.match(r'(size_t|bool|float|std::string|int)\s+(\w+)\s*=\s*([^;]+);', line)
+            if m:
+                ctype, name, default = m.groups()
+                default = default.strip().rstrip('f')
+                if ctype == "size_t" or ctype == "int":
+                    default = int(default)
+                elif ctype == "float":
+                    default = float(default)
+                elif ctype == "bool":
+                    default = default == "true"
+                elif "SamplingStrategyType" in default:
+                    default = default.split("::")[-1].lower()
+                fields.append((name, default, ctype))
+    except Exception as e:
+        logger.warning(f"解析hpp失败: {e}，使用默认字段")
+    return fields
+
+
+def generate_config_json():
+    config = {}
+    comments = {}
+    python_alias = {}
+    for name, default, _, comment, alias in MODEL_CONFIG_FIELDS + CAUSAL_LM_FIELDS:
+        config[name] = default
+        comments[name] = comment
+        if alias:
+            python_alias[name] = alias
+    config["_comment"] = comments
+    config["_python_alias"] = python_alias
+    return config
+
+
+def generate_special_tokens_map():
+    return {
+        "pad_token": "<pad>",
+        "pad_token_id": 0,
+        "bos_token": "<s>",
+        "bos_token_id": 1,
+        "eos_token": "</s>",
+        "eos_token_id": 2,
+        "unk_token": "<unk>",
+        "unk_token_id": 3,
+        "_comment": {
+            "pad_token": "填充token，用于batch对齐",
+            "bos_token": "序列起始token",
+            "eos_token": "序列结束token，生成时遇到此token停止",
+            "unk_token": "未知token，词表外字符映射到此",
+        },
+    }
+
+
+def generate_generation_config_json():
+    config = {}
+    comments = {}
+    for name, default, _, comment, _ in GENERATE_FIELDS:
+        config[name] = default
+        comments[name] = comment
+    config["_comment"] = comments
+    return config
+
+
+def validate_config(config, schema_type):
+    errors = []
+    if schema_type == "config":
+        if config.get("vocab_size", 0) <= 0:
+            errors.append("vocab_size必须>0")
+        if config.get("input_dim", 0) <= 0:
+            errors.append("input_dim必须>0")
+    elif schema_type == "generation_config":
+        t = config.get("temperature", 0)
+        if t <= 0 or t > 2.0:
+            errors.append(f"temperature须在(0.0,2.0]，当前={t}")
+        s = config.get("strategy", "")
+        if s not in ("greedy", "top_k", "top_p", "top_k_top_p"):
+            errors.append(f"strategy须为greedy/top_k/top_p/top_k_top_p，当前={s}")
+    return errors
+
+
+def save_all_configs(output_dir, model_hpp=None, generative_hpp=None):
+    os.makedirs(output_dir, exist_ok=True)
+
+    config = generate_config_json()
+    if model_hpp:
+        parsed = parse_config_from_hpp(model_hpp, "Config")
+        if parsed:
+            for name, default, _ in parsed:
+                if name in config and name not in ("_comment", "_python_alias"):
+                    config[name] = default
+    errors = validate_config(config, "config")
+    if errors:
+        logger.warning(f"config.json校验问题: {errors}")
+    with open(os.path.join(output_dir, "config.json"), "w", encoding="utf-8") as f:
+        json.dump(config, f, indent=2, ensure_ascii=False)
+
+    special = generate_special_tokens_map()
+    with open(os.path.join(output_dir, "special_tokens_map.json"), "w", encoding="utf-8") as f:
+        json.dump(special, f, indent=2, ensure_ascii=False)
+
+    gen_config = generate_generation_config_json()
+    if generative_hpp:
+        parsed = parse_config_from_hpp(generative_hpp, "GenerateConfig")
+        if parsed:
+            for name, default, _ in parsed:
+                if name in gen_config and name not in ("_comment",):
+                    gen_config[name] = default
+    errors = validate_config(gen_config, "generation_config")
+    if errors:
+        logger.warning(f"generation_config.json校验问题: {errors}")
+    with open(os.path.join(output_dir, "generation_config.json"), "w", encoding="utf-8") as f:
+        json.dump(gen_config, f, indent=2, ensure_ascii=False)
+
+    logger.info(f"配置文件已保存到 {output_dir}")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    parser = argparse.ArgumentParser(description="NeuroFlow配置文件生成器")
+    parser.add_argument("--model-hpp", type=str, default="", help="model.hpp路径")
+    parser.add_argument("--generative-hpp", type=str, default="", help="generative.hpp路径")
+    parser.add_argument("--output-dir", type=str, default="configs", help="输出目录")
+    parser.add_argument("--reference-config", type=str, default="", help="Python训练系统参考配置")
+    args = parser.parse_args()
+    save_all_configs(args.output_dir, args.model_hpp or None, args.generative_hpp or None)