File size: 1,088 Bytes
ce25ab4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | """
GGUF + DAG Pipeline — load GGUF model, run through networkx DAG
Author: Ahmad Ali Parr · Trust: Bel Esprit D'Accord Irrevocable Trust
"""
# pip install llama_cpp_python networkx
from llama_cpp import Llama
import networkx as nx
model = Llama(model_path="model.gguf", n_ctx=2048, n_threads=8)
# Define DAG
dag = nx.DiGraph()
dag.add_nodes_from(["parse_binary", "convert_numpy", "convert_torch", "inference"])
dag.add_edges_from([
("parse_binary", "convert_numpy"),
("convert_numpy", "convert_torch"),
("convert_torch", "inference"),
])
def parse_binary(path):
with open(path, "rb") as f:
return f.read()
def convert_numpy(bin_data):
import numpy as np
return np.frombuffer(bin_data, dtype=np.float32)
def convert_torch(np_arr):
import torch
return torch.from_numpy(np_arr)
def inference(tensor):
return model("Translate this sentence to Korean:")
# Execute
binary_data = parse_binary("model.gguf")
np_arr = convert_numpy(binary_data)
torch_tensor = convert_torch(np_arr)
output = inference(torch_tensor)
print(output)
|