File size: 1,399 Bytes
2c20074 e9319be 2c20074 9534e0b 0becb0a 2c20074 e9319be 2c20074 e9319be 2c20074 9534e0b 2c20074 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | #include <torch/library.h>
#include "registration.h"
#include "torch_binding.h"
TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
ops.def(
"get_rows(Tensor blocks, Tensor indices, int ggml_type, int cols, ScalarType dtype) -> Tensor");
ops.def("dequantize(Tensor blocks, int ggml_type, int rows, int cols, ScalarType dtype) -> Tensor");
ops.def("mul_mat_vec(Tensor blocks, Tensor x, int ggml_type, int out_features) -> Tensor");
ops.def(
"mul_mat_id(Tensor blocks, Tensor x, Tensor ids, int ggml_type, int out_features) -> Tensor");
// Takes no tensor, so it has no device to dispatch on and is registered as a catch-all. Each
// backend's shared object is its own library namespace, so there is one implementation per build.
ops.def("gemv_types() -> int[]");
ops.impl("gemv_types", &gemv_types);
// The schema is the same for every backend; only the implementation differs.
#if defined(CUDA_KERNEL) || defined(ROCM_KERNEL)
ops.impl("get_rows", torch::kCUDA, &get_rows);
ops.impl("dequantize", torch::kCUDA, &dequantize);
ops.impl("mul_mat_vec", torch::kCUDA, &mul_mat_vec);
#elif defined(METAL_KERNEL)
ops.impl("get_rows", torch::kMPS, &get_rows);
ops.impl("dequantize", torch::kMPS, &dequantize);
ops.impl("mul_mat_vec", torch::kMPS, &mul_mat_vec);
ops.impl("mul_mat_id", torch::kMPS, &mul_mat_id);
#endif
}
REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
|