Spaces:
Sleeping
Sleeping
isaachaaseinstitutemail-spec commited on
Commit ·
3ea0102
1
Parent(s): 24dca34
init: added llama-cpp-python API server with Qwen 1.5B
Browse files- Dockerfile +15 -0
Dockerfile
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.9-slim
|
| 2 |
+
|
| 3 |
+
# Install compilation tools
|
| 4 |
+
RUN apt-get update && apt-get install -y build-essential curl
|
| 5 |
+
|
| 6 |
+
# Install llama-cpp-python with a server
|
| 7 |
+
RUN pip install "llama-cpp-python[server]" huggingface-hub
|
| 8 |
+
|
| 9 |
+
# Download a fast, compressed model (Qwen 1.5B is perfect for free CPUs)
|
| 10 |
+
RUN huggingface-cli download Qwen/Qwen2.5-1.5B-Instruct-GGUF qwen2.5-1.5b-instruct-q4_k_m.gguf --local-dir /model
|
| 11 |
+
|
| 12 |
+
EXPOSE 7860
|
| 13 |
+
|
| 14 |
+
# Start the OpenAI-compatible API server
|
| 15 |
+
CMD ["python3", "-m", "llama_cpp.server", "--model", "/model/qwen2.5-1.5b-instruct-q4_k_m.gguf", "--host", "0.0.0.0", "--port", "7860", "--n_ctx", "2048"]
|