Build the docker image

docker build -t llama-server-cuda .

Download the model

mkdir -p ~/llama-models
cd ~/llama-models
wget https://huggingface.co/bartowski/Mistral-7B-Instruct-v0.3-GGUF/resolve/main/Mistral-7B-Instruct-v0.3-Q4_K_M.gguf -O model.gguf

Run the docker container

docker run --rm --gpus all \
  -p 8080:8080 \
  -v $(pwd)/llama-models:/llama.cpp/models \
  llama-server-cuda \
  ./build/bin/llama-server \
    --model /llama.cpp/models/model.gguf \
    --host 0.0.0.0 \
    --port 8080 \
    --n-gpu-layers 20

Test

curl --request POST     --url httcurl --request POST     --url http://localhost:8080/v1/chat/completions     --header "Content-Type: application/json"     --data '{
         "model": "mistral",
         "messages": [
             { 
                 "role": "user", 
                 "content": "Escribe un pequeño poema sobre el océano." 
             } 
         ] 
     }'