diff options
Diffstat (limited to 'docker-compose.yml')
| -rw-r--r-- | docker-compose.yml | 68 |
1 files changed, 16 insertions, 52 deletions
diff --git a/docker-compose.yml b/docker-compose.yml index 1b58f68..bb5046b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,65 +1,29 @@ services: - inference: - image: ghcr.io/ggml-org/llama.cpp:server-rocm - restart: always - command: >- - -m /models/gemma-4-26B-A4B-it-qat-UD-Q4_K_XL.gguf - --host 0.0.0.0 - --port 8080 - --api-key sk-7bbd5413f34151830df2ff33b4956ec10bb8029d73a6d53d3da30a59640b3232 - --ctx-size 262144 - -ngl 12 - --jinja - --alias "Gemma 4" - volumes: - - type: bind - source: ./inference/models - target: /models - read_only: false - networks: - - local-ai - ports: - - 8080:8080 - privileged: true - shm_size: 16g - security_opt: - - seccomp:unconfined - cap_add: - - SYS_PTRACE - group_add: - - video - devices: - - /dev/kfd:/dev/kfd - - /dev/dri:/dev/dri - embedding: - image: ollama/ollama:latest + inference: + image: ollama/ollama:rocm restart: always environment: - OLLAMA_MAX_QUEUE: 8192 + OLLAMA_MAX_QUEUE: 512 + OLLAMA_MAX_LOADED_MODELS: 1 + OLLAMA_NUM_PARALLEL: 1 + OLLAMA_KEEP_ALIVE: 15m + OLLAMA_FLASH_ATTENTION: 1 + OLLAMA_GPU_OVERHEAD: 2147483648 + OLLAMA_NO_CLOUD: 1 + LLAMA_ARG_IMAGE_MAX_TOKENS: 1120 volumes: - type: bind - source: ./embedding/ollama + source: ./inference/ollama target: /root/.ollama read_only: false - networks: - - local-ai + - type: bind + source: ./inference/modelfiles + target: /root/modelfiles + read_only: false ports: - 11434:11434 - privileged: true - security_opt: - - seccomp:unconfined - cap_add: - - SYS_PTRACE - group_add: - - video devices: - /dev/kfd:/dev/kfd - /dev/dri:/dev/dri - -networks: - local-ai: - driver: bridge - -volumes: - search-cache: + shm_size: 4gb |