diff options
| author | Felix Morgner <felix.morgner@gmail.com> | 2026-09-18 07:08:07 +0200 |
|---|---|---|
| committer | Felix Morgner <felix.morgner@gmail.com> | 2026-09-18 07:09:00 +0200 |
| commit | 9496b3ade6bdaa77b58dfd1ac4b3ed89c5067834 (patch) | |
| tree | 0a90828b9b4460b314ae95a1ae2d71885e4d2223 /docker-compose.yml | |
| download | local-ai-9496b3ade6bdaa77b58dfd1ac4b3ed89c5067834.tar.xz local-ai-9496b3ade6bdaa77b58dfd1ac4b3ed89c5067834.zip | |
initial commit
Diffstat (limited to 'docker-compose.yml')
| -rw-r--r-- | docker-compose.yml | 70 |
1 files changed, 70 insertions, 0 deletions
diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8079b2a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,70 @@ +services: + inference: + image: ghcr.io/ggml-org/llama.cpp:server-rocm + restart: always + command: >- + -m /models/gemma-4-26B-A4B-it-qat-UD-Q4_K_XL.gguf + -md /models/mtp-gemma-4-26B-A4B-it.gguf + -mm /models/mmproj-BF16.gguf + --host 0.0.0.0 + --port 8080 + --api-key sk-7bbd5413f34151830df2ff33b4956ec10bb8029d73a6d53d3da30a59640b3232 + --ctx-size 131072 + -ngl 14 + -ngld -1 + --jinja + --alias "Gemma 4" + --spec-type draft-mtp + --spec-draft-n-max 4 + volumes: + - type: bind + source: ./inference/models + target: /models + read_only: false + networks: + - local-ai + ports: + - 8080:8080 + privileged: true + shm_size: 16g + security_opt: + - seccomp:unconfined + cap_add: + - SYS_PTRACE + group_add: + - video + devices: + - /dev/kfd:/dev/kfd + - /dev/dri:/dev/dri + + embedding: + image: ollama/ollama:latest + restart: always + environment: + OLLAMA_MAX_QUEUE: 8192 + volumes: + - type: bind + source: ./embedding/ollama + target: /root/.ollama + read_only: false + networks: + - local-ai + ports: + - 11434:11434 + privileged: true + security_opt: + - seccomp:unconfined + cap_add: + - SYS_PTRACE + group_add: + - video + devices: + - /dev/kfd:/dev/kfd + - /dev/dri:/dev/dri + +networks: + local-ai: + driver: bridge + +volumes: + search-cache: |