From 9496b3ade6bdaa77b58dfd1ac4b3ed89c5067834 Mon Sep 17 00:00:00 2001 From: Felix Morgner Date: Fri, 18 Sep 2026 07:08:07 +0200 Subject: initial commit --- docker-compose.yml | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 docker-compose.yml (limited to 'docker-compose.yml') diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8079b2a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,70 @@ +services: + inference: + image: ghcr.io/ggml-org/llama.cpp:server-rocm + restart: always + command: >- + -m /models/gemma-4-26B-A4B-it-qat-UD-Q4_K_XL.gguf + -md /models/mtp-gemma-4-26B-A4B-it.gguf + -mm /models/mmproj-BF16.gguf + --host 0.0.0.0 + --port 8080 + --api-key sk-7bbd5413f34151830df2ff33b4956ec10bb8029d73a6d53d3da30a59640b3232 + --ctx-size 131072 + -ngl 14 + -ngld -1 + --jinja + --alias "Gemma 4" + --spec-type draft-mtp + --spec-draft-n-max 4 + volumes: + - type: bind + source: ./inference/models + target: /models + read_only: false + networks: + - local-ai + ports: + - 8080:8080 + privileged: true + shm_size: 16g + security_opt: + - seccomp:unconfined + cap_add: + - SYS_PTRACE + group_add: + - video + devices: + - /dev/kfd:/dev/kfd + - /dev/dri:/dev/dri + + embedding: + image: ollama/ollama:latest + restart: always + environment: + OLLAMA_MAX_QUEUE: 8192 + volumes: + - type: bind + source: ./embedding/ollama + target: /root/.ollama + read_only: false + networks: + - local-ai + ports: + - 11434:11434 + privileged: true + security_opt: + - seccomp:unconfined + cap_add: + - SYS_PTRACE + group_add: + - video + devices: + - /dev/kfd:/dev/kfd + - /dev/dri:/dev/dri + +networks: + local-ai: + driver: bridge + +volumes: + search-cache: -- cgit v1.2.3