92 lines
2.8 KiB
Markdown
92 lines
2.8 KiB
Markdown
**vaessl: Setup LiteLLM**
|
|
|
|
# LiteLLM Docker Compose setup
|
|
|
|
I chose to use my existing Docker and PostgreSQL instance to spin up a LiteLLM container. I use Portainer stacks to write the docker-compose.yaml. Instead of creating a .env file I just wrote all environment variables into the yaml. The official docker-compose.yaml may vary.
|
|
|
|
Before deploying the container create a config.yaml and prometheus.yml in the root folder:
|
|
|
|
*config.yaml*
|
|
```
|
|
general_settings:
|
|
master_key: sk-key # 🔑 your proxy admin key (must start with sk-)
|
|
database_url: "postgresql://pgUser:pgUserPw@192.168.1.208:5432/litellm"
|
|
```
|
|
|
|
*prometheus.yml*
|
|
|
|
```
|
|
global:
|
|
scrape_interval: 15s
|
|
evaluation_interval: 15s
|
|
|
|
scrape_configs:
|
|
- job_name: "litellm"
|
|
static_configs:
|
|
- targets: ["litellm:4000"]
|
|
```
|
|
|
|
*Portainer stack*
|
|
```
|
|
services:
|
|
litellm:
|
|
#build:
|
|
# context: .
|
|
# args:
|
|
# target: runtime
|
|
image: docker.litellm.ai/berriai/litellm:main-stable
|
|
volumes:
|
|
- /home/pi/docker/litellm/config.yaml:/app/config.yaml
|
|
command:
|
|
- "--config=/app/config.yaml"
|
|
ports:
|
|
- "4000:4000" # Map the container port to the host, change the host port if necessary
|
|
environment:
|
|
DATABASE_URL: "postgresql://pgUser:pgUserPw@192.168.1.208:5432/litellm"
|
|
STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
|
|
OPENAI_API_KEY: "sk-key"
|
|
OPENAI_BASE_URL: "https://api.openai.com/v1"
|
|
LITELLM_MASTER_KEY: "sk-key"
|
|
LITELLM_SALT_KEY: "sk-saltkey"
|
|
PUID: 1000
|
|
PGID: 1000
|
|
#env_file:
|
|
# - /home/pi/docker/litellm/.env
|
|
healthcheck:
|
|
test:
|
|
- CMD-SHELL
|
|
- python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')" # Command to execute for health check
|
|
interval: 30s # Perform health check every 30 seconds
|
|
timeout: 10s # Health check command times out after 10 seconds
|
|
retries: 3 # Retry up to 3 times if health check fails
|
|
start_period: 40s # Wait 40 seconds after container start before beginning health checks
|
|
networks:
|
|
- postgres_pg_network
|
|
|
|
prometheus:
|
|
image: prom/prometheus
|
|
volumes:
|
|
- /home/pi/docker/litellm/prometheus_data:/prometheus
|
|
- /home/pi/docker/litellm/prometheus.yml:/etc/prometheus/prometheus.yml
|
|
ports:
|
|
- "9090:9090"
|
|
command:
|
|
- "--config.file=/etc/prometheus/prometheus.yml"
|
|
- "--storage.tsdb.path=/prometheus"
|
|
- "--storage.tsdb.retention.time=15d"
|
|
restart: always
|
|
|
|
volumes:
|
|
prometheus_data:
|
|
driver: local
|
|
postgres_data:
|
|
name: litellm_postgres_data # Named volume for Postgres data persistence
|
|
|
|
networks:
|
|
postgres_pg_network:
|
|
external: true
|
|
```
|
|
|
|
|
|
|
|
Models can be added in the UI under localhost:4000 and will be saved into the database. No need to define them in the config.yaml |