Resource Hub

1version: '3.8'
2services:
3  vllm:
4    image: vllm/vllm-openai:latest
5    runtime: nvidia
6    environment:
7      - NVIDIA_VISIBLE_DEVICES=all
8    volumes:
9      - ~/.cache/huggingface:/root/.cache/huggingface
10    ports:
11      - "8000:8000"
12    command: --model meta-llama/Meta-Llama-3.1-8B-Instruct --max-model-len 8192

1inference:
2  engine: vllm
3  base_url: http://localhost:8000/v1
4  model: meta-llama/Meta-Llama-3.1-8B-Instruct
5  temperature: 0.7
6  max_tokens: 1024
7
8plugins:
9  directory: ./ava-sdk-plugins
10  auto_load: true
11
12system_prompt: |
13  You are AVA, an AI assistant specialized in gaming
14  and development. Respond clearly and concisely.