name: nim-llama31-8b-ins-v03
type: service
image:
type: image
image_uri: nvcr.io/nim/meta/llama-3.1-8b-instruct:1.3.3
docker_registry: tenant:custom:nvcr:docker-registry:nvcr-truefoundry
ports:
- host: <your-host>
port: 8000
expose: true
protocol: TCP
app_protocol: http
env:
NGC_API_KEY: tfy-secret://tenant:secret-group:NGC_API_KEY
NIM_LOG_LEVEL: DEFAULT
NIM_SERVER_PORT: '8000'
NIM_JSONL_LOGGING: '1'
NIM_MAX_MODEL_LEN: '4096'
NIM_MODEL_PROFILE: vllm-bf16-tp2
NIM_LOW_MEMORY_MODE: '1'
NIM_SERVED_MODEL_NAME: llm
NIM_TRUST_CUSTOM_CODE: '1'
NIM_ENABLE_KV_CACHE_REUSE: '1'
NIM_CACHE_PATH: /opt/nim/.cache
labels:
tfy_model_server: vLLM
tfy_openapi_path: openapi.json
tfy_sticky_session_header_name: x-truefoundry-sticky-session-id
replicas: 1
resources:
node:
type: node_selector
capacity_type: on_demand
devices:
- name: T4
type: nvidia_gpu
count: 2
cpu_limit: 8
cpu_request: 6
memory_limit: 32000
memory_request: 27200
shared_memory_size: 24000
ephemeral_storage_limit: 100000
ephemeral_storage_request: 20000
workspace_fqn: <your-workspace-fqn>
readiness_probe:
config:
path: /v1/health/ready
port: 8000
type: http
period_seconds: 10
timeout_seconds: 1
failure_threshold: 3
success_threshold: 1
initial_delay_seconds: 0
allow_interception: false