version: '0.13.0'
services:
  vllm:
    image: vllm/vllm-openai:latest
    container_name: vllm
    ports:
      - "8000:8000"
    volumes:
      - /mnt/models:/models:ro
      - ./config:/config:ro
    environment:
      - CUDA_VISIBLE_DEVICES=0,1
    command: >
      --model /models/<MODEL_DIR>
      --tensor-parallel-size 2
      --max-model-len 8192
      --max-num-seqs 256
      --gpu-memory-utilization 0.90
      --enable-prefix-caching
      --api-key ${VLLM_API_KEY}
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 2
              capabilities: [gpu]
        limits:
          memory: 128G
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 5

  nginx:
    image: nginx:alpine
    container_name: nginx-proxy
    ports:
      - "443:443"
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf:ro
      - ./certs:/etc/nginx/certs:ro
    depends_on:
      - vllm
    restart: unless-stopped

version: '3.8'
services:
  localai:
    image: localai/localai:latest-aio-gpu-nvidia-cuda-12
    container_name: localai
    ports:
      - "8080:8080"
    volumes:
      - ./models:/models
      - ./config:/config
    environment:
      - THREADS=8
      - CONTEXT_SIZE=4096
      - MODELS_PATH=/models
      - CONFIG_FILE=/config/models.yaml
      - BUILD_TYPE=cublas
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 1
              capabilities: [gpu]
    restart: unless-stopped

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: vllm
  namespace: ai-inference
spec:
  serviceName: vllm
  replicas: 2
  selector:
    matchLabels:
      app: vllm
  template:
    metadata:
      labels:
        app: vllm
    spec:
      containers:
      - name: vllm
        image: vllm/vllm-openai:latest
        ports:
        - containerPort: 8000
        resources:
          limits:
            nvidia.com/gpu: 2
            memory: "128Gi"
            cpu: "32"
          requests:
            nvidia.com/gpu: 2
            memory: "64Gi"
            cpu: "16"
        volumeMounts:
        - name: model-storage
          mountPath: /models
        - name: config
          mountPath: /config
        env:
        - name: VLLM_API_KEY
          valueFrom:
            secretKeyRef:
              name: vllm-secrets
              key: api-key
        args:
        - "--model"
        - "/models/<MODEL_DIR>"
        - "--tensor-parallel-size"
        - "2"
        - "--max-model-len"
        - "8192"
        - "--port"
        - "8000"
        livenessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 300
          periodSeconds: 30
          failureThreshold: 3
        readinessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 120
          periodSeconds: 10
      volumes:
      - name: config
        configMap:
          name: vllm-config
      nodeSelector:
        accelerator: nvidia-a100
      tolerations:
      - key: nvidia.com/gpu
        operator: Exists
        effect: NoSchedule
  volumeClaimTemplates:
  - metadata:
      name: model-storage
    spec:
      accessModes: ["ReadWriteOnce"]
      storageClassName: fast-ssd
      resources:
        requests:
          storage: 500Gi

# Maximum throughput configuration
vllm serve /models/<MODEL_DIR> \
  --tensor-parallel-size 2 \              # Multi-GPU parallelism
  --max-model-len 8192 \                  # Context length (shorter = faster)
  --max-num-seqs 256 \                    # Max concurrent sequences
  --gpu-memory-utilization 0.90 \         # VRAM usage (0.90 = 90%)
  --enable-prefix-caching \               # Cache common prefixes
  --enable-chunked-prefill \              # Better batching
  --max-num-batched-tokens 32768 \        # Batch size limit
  --disable-log-requests                  # Reduce logging overhead

# Example (binary name varies by build; often `llama-server`)
# Expose a minimal local endpoint on port 8081
./llama-server -m /mnt/models/<MODEL_FILE>.gguf -c 8192 --host 0.0.0.0 --port 8081

# Basic smoke test (API shape depends on build/flags)
curl -s http://localhost:8081/health || true

#!/bin/bash
# prepare-offline-package.sh

PACKAGE_DIR="./offline-llm-package"
mkdir -p ${PACKAGE_DIR}/{images,models,configs}

# 1. Save Docker images
docker pull vllm/vllm-openai:latest
docker pull nginx:alpine
docker save vllm/vllm-openai:latest -o ${PACKAGE_DIR}/images/vllm.tar
docker save nginx:alpine -o ${PACKAGE_DIR}/images/nginx.tar

# 2. Download models (example model package)
pip install huggingface-hub
# Download from your approved model source (Hugging Face / internal registry)
# Prefer new-generation families such as Qwen-Next, MiniMax-M2, GLM-4.6.
huggingface-cli download <provider>/<model> \
  --local-dir ${PACKAGE_DIR}/models/<MODEL_DIR>

# 3. Copy configurations
cp docker-compose.yml ${PACKAGE_DIR}/configs/
cp nginx.conf ${PACKAGE_DIR}/configs/
cp .env.example ${PACKAGE_DIR}/configs/

# 4. Create installation script
cat > ${PACKAGE_DIR}/install.sh << 'EOF'
#!/bin/bash
docker load -i images/vllm.tar
docker load -i images/nginx.tar
cp -r models /mnt/ai-models/
cp configs/* /opt/llm/
cd /opt/llm && docker compose up -d
EOF
chmod +x ${PACKAGE_DIR}/install.sh

# 5. Package for transfer
tar -czvf offline-llm-package.tar.gz ${PACKAGE_DIR}

#!/bin/bash
# install-airgapped.sh

# Verify we're offline
if ping -c 1 google.com &>/dev/null; then
  echo "WARNING: Network connectivity detected. Expected air-gapped environment."
  exit 1
fi

# Load Docker images
docker load -i /media/transfer/images/vllm.tar
docker load -i /media/transfer/images/nginx.tar

# Copy models
cp -r /media/transfer/models/* /mnt/ai-models/

# Copy and customize configuration
cp /media/transfer/configs/* /opt/llm/
# Edit /opt/llm/.env with local settings

# Start services
cd /opt/llm && docker compose up -d

# Verify
curl http://localhost:8000/health

upstream vllm_backend {
    server vllm:8000;
    keepalive 32;
}

server {
    listen 443 ssl http2;
    server_name llm.internal.company.com;

    ssl_certificate /etc/nginx/certs/server.crt;
    ssl_certificate_key /etc/nginx/certs/server.key;
    ssl_protocols TLSv1.2 TLSv1.3;

    # Security headers
    add_header X-Content-Type-Options nosniff;
    add_header X-Frame-Options DENY;
    add_header X-XSS-Protection "1; mode=block";

    # Rate limiting
    limit_req_zone $binary_remote_addr zone=api_limit:10m rate=10r/s;
    limit_req zone=api_limit burst=20 nodelay;

    # Request size limit
    client_max_body_size 10M;

    location /v1/ {
        proxy_pass http://vllm_backend;
        proxy_http_version 1.1;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header Connection "";

        # Streaming support
        proxy_buffering off;
        proxy_read_timeout 300s;

        # Authentication (example: header-based)
        if ($http_authorization = "") {
            return 401;
        }
    }

    location /health {
        proxy_pass http://vllm_backend/health;
        access_log off;
    }

    location /metrics {
        proxy_pass http://vllm_backend/metrics;
        # Restrict to monitoring network
        allow 10.0.0.0/8;
        deny all;
    }
}