config.yml

# Sample YAML file for configuration.
# Comment and uncomment values as needed. Every value has a default within the application.
# This file serves to be a drop in for config.yml

# Unless specified in the comments, DO NOT put these options in quotes!
# You can use https://www.yamllint.com/ if you want to check your YAML formatting.

# Options for networking
network:
  # The IP to host on (default: 127.0.0.1).
  # Use 0.0.0.0 to expose on all network adapters
  host: 0.0.0.0

  # The port to host on (default: 5000)
  port: 5000

  # Disable HTTP token authenticaion with requests
  # WARNING: This will make your instance vulnerable!
  # Turn on this option if you are ONLY connecting from localhost
  disable_auth: False

  # Send tracebacks over the API to clients (default: False)
  # NOTE: Only enable this for debug purposes
  send_tracebacks: False

  # Select API servers to enable (default: ["OAI"])
  # Possible values: OAI
  api_servers: ["OAI"]

# Options for logging
logging:
  # Enable prompt logging (default: False)
  prompt: False

  # Enable generation parameter logging (default: False)
  generation_params: False

  # Enable request logging (default: False)
  # NOTE: Only use this for debugging!
  requests: False

# Options for sampling
sampling:
  # Override preset name. Find this in the sampler-overrides folder (default: None)
  # This overrides default fallbacks for sampler values that are passed to the API
  # Server-side overrides are NOT needed by default
  # WARNING: Using this can result in a generation speed penalty
  #override_preset: 

# Options for development and experimentation
developer:
  # Skips exllamav2 version check (default: False)
  # It's highly recommended to update your dependencies rather than enabling this flag
  # WARNING: Don't set this unless you know what you're doing!
  #unsafe_launch: False

  # Disable all request streaming (default: False)
  # A kill switch for turning off SSE in the API server
  #disable_request_streaming: False

  # Enable the torch CUDA malloc backend (default: False)
  # This can save a few MBs of VRAM, but has a risk of errors. Use at your own risk.
  cuda_malloc_backend: True

  # Enable Uvloop or Winloop (default: False)
  # Make the program utilize a faster async event loop which can improve performance
  # NOTE: It's recommended to enable this, but if something breaks, turn this off.
  uvloop: True

  # Set process to use a higher priority
  # For realtime process priority, run as administrator or sudo
  # Otherwise, the priority will be set to high
  realtime_process_priority: True

# Options for model overrides and loading
# Please read the comments to understand how arguments are handled between initial and API loads
model:
  # Overrides the directory to look for models (default: models)
  # Windows users, DO NOT put this path in quotes! This directory will be invalid otherwise.
  model_dir: models

  # Sends dummy model names when the models endpoint is queried
  # Enable this if the program is looking for a specific OAI model
  #use_dummy_models: False

  # An initial model to load. Make sure the model is located in the model directory!
  # A model can be loaded later via the API.
  # REQUIRED: This must be filled out to load a model on startup!
  model_name: Mistral-Large-Instruct-2407_exl2_2.85bpw

  # The below parameters only apply for initial loads
  # All API based loads do NOT inherit these settings unless specified in use_as_default

  # Names of args to use as a default fallback for API load requests (default: [])
  # For example, if you always want cache_mode to be Q4 instead of on the inital model load,
  # Add "cache_mode" to this array
  # Ex. ["max_seq_len", "cache_mode"]
  #use_as_default: []

  # The below parameters apply only if model_name is set

  # Max sequence length (default: Empty)
  # Fetched from the model's base sequence length in config.json by default
  max_seq_len: 32768

  # Overrides base model context length (default: Empty)
  # WARNING: Don't set this unless you know what you're doing!
  # Again, do NOT use this for configuring context length, use max_seq_len above ^
  # Only use this if the model's base sequence length in config.json is incorrect (ex. Mistral 7B)
  #override_base_seq_len:

  # Load model with tensor parallelism
  # If a GPU split isn't provided, the TP loader will fallback to autosplit
  # Enabling ignores the gpu_split_auto and autosplit_reserve values
  #tensor_parallel: True

  # Automatically allocate resources to GPUs (default: True)
  # NOTE: Not parsed for single GPU users
  gpu_split_auto: True

  # Reserve VRAM used for autosplit loading (default: 96 MB on GPU 0)
  # This is represented as an array of MB per GPU used
  autosplit_reserve: [0]

  # An integer array of GBs of vram to split between GPUs (default: [])
  # Used with tensor parallelism
  # NOTE: Not parsed for single GPU users
  #gpu_split: [20.6, 24]

  # Rope scale (default: 1.0)
  # Same thing as compress_pos_emb
  # Only use if your model was trained on long context with rope (check config.json)
  # Leave blank to pull the value from the model
  #rope_scale: 1.0

  # Rope alpha (default: 1.0)
  # Same thing as alpha_value
  # Leave blank to automatically calculate alpha
  #rope_alpha: 1.0

  # Enable different cache modes for VRAM savings (slight performance hit).
  # Possible values FP16, Q8, Q6, Q4. (default: FP16)
  cache_mode: Q4

  # Size of the prompt cache to allocate (default: max_seq_len)
  # This must be a multiple of 256. A larger cache uses more VRAM, but allows for more prompts to be processed at once.
  # NOTE: Cache size should not be less than max_seq_len.
  # For CFG, set this to 2 * max_seq_len to make room for both positive and negative prompts.
  # cache_size:

  # Chunk size for prompt ingestion. A lower value reduces VRAM usage at the cost of ingestion speed (default: 2048)
  # NOTE: Effects vary depending on the model. An ideal value is between 512 and 4096
  chunk_size: 1024

  # Set the maximum amount of prompts to process at one time (default: None/Automatic)
  # This will be automatically calculated if left blank.
  # A max batch size of 1 processes prompts one at a time.
  # NOTE: Only available for Nvidia ampere (30 series) and above GPUs
  #max_batch_size:

  # Set the prompt template for this model. If empty, attempts to look for the model's chat template. (default: None)
  # If a model contains multiple templates in its tokenizer_config.json, set prompt_template to the name
  # of the template you want to use.
  # NOTE: Only works with chat completion message lists!
  #prompt_template:

  # Number of experts to use PER TOKEN. Fetched from the model's config.json if not specified (default: Empty)
  # WARNING: Don't set this unless you know what you're doing!
  # NOTE: For MoE models (ex. Mixtral) only!
  #num_experts_per_token:

  # Enables fasttensors to possibly increase model loading speeds (default: False)
  fasttensors: true

  # Options for draft models (speculative decoding). This will use more VRAM!
  #draft:
    # Overrides the directory to look for draft (default: models)
    #draft_model_dir: models

    # An initial draft model to load. Make sure this model is located in the model directory!
    # A draft model can be loaded later via the API.
    #draft_model_name: A model name
  
    # The below parameters only apply for initial loads
    # All API based loads do NOT inherit these settings unless specified in use_as_default

    # Rope scale for draft models (default: 1.0)
    # Same thing as compress_pos_emb
    # Only use if your draft model was trained on long context with rope (check config.json)
    #draft_rope_scale: 1.0

    # Rope alpha for draft model (default: 1.0)
    # Same thing as alpha_value
    # Leave blank to automatically calculate alpha value
    #draft_rope_alpha: 1.0

    # Enable different draft model cache modes for VRAM savings (slight performance hit).
    # Possible values FP16, Q8, Q6, Q4. (default: FP16)
    #draft_cache_mode: FP16
  
  # Options for loras
  #lora:
    # Overrides the directory to look for loras (default: loras)
    #lora_dir: loras
    
    # List of loras to load and associated scaling factors (default: 1.0). Comment out unused entries or add more rows as needed.
    #loras:
    #- name: lora1
    #  scaling: 1.0

# Options for embedding models and loading.
# NOTE: Embeddings requires the "extras" feature to be installed
# Install it via "pip install .[extras]"
embeddings:
  # Overrides directory to look for embedding models (default: models)
  embedding_model_dir: models

  # Device to load embedding models on (default: cpu)
  # Possible values: cpu, auto, cuda
  # NOTE: It's recommended to load embedding models on the CPU.
  # If you'd like to load on an AMD gpu, set this value to "cuda" as well.
  embeddings_device: cpu

  # The below parameters only apply for initial loads
  # All API based loads do NOT inherit these settings unless specified in use_as_default

  # An initial embedding model to load on the infinity backend (default: None)
  embedding_model_name: