12b_instruct_infer.yaml

# Inference config for Gemma 3 12B Instruct.
#
# Model highlights:
#   - 12B parameter multimodal model from Google
#   - 128K context length input, 8K output
#   - Vision-language capable (text + images)
#   - 140+ language support
#
# Requirements:
#   - transformers >= 4.50.0
#   - Gemma license acceptance required: https://huggingface.co/google/gemma-3-12b-it
#
# Usage:
#   oumi infer -i -c configs/recipes/gemma3/inference/12b_instruct_infer.yaml
#
# See Also:
#   - Documentation: https://oumi.ai/docs/en/latest/user_guides/infer/infer.html
#   - Config class: oumi.core.configs.InferenceConfig
#   - Config source: https://github.com/oumi-ai/oumi/blob/main/src/oumi/core/configs/inference_config.py
#   - Other inference configs: configs/**/inference/

model:
  model_name: "google/gemma-3-12b-it"
  model_max_length: 8192
  torch_dtype_str: "bfloat16"
  attn_implementation: "sdpa"
  trust_remote_code: True

generation:
  max_new_tokens: 2048
  temperature: 0.7
  top_p: 0.9

engine: NATIVE # Can also use `VLLM` for faster inference on GPUs.