Gemma 3 Guide 2025: Google's Best Open-Source LLM

Install Ollama + pull Gemma 3

# macOS / Linux
curl -fsSL https://ollama.com/install.sh | sh

# Pull and run Gemma 3 4B (vision-capable, ~3 GB Q4)
ollama run gemma3:4b

# Pull and run Gemma 3 27B (flagship, ~17 GB Q4)
ollama run gemma3:27b

# Run with vision input from the CLI
ollama run gemma3:4b "Describe this image" --image /path/to/image.jpg

Query via OpenAI SDK (Ollama serves on port 11434)

from openai import OpenAI

client = OpenAI(base_url="http://localhost:11434/v1", api_key="ollama")

response = client.chat.completions.create(
    model="gemma3:4b",
    messages=[{"role": "user", "content": "Explain attention mechanisms."}],
)
print(response.choices[0].message.content)

Text generation — gemma-3-27b-it

from transformers import AutoTokenizer, AutoModelForCausalLM
import torch

model_id = "google/gemma-3-27b-it"

tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    torch_dtype=torch.bfloat16,
    device_map="auto",
)

messages = [
    {"role": "user", "content": "What is the capital of France?"}
]

inputs = tokenizer.apply_chat_template(
    messages,
    return_tensors="pt",
    return_dict=True,
).to(model.device)

outputs = model.generate(**inputs, max_new_tokens=256)
print(tokenizer.decode(outputs[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True))

Vision input — gemma-3-4b-it with PIL image

from transformers import AutoProcessor, AutoModelForImageTextToText
from PIL import Image
import torch, requests

model_id = "google/gemma-3-4b-it"

processor = AutoProcessor.from_pretrained(model_id)
model = AutoModelForImageTextToText.from_pretrained(
    model_id, torch_dtype=torch.bfloat16, device_map="auto"
)

image = Image.open(requests.get("https://example.com/chart.png", stream=True).raw)

messages = [
    {
        "role": "user",
        "content": [
            {"type": "image", "image": image},
            {"type": "text", "text": "Describe what you see in this chart."},
        ],
    }
]

inputs = processor.apply_chat_template(
    messages, add_generation_prompt=True, return_dict=True, return_tensors="pt"
).to(model.device)

outputs = model.generate(**inputs, max_new_tokens=512)
print(processor.decode(outputs[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True))

Install the Google Gen AI SDK

pip install google-genai

Call gemma-3-27b-it via Gemini API (OpenAI-compatible)

from openai import OpenAI

# Get a free API key at aistudio.google.com
client = OpenAI(
    base_url="https://generativelanguage.googleapis.com/v1beta/openai/",
    api_key="YOUR_GEMINI_API_KEY",
)

response = client.chat.completions.create(
    model="gemma-3-27b-it",
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Explain the transformer architecture."},
    ],
    max_tokens=1024,
)
print(response.choices[0].message.content)

Groq — Gemma 2 9B at 800+ tok/s

pip install groq

groq_gemma.py

from groq import Groq

client = Groq(api_key="YOUR_GROQ_API_KEY")

response = client.chat.completions.create(
    model="gemma2-9b-it",   # Groq hosts Gemma 2 9B
    messages=[
        {"role": "user", "content": "Write a quicksort in Python."}
    ],
    temperature=0.7,
    max_tokens=1024,
)
print(response.choices[0].message.content)

Run Gemma 3n via MediaPipe Python (desktop testing)

pip install mediapipe

from mediapipe.tasks.python import text

# Download model from: ai.google.dev/edge/mediapipe/solutions/genai/llm_inference
model_path = "gemma3n_e4b_it_q8_ekv1024.task"

options = text.language_model.LanguageModelOptions(
    base_options=text.BaseOptions(model_asset_path=model_path),
    max_tokens=512,
)

with text.LanguageModel.create_from_options(options) as llm:
    result = llm.generate_response("Translate 'hello' to French.")
    print(result.text)

Install fine-tuning dependencies

pip install trl peft bitsandbytes accelerate datasets

finetune_gemma3.py — LoRA SFT on a custom dataset

from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
from peft import LoraConfig, get_peft_model
from trl import SFTTrainer, SFTConfig
from datasets import load_dataset
import torch

model_id = "google/gemma-3-4b-it"

# 4-bit QLoRA for consumer GPUs (16 GB VRAM sufficient for 4B)
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
)

model = AutoModelForCausalLM.from_pretrained(
    model_id,
    quantization_config=bnb_config,
    device_map="auto",
)
tokenizer = AutoTokenizer.from_pretrained(model_id)

lora_config = LoraConfig(
    r=16,
    lora_alpha=32,
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
)

model = get_peft_model(model, lora_config)
model.print_trainable_parameters()
# trainable params: ~8.4M / 4B total — 0.2%

dataset = load_dataset("your-org/your-dataset", split="train")

training_args = SFTConfig(
    output_dir="./gemma3-4b-lora",
    num_train_epochs=3,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,
    learning_rate=2e-4,
    fp16=False,
    bf16=True,
    logging_steps=10,
    save_steps=100,
)

trainer = SFTTrainer(
    model=model,
    args=training_args,
    train_dataset=dataset,
    tokenizer=tokenizer,
)

trainer.train()