LangSmith Guide 2025: LLM Observability, Tracing & Evaluation

Install

pip install langsmith

Environment variables (.env)

LANGCHAIN_API_KEY=ls__...          # from smith.langchain.com → Settings → API Keys
LANGCHAIN_TRACING_V2=true
LANGCHAIN_PROJECT=my-app           # optional — groups traces into a named project

rag_chain.py — traced automatically

import os
os.environ["LANGCHAIN_TRACING_V2"] = "true"
os.environ["LANGCHAIN_API_KEY"] = "ls__..."
os.environ["LANGCHAIN_PROJECT"] = "rag-demo"

from langchain_openai import ChatOpenAI, OpenAIEmbeddings
from langchain_community.vectorstores import FAISS
from langchain.chains import RetrievalQA

embeddings = OpenAIEmbeddings()
vectorstore = FAISS.from_texts(["LangSmith traces LangChain apps."], embeddings)
retriever = vectorstore.as_retriever()

llm = ChatOpenAI(model="gpt-4o-mini")
qa_chain = RetrievalQA.from_chain_type(llm=llm, retriever=retriever)

# This call is fully traced in LangSmith — retrieval + LLM + chain
result = qa_chain.invoke({"query": "What does LangSmith do?"})
print(result["result"])

traceable_example.py

from langsmith import traceable
from openai import OpenAI
import anthropic

openai_client = OpenAI()
anthropic_client = anthropic.Anthropic()

@traceable(name="openai-call", run_type="llm")
def call_openai(user_message: str) -> str:
    response = openai_client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": user_message}],
    )
    return response.choices[0].message.content

@traceable(name="anthropic-call", run_type="llm")
def call_anthropic(user_message: str) -> str:
    message = anthropic_client.messages.create(
        model="claude-3-5-haiku-20241022",
        max_tokens=1024,
        messages=[{"role": "user", "content": user_message}],
    )
    return message.content[0].text

@traceable(name="my-pipeline")
def pipeline(query: str) -> str:
    openai_answer = call_openai(query)
    anthropic_answer = call_anthropic(query)
    return f"OpenAI: {openai_answer}\nAnthropic: {anthropic_answer}"

result = pipeline("Summarize observability in one sentence.")

Push a prompt to Prompt Hub

from langsmith import Client
from langchain_core.prompts import ChatPromptTemplate

client = Client()

# Create and push a versioned prompt
prompt = ChatPromptTemplate.from_messages([
    ("system", "You are a helpful assistant that answers in {language}."),
    ("human", "{question}"),
])

client.push_prompt("my-qa-prompt", object=prompt)
# Creates: smith.langchain.com/hub/your-org/my-qa-prompt

Pull a prompt in code

from langchain import hub

# Pull the latest version
prompt = hub.pull("your-org/my-qa-prompt")

# Pull a specific commit hash for reproducibility
prompt = hub.pull("your-org/my-qa-prompt:abc12345")

# Use in a chain
from langchain_openai import ChatOpenAI
chain = prompt | ChatOpenAI(model="gpt-4o-mini")
result = chain.invoke({"language": "French", "question": "What is LangSmith?"})

Create dataset and run eval

from langsmith import Client
from langsmith.evaluation import evaluate, LangChainStringEvaluator
from langchain_openai import ChatOpenAI

client = Client()

# 1. Create a dataset
dataset = client.create_dataset("qa-eval-dataset", description="QA pairs for eval")
client.create_examples(
    inputs=[
        {"question": "What is the capital of France?"},
        {"question": "Who invented the telephone?"},
    ],
    outputs=[
        {"answer": "Paris"},
        {"answer": "Alexander Graham Bell"},
    ],
    dataset_id=dataset.id,
)

# 2. Define the system under test
def my_app(inputs: dict) -> dict:
    llm = ChatOpenAI(model="gpt-4o-mini")
    response = llm.invoke(inputs["question"])
    return {"answer": response.content}

# 3. Run evaluation with LLM-as-judge
results = evaluate(
    my_app,
    data="qa-eval-dataset",
    evaluators=[
        LangChainStringEvaluator("qa"),        # correctness judge
        LangChainStringEvaluator("criteria",   # custom criterion
            config={"criteria": "conciseness"}),
    ],
    experiment_prefix="gpt-4o-mini-baseline",
)

print(results.to_pandas())

Add trace examples to a dataset via API

from langsmith import Client

client = Client()

# Get runs from a project that you've flagged as interesting
runs = client.list_runs(
    project_name="my-app",
    filter='and(eq(feedback_key, "user_thumbs_down"), eq(feedback_score, 0))',
)

# Add them to a dataset for later eval
dataset = client.read_dataset(dataset_name="failures-dataset")
client.create_examples(
    inputs=[run.inputs for run in runs],
    outputs=[run.outputs for run in runs],
    dataset_id=dataset.id,
)