Evaluation quick start

This quick start will get you up and running with our evaluation SDK and Experiments UI.

1. Install LangSmith

Python

pip install -U langsmith

TypeScript

yarn add langsmith

2. Create an API key

To create an API key head to the Settings page. Then click Create API Key.

3. Set up your environment

Shell

export LANGCHAIN_TRACING_V2=true
export LANGCHAIN_API_KEY=<your-api-key>

4. Run your evaluation

Python

Requires langsmith>=0.2.0

from langsmith import Client

# 1. Create and/or select your dataset
client = Client()
dataset = client.clone_public_dataset(
    "https://smith.langchain.com/public/a63525f9-bdf2-4512-83e3-077dc9417f96/d"
)

# 2. Define an evaluator
def is_concise(outputs: dict, reference_outputs: dict) -> bool:
    return len(outputs["answer"]) < (3 * len(reference_outputs["answer"]))

# 3. Define the interface to your app
def chatbot(inputs: dict) -> dict:
    return {"answer": inputs["question"] + " is a good question. I don't know the answer."}

# 4. Run an evaluation
experiment_results = client.evaluate(
    chatbot,
    data=dataset,
    evaluators=[is_concise],
    experiment_prefix="my first experiment ",
    max_concurrency=4,
)

TypeScript

Requires langsmith>=0.2.9

import { Client } from "langsmith";
import { evaluate } from "langsmith/evaluation";
import type { EvaluationResult } from "langsmith/evaluation";

// 1. Define a dataset
const client = new Client();
const datasetName = "my first dataset"
const dataset = await client.clonePublicDataset(
"https://smith.langchain.com/public/a63525f9-bdf2-4512-83e3-077dc9417f96/d",
{ datasetName: datasetName }
)

// 2. Define an evaluator
function isConcise({ outputs, referenceOutputs }: { outputs?: Record<string, any>, referenceOutputs?: Record<string, any> }): EvaluationResult {
const score = outputs?.answer.length < 3 * referenceOutputs?.answer.length;
return { key: "is_concise", score: score };
}

// 3. Run an evaluation
await evaluate(
(inputs: { question: string }) => {
return {
answer: inputs.question + " Good question. I don't know the answer"
};
}, {
data: datasetName,
evaluators: [isConcise],
experimentPrefix: "my first experiment ",
maxConcurrency: 4,
});

5. View Experiments UI

Click the link printed out by your evaluation run to access the LangSmith Experiments UI, and explore the results of your evaluation.

Next steps

For conceptual explanations see the Conceptual guide. See the How-to guides for answers to “How do I….?” format questions. For end-to-end walkthroughs see Tutorials. For comprehensive descriptions of every class and function see the API reference.