Evaluation

from netra import Netra
from netra.evaluation import TurnType

Netra.init(app_name="sample-app")

result = Netra.evaluation.create_dataset(
    name="Customer Support QA",
    tags=["support", "qa", "v1"],
    turn_type=TurnType.SINGLE,  # or TurnType.MULTI for multi-turn
)

print(f"Dataset created: {result.id}")
print(f"Name: {result.name}")
print(f"Tags: {result.tags}")

create_dataset(
    name: str,
    tags: Optional[List[str]] = None,
    turn_type: TurnType = TurnType.SINGLE,
) -> CreateDatasetResponse | None

from netra import Netra
from netra.evaluation import DatasetItem

Netra.init(app_name="sample-app")

result = Netra.evaluation.add_dataset_item(
    dataset_id="dataset-123",
    item=DatasetItem(
        input="What is the return policy for electronics?",
        expected_output="Electronics can be returned within 30 days with original packaging.",
        tags=["policy", "returns"],
        metadata={"category": "electronics", "priority": "high"},
    ),
)

print(f"Item added: {result.id}")
print(f"Input: {result.input}")

add_dataset_item(
    dataset_id: str,
    item: DatasetItem,
) -> AddDatasetItemResponse | None

from netra import Netra

Netra.init(app_name="sample-app")

# Fetch all datasets
all_datasets = Netra.evaluation.get_all_datasets()

# Fetch only datasets that include a specific tag
tagged_datasets = Netra.evaluation.get_all_datasets(tag="production")

if tagged_datasets:
    print(f"Found {len(tagged_datasets.datasets)} dataset(s)")
    for dataset in tagged_datasets.datasets:
        print(f"{dataset.id} | {dataset.name} | {dataset.tags}")

get_all_datasets(
    tag: Optional[str] = None,
) -> GetAllDatasetsResponse | None

from netra import Netra

Netra.init(app_name="sample-app")

dataset = Netra.evaluation.get_dataset(dataset_id="dataset-123")

print(f"Total items: {len(dataset.items)}")

for item in dataset.items:
    print(f"ID: {item.id}")
    print(f"Input: {item.input}")
    print(f"Expected: {item.expected_output}")
    print("---")

get_dataset(
    dataset_id: str,
) -> GetDatasetItemsResponse | None

from netra import Netra
from openai import OpenAI

Netra.init(app_name="sample-app")

client = OpenAI()

def my_task(input_data):
    """Task function that processes each dataset item."""
    response = client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": input_data},
        ],
    )
    return response.choices[0].message.content

# Get dataset
dataset = Netra.evaluation.get_dataset(dataset_id="dataset-123")

# Run test suite
result = Netra.evaluation.run_test_suite(
    name="GPT-4o Mini Evaluation",
    data=dataset,
    task=my_task,
    evaluators=["correctness", "relevance"],  # Optional evaluator IDs
    max_concurrency=10,
)

print(f"Run ID: {result['runId']}")
print(f"Items processed: {len(result['items'])}")

run_test_suite(
    name: str,
    data: Dataset,
    task: Callable[[Any], Any],
    evaluators: Optional[List[Any]] = None,
    max_concurrency: int = 50,
) -> Optional[Dict[str, Any]]

from netra import Netra

Netra.init(app_name="sample-app")

# After running a test suite
result = Netra.evaluation.run_test_suite(
    name="GPT-4o Mini Evaluation",
    data=dataset,
    task=my_task,
    evaluators=["correctness", "relevance"],
)

run_id = result["runId"]

# Fetch the full run results
run_results = Netra.evaluation.get_run_results(run_id=run_id)

print(f"Run results: {run_results}")

get_run_results(
    run_id: str,
) -> Any | None