Runs resource

Access these methods through client.runs.

`client.runs.start`

Start a run from an uploaded agent name or complete contract.

Signature

async def start(
    agent: str | AgentContractRequest,
    benchmark: str,
    *,
    model: str | None = None,
    concurrency: int = 5,
    task_ids: Sequence[str] | None = None,
    slice_str: str | None = None,
    dataset: str | None = None,
    label: str | None = None,
    lambda_function: str | None = None,
    provider: str | None = None,
    agent_kwargs: Mapping[str, str] | None = None,
    secrets: Mapping[str, str] | None = None,
    service_headers: Mapping[str, str] | None = None,
    webhook_intervals: Sequence[int] | None = None,
    ignore_custom_services: bool = False,
) -> StartBenchmarkResponse

Parameters

required · str | AgentContractRequest

required · str

optional · str | None · default:

optional · int · default:

optional · Sequence[str] | None · default:

optional · str | None · default:

optional · Mapping[str, str] | None · default:

optional · Sequence[int] | None · default:

optional · bool · default:

Returns StartBenchmarkResponse For examples, see the Python SDK guide.

`client.runs.fetch`

Fetch the latest state of a run.

Signature

async def fetch(run_id: UUID) -> FetchBenchmarkResponse

Parameters

required · UUID

Returns FetchBenchmarkResponse For examples, see the Python SDK guide.

`client.runs.list`

List runs using typed filters and pagination.

Signature

async def list(request: FetchBenchmarksRequest | None = None) -> FetchBenchmarksResponse

Parameters

optional · FetchBenchmarksRequest | None · default:

Returns FetchBenchmarksResponse For examples, see the Python SDK guide.

`client.runs.stream`

Yield typed updates until the run completes or disconnects.

Signature

async def stream(run_id: UUID) -> AsyncIterator[FetchBenchmarkResponse]

Parameters

required · UUID

Returns AsyncIterator[FetchBenchmarkResponse] For examples, see the Python SDK guide.

`client.runs.results`

Fetch final results or upload them and return S3 links.

Overload 1

async def results(
    run_id: UUID,
    *,
    task_ids: Sequence[str] | None = None,
    upload_to_s3: Literal[False] = False,
) -> FinalViewResponse

Overload 2

async def results(
    run_id: UUID,
    *,
    task_ids: Sequence[str] | None = None,
    upload_to_s3: Literal[True],
) -> S3UploadResultsResponse

Overload 3

async def results(
    run_id: UUID,
    *,
    task_ids: Sequence[str] | None = None,
    upload_to_s3: bool,
) -> RetrieveResultsResponse

Parameters

required · UUID

optional · Sequence[str] | None · default:

optional · bool · default:

Returns The selected overload determines the return type.

Overload	Return type
1	`FinalViewResponse`
2	`S3UploadResultsResponse`
3	RetrieveResultsResponse

For examples, see the Python SDK guide.

`client.runs.metadata`

Fetch the stored launch metadata for a run.

Signature

async def metadata(run_id: UUID) -> FetchBenchmarkMetadataResponse

Parameters

required · UUID

Returns FetchBenchmarkMetadataResponse For examples, see the Python SDK guide.

`client.runs.results_exist`

Check whether the canonical result file already exists in S3.

Signature

async def results_exist(run_id: UUID) -> ResultsExistResponse

Parameters

required · UUID

Returns ResultsExistResponse For examples, see the Python SDK guide.

`client.runs.analyze`

Yield typed progress events while analyzing a finished run.

Signature

async def analyze(
    run_id: UUID,
    *,
    no_cache: bool = False,
    lambda_function: str | None = None,
) -> AsyncIterator[AnalyzeEvent]

Parameters

required · UUID

optional · bool · default:

optional · str | None · default:

Returns AsyncIterator[AnalyzeEvent] For examples, see the Python SDK guide.

`client.runs.stream_outputs`

Yield chunks from the tar archive containing run outputs.

Signature

async def stream_outputs(run_id: UUID, *, task_ids: Sequence[str] | None = None) -> AsyncIterator[bytes]

Parameters

required · UUID

optional · Sequence[str] | None · default:

Returns AsyncIterator[bytes] For examples, see the Python SDK guide.

`client.runs.stop`

Stop a run or selected tasks.

Signature

async def stop(
    run_id: UUID,
    *,
    force: bool = False,
    task_ids: Sequence[str] | None = None,
) -> StopBenchmarkResponse

Parameters

required · UUID

optional · bool · default:

optional · Sequence[str] | None · default:

Returns StopBenchmarkResponse For examples, see the Python SDK guide.

`client.runs.resume`

Resume unfinished work for a run.

Signature

async def resume(
    run_id: UUID,
    *,
    concurrency: int | None = None,
    task_ids: Sequence[str] | None = None,
    secrets: Mapping[str, str] | None = None,
    service_headers: Mapping[str, str] | None = None,
    from_scratch: bool = False,
    benchmark_url: str | None = None,
) -> RetryOrResumeBenchmarkResponse

Parameters

required · UUID

optional · int | None · default:

optional · Sequence[str] | None · default:

optional · Mapping[str, str] | None · default:

optional · bool · default:

optional · str | None · default:

Returns RetryOrResumeBenchmarkResponse For examples, see the Python SDK guide.

`client.runs.retry`

Retry failed or selected work for a run.

Signature

async def retry(
    run_id: UUID,
    *,
    concurrency: int | None = None,
    task_ids: Sequence[str] | None = None,
    secrets: Mapping[str, str] | None = None,
    service_headers: Mapping[str, str] | None = None,
    from_scratch: bool = False,
    benchmark_url: str | None = None,
) -> RetryOrResumeBenchmarkResponse

Parameters

required · UUID

optional · int | None · default:

optional · Sequence[str] | None · default:

optional · Mapping[str, str] | None · default:

optional · bool · default:

optional · str | None · default:

Returns RetryOrResumeBenchmarkResponse For examples, see the Python SDK guide.

Get started

Agents

Runs

Benchmarks

Integrations

Python SDK

CLI reference

Python SDK reference

Self-hosting

`client.runs.start`

`client.runs.fetch`

`client.runs.list`

`client.runs.stream`

`client.runs.results`

`client.runs.metadata`

`client.runs.results_exist`

`client.runs.analyze`

`client.runs.stream_outputs`

`client.runs.stop`

`client.runs.resume`

`client.runs.retry`

​client.runs.start

​client.runs.fetch

​client.runs.list

​client.runs.stream

​client.runs.results

​client.runs.metadata

​client.runs.results_exist

​client.runs.analyze

​client.runs.stream_outputs

​client.runs.stop

​client.runs.resume

​client.runs.retry

`client.runs.start`

`client.runs.fetch`

`client.runs.list`

`client.runs.stream`

`client.runs.results`

`client.runs.metadata`

`client.runs.results_exist`

`client.runs.analyze`

`client.runs.stream_outputs`

`client.runs.stop`

`client.runs.resume`

`client.runs.retry`