evals

Submodules

• harness
  • EvalAssertionFailure
    • EvalAssertionFailure.turn_index
    • EvalAssertionFailure.expectation_index
    • EvalAssertionFailure.event_name
    • EvalAssertionFailure.reason
  • EvalResult
    • EvalResult.scenario_name
    • EvalResult.passed
    • EvalResult.failures
    • EvalResult.duration_ms
    • EvalResult.events_seen
    • EvalResult.debug_log
    • EvalResult.skipped
  • EvalTurnProgress
    • EvalTurnProgress.turn_index
    • EvalTurnProgress.expectation_index
    • EvalTurnProgress.event_name
    • EvalTurnProgress.status
    • EvalTurnProgress.detail
  • EvalSession
    • EvalSession.__init__()
    • EvalSession.from_scenario()
    • EvalSession.run()
• judge
  • JudgeVerdict
    • JudgeVerdict.verdict
    • JudgeVerdict.reason
    • JudgeVerdict.raw_response
    • JudgeVerdict.passed
  • EvalJudge
    • EvalJudge.__init__()
    • EvalJudge.from_config()
    • EvalJudge.add_user_message()
    • EvalJudge.add_assistant_message()
    • EvalJudge.evaluate()
• scenario
  • EvalFunctionCall
    • EvalFunctionCall.name
    • EvalFunctionCall.args
  • EvalExpectation
    • EvalExpectation.event
    • EvalExpectation.within_ms
    • EvalExpectation.text_contains
    • EvalExpectation.calls
    • EvalExpectation.eval
    • EvalExpectation.absent
  • EvalSendAfter
    • EvalSendAfter.event
    • EvalSendAfter.delay_ms
  • EvalTurn
    • EvalTurn.user
    • EvalTurn.expect
    • EvalTurn.dtmf
    • EvalTurn.send_after
    • EvalTurn.image
  • EvalScenario
    • EvalScenario.name
    • EvalScenario.turns
    • EvalScenario.context
    • EvalScenario.judge
    • EvalScenario.bot_audio
    • EvalScenario.transcriber
    • EvalScenario.user_audio
    • EvalScenario.trigger_disconnect
    • EvalScenario.source_path
    • EvalScenario.load()
  • describe_config()
• serializer
  • RTVIEvalSerializer
    • RTVIEvalSerializer.__init__()
    • RTVIEvalSerializer.set_capture_audio()
    • RTVIEvalSerializer.get_user_image()
    • RTVIEvalSerializer.serialize()
    • RTVIEvalSerializer.deserialize()
• services
  • kokoro_service()
  • cartesia_service()
  • whisper_service()
  • moonshine_service()
  • ollama_service()
  • openai_service()
• speech
  • tts_sample_rate()
  • tts_cache_key()
  • EvalSpeech
    • EvalSpeech.__init__()
    • EvalSpeech.sample_rate
    • EvalSpeech.from_config()
    • EvalSpeech.start()
    • EvalSpeech.generate()
    • EvalSpeech.aclose()
• suite
  • capture_pipeline_logs()
  • EvalRun
    • EvalRun.bot
    • EvalRun.scenario
    • EvalRun.scenario_path
    • EvalRun.bot_path
    • EvalRun.bot_url
    • EvalRun.runner_body_path
    • EvalRun.status
    • EvalRun.result
    • EvalRun.error
    • EvalRun.started_at
    • EvalRun.duration_ms
  • EvalManifest
    • EvalManifest.runs
    • EvalManifest.spawn
    • EvalManifest.python
    • EvalManifest.concurrency
    • EvalManifest.base_port
    • EvalManifest.runs_dir
    • EvalManifest.record
    • EvalManifest.cache_dir
    • EvalManifest.load()
  • EvalSuite
    • EvalSuite.__init__()
    • EvalSuite.filter()
    • EvalSuite.run()
• transcribe
  • EvalTranscriber
    • EvalTranscriber.__init__()
    • EvalTranscriber.from_config()
    • EvalTranscriber.start()
    • EvalTranscriber.transcribe()
    • EvalTranscriber.aclose()
• transport
  • EvalMicrophone
    • EvalMicrophone.__init__()
    • EvalMicrophone.add_audio()
    • EvalMicrophone.reset()
    • EvalMicrophone.run()
  • EvalTransportParams
    • EvalTransportParams.add_wav_header
    • EvalTransportParams.serializer
    • EvalTransportParams.session_timeout
    • EvalTransportParams.allowed_origins
  • EvalInputTransport
    • EvalInputTransport.__init__()
    • EvalInputTransport.configure_mic()
    • EvalInputTransport.process_frame()
    • EvalInputTransport.stop()
    • EvalInputTransport.cancel()
  • EvalOutputTransport
  • EvalTransport
    • EvalTransport.__init__()
    • EvalTransport.input()
    • EvalTransport.output()