Hi all, the last couple of days i've ran into a pr...
# agents
p
Hi all, the last couple of days i've ran into a problem when using the openai realtime model with a seperate whisper-1 STT model. I'm working on an app where the user can speak with an ai agent and is getting a transcript at the end of the conversation. My problem is that the transcript is very inaccurate and sometimes even complete gibberish using multiple languages in 1 sentence, making sentences that weren't said at all... Has anyone experienced the same as well? This is my current configuration, i really hope any of you can help:
Copy code
transcription_model = "whisper-1"

    transcription_language = str(agent_config.get("language") or "en")
    noise_reduction_profile = "near_field"

    stt_kwargs: Dict[str, Any] = {
        "model": transcription_model,
        "language": transcription_language,
        "noise_reduction_type": noise_reduction_profile,
    }

    stt_backend = lk_stt.STT(**stt_kwargs)
    stt_model_name = getattr(stt_backend, "model", transcription_model)
    stt_provider = getattr(stt_backend, "provider", None)

    avatar_id = agent_config.get("avatar_id")
    voice = avatar_voices.get(avatar_id)

    vad = silero.VAD.load(
        min_speech_duration=0.3,
        min_silence_duration=0.8,
        prefix_padding_duration=0.3,
        activation_threshold=0.5,
    )

    session = AgentSession(
        llm=lk_realtime.RealtimeModel(
            modalities=["text", "audio"],
            voice=voice,
        ),
        stt=stt_backend,
        vad=vad,
        resume_false_interruption=False,
    )
l
I tried the following with a slight modification to the agent starter and it seems reliable with limited testing... what about this?
Copy code
import asyncio
import logging

from dotenv import load_dotenv
from livekit import rtc
from livekit.agents import (
    Agent,
    AgentServer,
    AgentSession,
    JobContext,
    JobProcess,
    cli,
    inference,
    room_io,
)
from livekit.plugins import noise_cancellation, silero
from livekit.plugins.turn_detector.multilingual import MultilingualModel
from livekit.plugins import openai
from livekit.agents import ConversationItemAddedEvent

logger = logging.getLogger("agent")

load_dotenv(".env.local")


class Assistant(Agent):
    def __init__(self) -> None:
        super().__init__(
            instructions="""You are a helpful voice AI assistant. The user is interacting with you via voice, even if you perceive the conversation as text.
            You eagerly assist users with their questions by providing information from your extensive knowledge.
            Your responses are concise, to the point, and without any complex formatting or punctuation including emojis, asterisks, or other symbols.
            You are curious, friendly, and have a sense of humor.""",
        )


server = AgentServer()

def prewarm(proc: JobProcess):
    proc.userdata["vad"] = silero.VAD.load()

server.setup_fnc = prewarm

@server.rtc_session()
async def my_agent(ctx: JobContext):
    ctx.log_context_fields = {
        "room": ctx.room.name,
    }

    session = AgentSession(
        stt=openai.STT(model="whisper-1"), 
        llm=openai.realtime.RealtimeModel(voice="marin"),
        vad=ctx.proc.userdata["vad"],
    )

    await session.start(
        agent=Assistant(),
        room=ctx.room,
        room_options=room_io.RoomOptions(
            audio_input=room_io.AudioInputOptions(
                noise_cancellation=lambda params: noise_cancellation.BVCTelephony()
                if params.participant.kind == rtc.ParticipantKind.PARTICIPANT_KIND_SIP
                else noise_cancellation.BVC(),
            ),
        ),
    )

    @session.on("conversation_item_added")
    def on_conversation_item_added(event: ConversationItemAddedEvent):
        <http://logger.info|logger.info>(f"Conversation item added from {event.item.role}: {event.item.text_content}. interrupted: {event.item.interrupted}")


if __name__ == "__main__":
    cli.run_app(server)
p
Hey Darryn thank you for your response, i've found that the issue didn't necessarily come form my configurations, but from using the STT model through the livekit openai sdk. After trying without the livekit wrapper and just using the direct openai sdk the transcriptions became way more accurate.