import logging import os from datetime import datetime, timezone import time from pathlib import Path import json from dotenv import load_dotenv from livekit import ( rtc, api, ) from livekit.agents import ( Agent, AgentServer, AgentSession, JobContext, JobProcess, RunContext, ChatContext, cli, function_tool, room_io, metrics ) from livekit.agents import UserStateChangedEvent, AgentStateChangedEvent from livekit.agents import AutoSubscribe from livekit.plugins import ( openai, elevenlabs, silero, ) from livekit.plugins.turn_detector.multilingual import MultilingualModel from livekit.plugins import noise_cancellation from stt import build_stt, build_cloud_stt, build_local_stt from llm import build_llm, build_cloud_llm, build_local_llm from tts import build_tts from fino_rag import query_info, query_info_stub from fino_indexing import get_index from config import SYSTEM_PROMPT, execution_mode from mongo_push import push_to_mongo logger = logging.getLogger("finopayment_agent") # basic formatter logging.basicConfig( filename="app.log", level=logging.DEBUG, format="%(asctime)s [%(levelname)s] %(name)s - %(message)s", ) # make the noisy bits verbose for name in ( "livekit", "livekit.agents", "httpx", "openai", "elevenlabs", ): logging.getLogger(name).setLevel(logging.DEBUG) class Assistant(Agent): def __init__(self, system_prompt=SYSTEM_PROMPT) -> None: super().__init__( instructions=system_prompt, tools=[query_info if execution_mode == "local" else query_info_stub], ) async def on_enter(self): # await self.session.generate_reply() await self.session.say("Hello!, I am Purnima from Fino Payments Bank. Can I have a moment of your time?") server = AgentServer() def prewarm(proc: JobProcess): # load silero vad once per worker process proc.userdata["vad"] = silero.VAD.load( # activation_threshold=0.6, #Confidence threshold to classify audio as speech # min_silence_duration=0.3, # min_speech_duration=0.2, ) # warm up the retrieval index (loads embeddings, connects to qdrant) if execution_mode == "local": get_index() server.setup_fnc = prewarm async def write_transcript( ctx: JobContext, session: AgentSession, ): # This code runs when the agent task is shutting down current_date = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") reports_dir = Path("reports") reports_dir.mkdir(exist_ok=True) filename = reports_dir / f"transcript_{ctx.room.name}_{current_date}.json" try: # We use session.history.to_dict() to get the full chat log with open(filename, 'w', encoding='utf-8') as f: json.dump(session.history.to_dict(), f, indent=2, ensure_ascii=False) logger.info(f" Transcript saved to {filename}") return filename except Exception as e: logger.error(f"Failed to save transcript: {e}") return None async def push_session_to_mongo( end_time: float, start_time: float, ctx: JobContext, session: AgentSession, ): transcript_json = session.history.to_dict() duration = int(end_time - start_time) call_datetime = datetime.fromtimestamp(start_time, timezone.utc).strftime('%Y-%m-%d %H:%M:%S') recording_url = f"{os.getenv('EOS_ENDPOINT')}/{os.getenv('EOS_BUCKET_NAME')}/livekit/{ctx.room.name}.ogg" push_to_mongo(duration, call_datetime, transcript_json, recording_url) # async def log_usage(usage_collector: metrics.UsageCollector): # summary = usage_collector.get_summary() # logger.info(f"Usage: {summary}") @breezy-insurance-66772.rtc_session() async def entry( ctx: JobContext ): start_time = time.time() # add room info to all log entries ctx.log_context_fields = { "room": ctx.room.name, } await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY) if execution_mode == "local": stt_engine = build_local_stt() llm_engine = build_local_llm() else: stt_engine = build_cloud_stt() llm_engine = build_cloud_llm() # small hardcoded workaround to use cloud llm always, as we are evaluating gpt 4o mini # stt_engine = build_cloud_stt() # llm_engine = build_cloud_llm() # llm_engine = build_local_llm() tts_engine = build_tts() # usage_collector = metrics.UsageCollector() session = AgentSession( stt=stt_engine, llm=llm_engine, tts=tts_engine, max_endpointing_delay=0.2, # turn_detection=MultilingualModel(), # turn_detection="vad", vad=ctx.proc.userdata["vad"], preemptive_generation=True, # min_interruption_duration=0.2, false_interruption_timeout=2.0, resume_false_interruption=True, ) @ctx.room.on("track_published") def on_track_published( publication: rtc.RemoteTrackPublication, participant: rtc.RemoteParticipant ): # Only subscribe to audio tracks from participants (not screen shares) if publication.kind == rtc.TrackKind.KIND_AUDIO: publication.set_subscribed(True) # Subscribe to audio tracks from the primary speaker only for p in list(ctx.room.remote_participants.values())[1] # Only first participant for pub in p.track_publications.values(): if pub.kind == rtc.TrackKind.KIND_AUDIO: pub.set_subscribed(True) # ctx.add_shutdown_callback(lambda: write_transcript(ctx=ctx, session=session)) # ctx.add_shutdown_callback(lambda: log_usage(usage_collector)) ctx.add_shutdown_callback(lambda: push_session_to_mongo(end_time=time.time(), start_time=start_time, ctx=ctx, session=session)) egress_req = api.RoomCompositeEgressRequest( room_name=ctx.room.name, audio_only=True, file_outputs=[ api.EncodedFileOutput( file_type=api.EncodedFileType.OGG, filepath=f"livekit/{ctx.room.name}.ogg", s3=api.S3Upload( bucket=os.getenv("EOS_BUCKET_NAME"), access_key=os.getenv("EOS_ACCESS_KEY"), secret=os.getenv("EOS_SECRET_KEY"), endpoint=os.getenv("EOS_ENDPOINT"), ), ) ], ) lkapi = api.LiveKitAPI() await lkapi.egress.start_room_composite_egress(egress_req) # await lkapi.aclose() # await ctx.wait_for_participant() try: # Start egress BEFORE the session to ensure it's ready to capture audio # try: # await lkapi.egress.start_room_composite_egress(egress_req) # logger.info("Started room composite egress for audio recording to S3.") # except Exception as e: # logger.error(f"Failed to start egress: {e}", exc_info=True) # continue; session can still run even if egress fails # Now start the session await session.start( agent=Assistant(system_prompt=SYSTEM_PROMPT), room=ctx.room, room_options=room_io.RoomOptions( audio_input=room_io.AudioInputOptions( audio_sample_rate=16000, # noise_cancellation=noise_cancellation.BVC(), room_output_options=RoomOutputOptions(transcription_enabled=True) ), ) ) except Exception as e: logger.error(f"Failed to start session: {e}", exc_info=True) raise finally: try: await lkapi.aclose() logger.info("Closed lkapi.") except Exception as e: logger.debug(f"Failed to close lkapi: {e}", exc_info=True) # if name == "__main__": # cli.run_app(server) if name == "__main__": cli.run_app(WorkerOptions(entrypoint_fnc=entrypoint, prewarm_fnc=prewarm)) I'm facing background voices interruptions where agent is getting interrupted with multiple background human voice even after enabling noisecancellation bvc telephony . audio coming from exotel sip. and also stt is transcribing all those interuptions as well.

Thanks for reaching out, Developer. I'm an AI assistant and I'm here to help. Give me a moment while I research this topic.

all these are done. but still background voices are coming. exact noise cancellation is : # noise_cancellation=noise_cancellation.BVCTelephony(), prominiently background voices are from another human participant, where agent is not able to cancel their voice, ignore them. agent is getting interupted. and also, no noise cancellation is enabled from SIP trunk.

Thanks — I hear the frustration. You’ve enabled

BVCTelephony()

and confirmed no trunk-level filtering, but the agent is still getting interrupted by other human participants’ audio. That strongly points to the agent subscribing to multiple participant tracks (so noise filtering inside a single track can’t stop the agent from hearing other people). What I recommend next (short, actionable) 1) Restrict the agent to a single participant (prevent subscribing to other people) • Use RoomIO (or the

participant

option when creating RoomIO/RoomOptions) so the agent only subscribes to the intended caller’s audio track. This prevents the agent from receiving any other participant’s audio entirely. • Example snippet (where you create your session/room I/O):

# identify the intended participant identity (e.g., caller identity)
participant_identity = "caller-123"  # replace with actual identity

room_io = room_io.RoomIO(
    session=session,
    room=ctx.room,
    participant=participant_identity,
)

await room_io.start()
await session.start(agent=Assistant(system_prompt=SYSTEM_PROMPT))

Why: BVCTelephony filters inside a single participant’s stream; it cannot stop the agent from receiving separate tracks belonging to other participants. 2) Confirm and log which participants the agent is subscribed to • Add runtime logs to ensure the agent isn’t subscribed to extra participants or duplicate joins:

for p in ctx.room.remote_participants.values():
    <http://logger.info|logger.info>(f"remote participant: identity={p.identity} kind={p.kind} subscribed_tracks={list(p.track_publications.keys())}")

• Also log when the agent starts processing audio from a participant:

@session.on("user_speaking")
def on_user_speaking(participant):
    <http://logger.info|logger.info>(f"user speaking: {participant.identity}")

Why: This shows which identity is actually triggering the session and whether the agent is hearing unexpected participants. 3) Ensure no duplicate joins / multiple devices • Verify callers/agents aren’t joining the same LiveKit room from phone + web (or call-forward). If the same user is joined twice, the agent will receive both audio tracks and might transcribe both. 4) Keep BVCTelephony enabled (good) and avoid trunk-side filters simultaneously • You already have BVCTelephony referenced — ensure it is actively set in the RoomOptions audio input (not just imported/commented). Since your trunk has no filtering, keep BVCTelephony active and let it handle per-stream noise. 5) If you must allow multiple participants, treat them as separate inputs • If your workflow requires other participants in the room (e.g., supervisor, agent), then: ◦ Only subscribe to the caller’s audio for STT/agent processing; keep other participants unsubscribed for the agent. ◦ Or implement per-participant routing: only route the caller’s track into the AgentSession/STT pipeline. Diagnostic information I need if this persists • Paste the minimal snippet where you set up RoomOptions/RoomIO and start the session (the exact lines where you set noise_cancellation and participant selection). • A list of ctx.room.remote_participants printed at runtime (the logging snippet above). • Whether callers sometimes join via multiple devices (phone + web) or call transfers happen (that can create multiple tracks). If you share those three items I’ll: • Confirm if RoomIO is being used correctly or if the agent is inadvertently auto-subscribing to all tracks. • Suggest the exact code changes to force single-participant audio and, if needed, a short test plan to validate the fix. Would you like me to review the RoomOptions/RoomIO snippet next? Sources: How to set a fixed participant for agent audio processing | LiveKit | How to Test Your Agent Using Another Agent | LiveKit | Agents Framework | livekit/livekit_composite | DeepWiki

not other participants. It's like if i have received the call , and I wish to talk , but if there are multiple background voices at my end. agent is getting interrupted in all, and not able to ignore them and just focus on my mine.

Thanks — that clarification helps a lot. Since the background voices are physically near your microphone (your side of the call), the problem is that the mic is picking up multiple people and the agent’s VAD/STT pipeline is treating those voices as legitimate speech. BVCTelephony/BVC can help, but they can’t perfectly separate multiple co-located human speakers if they’re all being captured by the same microphone. Here’s a focused plan you can try next, plus a few diagnostic checks. I’ll keep steps short and actionable. Quick checklist to try now • Confirm which cancellation you actually set in RoomOptions (not just imported): ◦ If the microphone is a local device (laptop/phone/softphone), try

BVC()

instead of `BVCTelephony()`: ▪︎ Use

BVCTelephony()

only when the incoming audio is truly narrowband 8 kHz SIP on the server side. • Increase VAD selectivity so short/low-confidence speech is ignored: ◦ activation_threshold → 0.6–0.8 ◦ min_speech_duration → 0.2–0.3 s ◦ min_silence_duration → 0.8–1.5 s • Raise interruption requirements: ◦ min_interruption_duration → ~0.6–0.8 s ◦ min_interruption_words → 1–2 ◦ false_interruption_timeout → 1.0–1.5 s • Temporarily disable automatic interruptions while the agent speaks: ◦ Use

allow_interruptions=False

during agent TTS output or when the agent is mid-turn to avoid reacting to ambient speech. Why these help (short) • BVC is better for device/close-mic audio; BVCTelephony is tuned for narrowband SIP streams. Using the right model increases separation quality. • Tightening VAD and interruption thresholds reduces false triggers from brief background mutter or overlapping low-volume speech. • Disabling interruptions during agent speech prevents the agent from flipping state due to short bursts nearby. Hardware / environment suggestions (important) • Use a close-talk headset or directional mic — physical isolation is often the single most effective fix for co-located human voices. • If available, enable hardware-level beamforming/noise-suppression (e.g., NVIDIA Broadcast/RTX Voice) rather than stacking software filters. • Reduce competing sound sources (move others away, close doors) where possible. Diagnostic info I need if this continues Please share these minimal items so I can give a precise tweak: 1. The exact RoomOptions / noise_cancellation lines you set (paste 6–12 lines). 2. Whether the microphone is built-in or an external headset. 3. A short excerpt of logs showing an interruption event and the transcript the STT produced (timestamps if possible). If you prefer, paste the RoomOptions + AgentSession snippet you use to create

session

and start the agent — I’ll point out the smallest change to test first. Sources: Choosing the right noise cancellation for your agent | LiveKit | Voice AI quickstart | LiveKit Documentation | Google AI and LiveKit | LiveKit Documentation | Agents Framework | livekit/livekit_composite | DeepWiki | examples/voice_agents | livekit/livekit_composite | DeepWiki

Temporarily disable automatic interruptions while the agent speaks: ◦ Use

allow_interruptions=False

during agent TTS output or when the agent is mid-turn to avoid reacting to ambient speech. where to allow this?

Good question — you can disable interruptions in two places depending on how granular you want control: globally for the session, or per-turn when the agent is actually speaking. Below are both options with short code examples that fit into your existing flow. 1) Disable interruptions for the whole session (simple) Set

allow_interruptions=False

when you create the AgentSession. The agent will not be interrupted for the entire session.

session = AgentSession(
    stt=stt_engine,
    llm=llm_engine,
    tts=tts_engine,
    vad=ctx.proc.userdata["vad"],
    preemptive_generation=True,
    allow_interruptions=False,     # disable interruptions globally
    min_interruption_duration=0.8,
    false_interruption_timeout=1.5,
)

2) Disable interruptions only while the agent is speaking (recommended) Keep interruptions enabled generally, but temporarily turn them off while you generate/play TTS. Use the per-call option on the `say`/`generate_reply` or control the agent’s speaking handle. The API may expose an

allow_interruptions

argument when starting a speak action — set it False there, then restore behavior afterward. Example pattern (pseudocode showing the idea):

# allow normal interruptions by default
session = AgentSession(..., allow_interruptions=True, ...)

# when agent is about to speak:
speak_handle = await session.say("I will explain now...", allow_interruptions=False)
# `allow_interruptions=False` keeps the agent's TTS output from being interrupted by ambient speech

# optionally await completion
await speak_handle.wait_done()
# after done, session resumes normal interruptible behavior

If your SDK version does not accept

allow_interruptions

on

say

, use these alternatives: • Temporarily set

session.allow_interruptions = False

right before

say()

and set it back to True after the TTS completes. • Use

generate_reply(allow_interruptions=False)

if your agent uses that method to produce spoken replies. Where to add this in your code • Put per-turn

allow_interruptions=False

in the same place you call

session.say(...)

or

session.generate_reply(...)

inside your

entry

function or in the Agent’s

on_enter

/ reply flow. That ensures the agent is protected from background noise while it’s speaking. If you want, paste the snippet where you call

session.say(...)

or where AgentSession is created, and I’ll show the exact line to add in your script.

what are these two. and how are these helping? audio_sample_rate=16000, # noise_cancellation=noise_cancellation.BVCTelephony(), room_output_options=RoomOutputOptions(transcription_enabled=True)

Good question — I’ll explain each of those three settings, what they do, and how they affect noise cancellation and transcription so you know why they matter for your scenario. audio_sample_rate=16000 • What it is: The sample rate defines how many audio samples per second your pipeline expects (16,000 samples/sec = 16 kHz). • Why it matters: VAD, STT, and TTS models expect particular sample rates. Many STT engines and Silero VAD work well at 16 kHz. If the microphone or SIP leg is 8 kHz and you pass audio as 16 kHz without proper resampling, the model can behave poorly. • How it helps you: Using the correct sample rate ensures the VAD can detect speech reliably and the STT receives a compatible signal — both reduce false detections and improve transcription quality. noise_cancellation=noise_cancellation.BVCTelephony() • What it is: A noise-cancellation model tuned for telephony / narrowband audio (typical SIP/8 kHz scenarios). Its goal is to suppress background voices/noise within a single participant’s audio stream. • What it does and does not do: ◦ It reduces background noise that appears within the same microphone/track (e.g., room noise, far-off voices) and can improve STT accuracy for that track. ◦ It cannot prevent the agent from receiving separate audio tracks from other participants (i.e., it doesn’t “route away” other speakers). • How it helps you: For SIP/narrowband call audio it removes much ambient noise and some overlapping speech in that participant’s stream. If your microphone is capturing multiple people co-located near the mic, performance will be limited — even BVCTelephony has practical limits. If your local mic is full-bandwidth (device mic), using

BVC()

may perform better than

BVCTelephony()

. room_output_options=RoomOutputOptions(transcription_enabled=True) • What it is: Enables LiveKit to produce live transcription output (text) tied to the room’s audio. Transcriptions are published to the room’s text stream (e.g.,

lk.transcription

). • Why it matters: When enabled, STT results are emitted and can be used by your agent logic and logs. This is how you see what the engine detected from audio in real time. • How it helps you: It gives you visibility into what’s being transcribed (so you can confirm whether background voices are being transcribed and tune VAD/filters accordingly). If background speech is still appearing in these transcriptions, it indicates the audio pipeline is feeding that speech to STT (either because the mic captured it, or the agent is subscribed to an extra track). Putting them together — how to use these intentionally • Match

audio_sample_rate

to the actual audio you receive or resample accordingly (8 kHz SIP vs. 16 kHz device mic). Mismatches can harm VAD and STT. • Use

BVCTelephony()

when your input is genuinely narrowband SIP (8 kHz). Use

BVC()

for device/local full-bandwidth audio. Don’t double-apply noise cancellation (no trunk-side Krisp + agent-side BVC). • Keep

transcription_enabled=True

while debugging so you can see exactly what the STT is picking up; use those transcripts to decide whether to change VAD thresholds, mic hardware, or which noise filter to use. Practical checklist to reduce background human voices being transcribed • Confirm the actual incoming sample rate (8 kHz vs 16 kHz) and set or resample to match. • If the mic is local and captures multiple co-located people, try `BVC()`; if the audio is server-side SIP/narrowband, use

BVCTelephony()

. • Disable other noise-processing layers (trunk-side Krisp, client-side filters) while testing. • Increase VAD/interruption thresholds (activation threshold, min_speech_duration, min_silence_duration, min_interruption_duration) so short background speech is ignored. • Use a close-talk headset or hardware beamforming if possible — physical isolation often gives the biggest improvement. If you want, paste the exact RoomOptions block you currently pass in (the few lines where you set sample rate, noise_cancellation, and room_output_options). I’ll recommend the minimal change (switch to BVC vs BVCTelephony, adjust sample rate, or turn on transcription) to test next. Sources: Silero VAD plugin | LiveKit Documentation | Speechify TTS plugin guide | LiveKit Documentation | Live Conversations with AI using ChatGPT and WebRTC | Rime TTS plugin guide | LiveKit Documentation | Troubleshooting STT not picking up utterances | LiveKit