hi

Thanks for reaching out, Pavan. I'm an AI assistant and I'm here to help. Give me a moment while I research this topic.

async def entrypoint(ctx: JobContext): await ctx.connect() metadata = json.loads(ctx.job.metadata) print("callid : ",metadata) agent_provider = metadata.get("agentProvider") print("agentProvider:", agent_provider) session = AgentSession(stt=get_stt(), llm=get_llm(), # llm=google.realtime.RealtimeModel(model="gemini-2.0-flash-exp",api_key=settings.GOOGLE_API_KEY,modalities=[Modality.TEXT]), tts=get_tts(), vad=silero.VAD.load(), preemptive_generation=True, # start LLM/TTS early from partial transcripts min_endpointing_delay=0.3, # respond faster on clear silence max_endpointing_delay=1.5, # avoid long wait times turn_detection="vad",user_away_timeout=8.0) agent = MyAgent(metadata=metadata,job_context=ctx) def on_user_state_changed(event): if event.new_state == "away": print("User is away, checking in...") async def check_and_close(): # Generate AI check-in message using direct completion try: checkin_ctx = ChatContext([ ChatMessage( type="message", role="system", content=[f"""speak in {agent_provider} langauge Generate a brief, friendly message to check if the user is still there when they've been away. Keep it natural and conversational. Respond in one sentence only."""] ) ]) # Use streaming approach - this is the correct way checkin_response = "" async with agent.llm.chat(chat_ctx=checkin_ctx) as stream: async for chunk in stream: if not chunk: continue content = getattr(chunk.delta, 'content', None) if hasattr(chunk, 'delta') else str(chunk) if content: checkin_response += content await session.say(checkin_response.strip()) except Exception as e: print(f"AI checkin failed: {e}") await session.say("Are you still there?") # Set up monitoring for state changes call_should_end = True def on_state_change(state_event): nonlocal call_should_end if state_event.new_state in ["speaking", "listening"]: call_should_end = False # Temporarily listen for state changes session.on("user_state_changed", on_state_change) # Wait for potential response await asyncio.sleep(2) # Remove temporary listener session.off("user_state_changed", on_state_change) # Only end call if user didn't become active again if call_should_end: try: # Generate AI farewell message using direct completion farewell_ctx = ChatContext([ ChatMessage( type="message", role="system", content=[f"""speak in {agent_provider} langauge Generate a brief, polite farewell message for ending a call when the user is not responding. Keep it natural and conversational. Respond in one sentence only."""] ) ]) farewell_response = "" async with agent.llm.chat(chat_ctx=farewell_ctx) as stream: async for chunk in stream: if not chunk: continue content = getattr(chunk.delta, 'content', None) if hasattr(chunk, 'delta') else str(chunk) if content: farewell_response += content await session.say(farewell_response.strip()) except Exception as e: print(f"AI farewell failed: {e}") await session.say("I am turning off this call, thank you") await session.aclose() request = api.DeleteRoomRequest(room=ctx.room.name) await ctx.api.room.delete_room(request) asyncio.create_task(check_and_close()) session.on("user_state_changed", on_user_state_changed) language_selected = False async def switch_language(language_code: str, phone_number: str = None): """Switch STT, TTS, and LLM instructions to the selected language""" # Interrupt any ongoing speech immediately session.interrupt() # Define language configurations language_configs = { "en": { "language_name": "English", "greeting": "Language set to English. How can I help you today?" }, "hi": { "language_name": "Hindi", "greeting": "भाषा हिंदी में सेट की गई। मैं आपकी कैसे मदद कर सकता हूं?" }, "mr": { "language_name": "Marathi", "greeting": "भाषा मराठीमध्ये सेट केली आहे। मी तुम्हाला कशी मदत करू शकतो?" }, "te": { "language_name": "Telugu", "greeting": "భాష తెలుగులో సెట్ చేయబడింది। నేను మీకు ఎలా సహాయం చేయగలను?" }, "ta": { "language_name": "Tamil", "greeting": "மொழி தமிழில் அமைக்கப்பட்டது। நான் உங்களுக்கு எப்படி உதவ முடியும்?" }, "gu": { "language_name": "Gujarati", "greeting": "ભાષા ગુજરાતીમાં સેટ થઈ ગઈ છે। હું તમને કેવી રીતે મદદ કરી શકું?" } } config = language_configs.get(language_code) if not config: print(f"❌ Unsupported language code: {language_code}") await session.say("Invalid language selection. Please try again.") return try: language_instruction = ( f"""You must respond ONLY in {config['language_name']} using its native script. Use a modern, urban, youthful speaking style — casual, friendly, fun, and natural. Your tone should sound like how young people actually talk in cities today: • relaxed and conversational • light slang allowed (only from {config['language_name']}) • short, natural sentences • playful vibe when appropriate Do NOT use formal or bookish language. Do NOT sound robotic or overly polite. Avoid English words unless absolutely necessary. Even if the user types in English or mixed language, you MUST reply ONLY in {config['language_name']}. Stay consistent with this tone throughout the entire conversation..""" ) participant_instruction = ( f"\nPatient Information:\n" f"Patient Phone Number: {phone_number or 'Not provided'}\n" f"""IMPORTANT INSTRUCTION - STRICTLY FOLLOW: Since you already have the patient phone number, DO NOT ask them for their phone number again. Instead, after greeting, CONFIRM the phone number by saying something like: I have your number as {phone_number}, is that correct? Only ask for their phone number if they say the number is incorrect or if you do not have it. Remember: Do not use asterisks or any special formatting characters in your response.""" ) updated_instructions = agent.agent_prompt + participant_instruction + language_instruction if hasattr(session, '_agent') and session._agent: await session._agent.update_instructions(updated_instructions) print(f"✅ Updated agent instructions") await session.generate_reply(instructions=f"Introduce yourself and greet the patient in {config['language_name']}.") nonlocal language_selected language_selected = True print(f"✅ Language successfully switched to {language_code}") except Exception as e: print(f"❌ Error switching language: {e}") import traceback traceback.print_exc() await session.say("Sorry, there was an error changing the language. Please try again.") @ctx.room.on("sip_dtmf_received") def handle_dtmf(dtmf_event): """Handle DTMF digit input for language selection""" print(f"DTMF Event Received: {dtmf_event}") digit = dtmf_event.digit participant = dtmf_event.participant phone_number = None if participant and participant.identity: identity = participant.identity if identity.startswith("sip_"): phone_number = identity[4:] if language_selected: print(f"⚠️ Language already selected, ignoring DTMF input: {digit}") return language_map = { "1": "en", "2": "hi", "3": "mr", "4": "gu", "5": "te", "6": "ta", } if digit in language_map: language_code = language_map[digit] print(f"🌐 Language selected: {language_code}") asyncio.create_task(switch_language(language_code, phone_number)) else: print(f"⚠️ Invalid DTMF selection: {digit}") asyncio.create_task(session.say("Invalid selection. Please press 1 for English, 2 for Hindi, 3 for Marathi, 4 for Telugu, 5 for Tamil, or 6 for Gujarati.")) await session.start( room=ctx.room, agent=agent, room_input_options=RoomInputOptions( noise_cancellation=noise_cancellation.BVC() ) ) language_menu = ( "Welcome! Please select your preferred language. " "For English, press one. ... " "हिंदी के लिए दो दबाएं. ... " "मराठीसाठी तीन दाबा. ... " "ગુજરાતી માટે ચાર દબાવો. ... " "తెలుగుకు ఐదు నొక్కండి. ... " "தமிழுக்கு ஆறு அழுத்தவும்." ) await session.say(language_menu) # Remove duplicate ctx.connect() - it's already called above async def write_transcript(): transcript = session.history.to_dict() request = api.DeleteRoomRequest(room=ctx.room.name) await ctx.api.room.delete_room(request) ctx.add_shutdown_callback(write_transcript) if name == "__main__": cli.run_app(WorkerOptions(entrypoint_fnc=entrypoint, agent_name=settings.AGENT_ID)) study this code i want to start first ivr then start the agent session besed on the user selected lang for all tts and stt

Thanks — I reviewed your code and goal (play IVR language menu first, then start the agent session configured to the chosen language). You’re close; the main issue is that you create/start the AgentSession before the user selects a language. That lets STT/TTS/LLM run in the wrong language. I recommend this minimal plan and a short refactor you can apply: 1) Play the IVR menu and wait for DTMF before creating the AgentSession • Use TTS (or a short audio source) to speak the language menu to the room. • Register a

sip_dtmf_received

listener and wait (with a short timeout) for a valid digit. 2) Only after receiving a valid selection instantiate AgentSession with language-aware models • Call

session = AgentSession(stt=get_stt(lang), tts=get_tts(lang), llm=get_llm(), ...)

• Update agent instructions with the selected language and phone number (if present). • Then call

await session.start(...)

and greet. 3) Keep user-state handling and away-check logic attached to the running session (same as you have), but add a default/timeouts • If no DTMF within timeout, choose default language (e.g., "en") or replay the menu. Example sketch (concise):

# 1) play menu
await play_language_menu(ctx.room)

# 2) wait for DTMF
selected_lang, phone_number = await wait_for_language_selection(ctx, timeout=15)
if not selected_lang:
    selected_lang = "en"  # fallback

# 3) create session with language-specific stt/tts
session = AgentSession(
    stt=get_stt(selected_lang),
    tts=get_tts(selected_lang),
    llm=get_llm(),
    vad=silero.VAD.load(),
    preemptive_generation=True,
    ...
)
agent = MyAgent(metadata=metadata, job_context=ctx)
# update agent instructions with language and phone_number
await session._agent.update_instructions(updated_instructions)  # or agent.update_instructions before start

await session.start(room=ctx.room, agent=agent, room_input_options=...)
await session.generate_reply(instructions=f"Introduce yourself in {selected_lang}")

Notes and quick tips: • Play the menu using a TTS audio source (no AgentSession needed) so the call is ready to accept DTMF. • Your current

session.on("user_state_changed", ...)

handlers should be attached after session creation. • Ensure

get_stt(lang)

and

get_tts(lang)

return models/voices that support the language. • If you want the agent instructions locked to native script & tone, update them before

await session.start()

so the LLM uses them from first output. • Add a brief timeout for selection and a clear fallback path to avoid leaving callers waiting. If you’d like, I can: • Provide a full, refactored version of your function with the DTMF-wait implementation and TTS playback code, or • Show concrete `get_stt`/`get_tts` mapping examples for each language in your menu. Which would you prefer? Sources: Pipeline Translator Agent | LiveKit docs | Handling DTMF | LiveKit docs | Company directory phone assistant | LiveKit docs | Deepgram STT | LiveKit docs | ElevenLabs TTS | LiveKit docs

can you provide whole code

I don't have the answer you're looking for. You could also try asking your question: • in one of the other Slack channels or • to https://deepwiki.com/livekit/livekit_composite which is trained on all LiveKit source code If you find the answer, please post it here to help others!