For a React Native PTT implementation with RPC handlers, are there any known issues or best practices we should be aware of? We are planning to implement the following PTT plan but would appreciate thoughts / review / feedback: # Push-to-Talk : Separate Agent Deployments Implementation Plan Date: August 2, 2025 Status: DRAFT Confidence: 95% (Based on LiveKit official documentation) ## Overview This plan details implementing PTT as a completely separate agent deployment, leveraging LiveKit's explicit agent dispatch mechanism for clean routing between VAD and PTT modes. ## Architecture

┌─────────────────┐     ┌──────────────────┐     ┌─────────────────────┐
│   Mobile App    │     │ LiveKit Server   │     │   Agent Workers     │
├─────────────────┤     ├──────────────────┤     ├─────────────────────┤
│ Mode Selection  │────:arrow_forward:│ Job Dispatch     │────:arrow_forward:│ cora-voice-agent    │
│ - Live Voice    │     │ by agent_name    │     │ (VAD mode)          │
│ - Push to Talk  │     │                  │     ├─────────────────────┤
└─────────────────┘     └──────────────────┘     │ cora-ptt-agent      │
                                                  │ (PTT mode)          │
                                                  └─────────────────────┘

## Implementation Steps ### 1. Create Separate PTT Agent (

agent-ptt.py

)

python
_import_ asyncio
_import_ os
_import_ logging
_from_ dotenv _import_ load_dotenv
_from_ livekit _import_ agents
_from_ livekit.agents _import_ Agent, AgentSession, JobContext, WorkerOptions, cli

load_dotenv()
logger = logging.getLogger(__name__)

class CoraPTTAssistant(Agent):
    """PTT-specific assistant with manual turn detection"""
    def __init__(_self_, _chat_ctx_=None):
        super().__init__(
            _chat_ctx_=chat_ctx _or_ llm.ChatContext(),
            _llm_=anthropic.LLM(_model_="claude-sonnet-4-20250514"),
            _instructions_="""You are Cora, a supportive AI companion in push-to-talk mode.
            Users will press and hold a button to speak to you.
            Keep responses concise and wait for the user to finish their complete thought
            before responding."""
        )

async def entrypoint(_ctx_: JobContext):
    """PTT agent entrypoint - ONLY handles PTT mode"""
    <http://logger.info|logger.info>(f"PTT Agent starting for room {ctx.room.name}")

    _# Create PTT-only session_
    session = AgentSession(
        _stt_=deepgram.STT(
            _model_="nova-3",
            _interim_results_=False,
            _endpointing_ms_=1000,  _# Shorter for PTT_
            _punctuate_=True,
            _smart_format_=True
        ),
        _tts_=deepgram.TTS(_model_="aura-2-cora-en"),
        _vad_=None,  _# No VAD in PTT mode_
        _turn_detection_="manual",  _# CRITICAL: Manual mode for PTT_
        _# No interruption settings needed for PTT_
    )

    _# Register PTT RPC handlers_
    @ctx.ai_callable()
    async def start_turn(_participant_id_: str):
        """Called when user presses PTT button"""
        <http://logger.info|logger.info>(f"PTT: Starting turn for participant {participant_id}")
        _await_ ctx.room.local_participant.set_microphone_enabled(True)

    @ctx.ai_callable()
    async def end_turn():
        """Called when user releases PTT button"""
        <http://logger.info|logger.info>("PTT: Ending turn, processing speech")
        _await_ ctx.room.local_participant.set_microphone_enabled(False)
        _await_ session.flush()  _# Process any pending audio_

    @ctx.ai_callable()
    async def cancel_turn():
        """Called if user cancels PTT (e.g., swipe away)"""
        <http://logger.info|logger.info>("PTT: Cancelling turn")
        _await_ ctx.room.local_participant.set_microphone_enabled(False)
        session.clear_user_audio()

    _# Create and start PTT assistant_
    assistant = CoraPTTAssistant()
    _await_ session.start(_agent_=assistant, _room_=ctx.room)

_if_ __name__ == "__main__":
    cli.run_app(
        WorkerOptions(
            _entrypoint_fnc_=entrypoint,
            _api_key_=os.getenv("LIVEKIT_API_KEY"),
            _api_secret_=os.getenv("LIVEKIT_API_SECRET"),
            _ws_url_=os.getenv("LIVEKIT_WS_URL"),
            _# CRITICAL: Named agent for explicit dispatch_
            _agent_name_="cora-ptt-agent",
        )
    )

### 2. Update Main Agent to Use Named Dispatch

python
_# In agent.py, add agent_name for explicit dispatch_
_if_ __name__ == "__main__":
    cli.run_app(
        WorkerOptions(
            _entrypoint_fnc_=entrypoint,
            _api_key_=os.getenv("LIVEKIT_API_KEY"),
            _api_secret_=os.getenv("LIVEKIT_API_SECRET"),
            _ws_url_=os.getenv("LIVEKIT_WS_URL"),
            _# CRITICAL: Named agent for VAD mode_
            _agent_name_="cora-voice-agent",
        )
    )

### 3. Update Token Generator Bot for Agent Routing

typescript
_// In livekit-token-generator.ts_
export async function handler(_medplum_: MedplumClient, _event_: BotEvent): Promise<any> {
  const { roomName, participantName, metadata } = event.input;

  _// Determine which agent to dispatch_
  const mode = metadata?.mode || 'voice';
  const agentName = mode === 'push-to-talk' ? 'cora-ptt-agent' : 'cora-voice-agent';

  _// Create token with agent dispatch configuration_
  const at = new AccessToken(API_KEY, API_SECRET, {
    identity: participantName,
    metadata: JSON.stringify(metadata),
  });

  at.addGrant({
    roomJoin: true,
    room: roomName,
    canPublish: true,
    canSubscribe: true,
    _// Add agent dispatch configuration_
    roomConfig: {
      agents: [{
        agentName: agentName,
        metadata: JSON.stringify({ mode, ...metadata })
      }]
    }
  });

  return { token: at.toJwt() };
}

### 4. Client-Side Updates

typescript
_// In usePushToTalkVoice.ts_
const generateToken = async () => {
  const token = await executeBot('livekit-token-generator', {
    roomName: `ptt-${threadId}-${Date.now()}`,
    participantName: userName,
    metadata: {
      mode: 'push-to-talk',  _// This triggers PTT agent dispatch_
      threadId,
      patientId,
    }
  });
  return token;
};

_// In useLiveKitVoice.ts_  
const generateToken = async () => {
  const token = await executeBot('livekit-token-generator', {
    roomName: `voice-${threadId}`,
    participantName: userName,
    metadata: {
      mode: 'voice',  _// This triggers VAD agent dispatch_
      threadId,
      patientId,
    }
  });
  return token;
};

### 5. Deployment Configuration #### Docker Setup

dockerfile
_# Dockerfile.ptt_
FROM python:3.11-slim
WORKDIR /app
COPY requirements.txt .
RUN pip install -r requirements.txt
COPY agent-ptt.py .
COPY medplum_client.py .
CMD ["python", "agent-ptt.py"]

#### ECS Task Definition

json
{
  "family": "cora-ptt-agent",
  "networkMode": "awsvpc",
  "requiresCompatibilities": ["FARGATE"],
  "cpu": "2048",
  "memory": "4096",
  "containerDefinitions": [{
    "name": "cora-ptt-agent",
    "image": "${ECR_URI}/cora-ptt-agent:latest",
    "environment": [
      {"name": "LIVEKIT_WS_URL", "value": "<wss://your-livekit-url>"},
      {"name": "AGENT_NAME", "value": "cora-ptt-agent"}
    ],
    "secrets": [
      {"name": "LIVEKIT_API_KEY", "valueFrom": "arn:aws:ssm:..."},
      {"name": "LIVEKIT_API_SECRET", "valueFrom": "arn:aws:ssm:..."}
    ]
  }]
}

Thanks for reaching out, Albert. I'm an AI assistant and I'm here to help. Give me a moment while I research this topic.

We're deploying two separate agents with different agent_names ('cora-voice-agent' and 'cora-ptt-agent'). Can you confirm that using roomConfig.agents in the token to specify which agent handles each room will correctly route jobs? Example: roomConfig: { agents: [{ agentName: 'cora-ptt-agent' }] }

Absolutely! When you specify

roomConfig.agents

in the access token with an

agentName

, it will route jobs directly to that agent in LiveKit. This is part of LiveKit’s explicit agent dispatch feature, designed to make things smoother for you. Here’s a quick rundown of how it works: • Token Configuration: You include a

RoomConfiguration

object in the token that lists one or more

RoomAgentDispatch

entries. Each entry has an

agentName

to identify which agent to send jobs to. For example:

roomConfig: {
  agents: [
    {
      agentName: 'cora-ptt-agent',
      metadata: JSON.stringify({ mode: 'push-to-talk' })
    }
  ]
}

• Agent Dispatch Behavior: When someone joins the room with this token, the specified agent(s) will be dispatched right away. Just make sure the agent is registered with the LiveKit server using the same

agentName

. If you have any questions or want me to walk you through it, just let me know—I’m here to help! Sources: Authentication | LiveKit Docs | Dispatching agents | LiveKit Docs | Dispatching agents | LiveKit Docs

# Final PTT Implementation Questions for LiveKit Support Thanks for confirming the agent dispatch approach! A few final implementation details: ## 1. Agent Registration and Job Handling When running multiple agents with different names: - If a room is created WITHOUT

roomConfig.agents

specified, will named agents ignore it? Or do we need to explicitly reject jobs in a

request_fnc

? - Is there a recommended pattern for health checks across multiple agent deployments? ## 2. RPC Method Timing for PTT For

turn_detection="manual"

with RPC handlers:

python
@ctx.ai_callable()
async def start_turn(_participant_id_: str):
    _await_ ctx.room.local_participant.set_microphone_enabled(True)

- Should we use

ctx.room.local_participant.set_microphone_enabled()

or direct participant reference? - Is there a delay between RPC registration and availability to clients we should account for? ## 3. PTT Audio Handling For push-to-talk specific concerns: - With

turn_detection="manual"

, what happens to audio received between

end_turn()

and the next

start_turn()

? Is it automatically discarded? - Should we call

session.flush()

in

end_turn()

to ensure all audio is processed? - Is there a recommended timeout for uncommitted audio in PTT mode? ## 4. Client Disconnect Edge Cases - If a React Native client disconnects while holding the PTT button (mid-turn), will the agent receive any notification to clean up the pending audio? - Should we implement a dead-man's switch timeout for PTT turns? ## 5. Performance and Scaling Since we're running two separate agents: - Is there overhead in having multiple agents connect to the same LiveKit server? - Can both agents share the same API key/secret, or should they have separate credentials? - Any recommendations for resource allocation (CPU/memory) specifically for PTT agents which have sporadic usage? ## Context We're implementing PTT as a separate agent (Option 2) based on your confirmation. Just want to handle edge cases properly! Thanks!

Here are some answers to your detailed questions about implementing PTT with LiveKit: • Agent Registration and Job Handling: Named agents will simply ignore rooms where they’re not explicitly listed in

roomConfig.agents

. You don’t need to explicitly reject jobs in a

request_fnc

unless you want more control. For health checks, it’s a good idea to use periodic signals to keep an eye on agent availability and log their status centrally. • RPC Method Timing for PTT: Using

ctx.room.local_participant.set_microphone_enabled()

is the way to go for consistency and to avoid referencing participants directly. RPC methods become available right after registration, so there’s no need to worry about extra delay handling. • PTT Audio Handling: Any audio that comes in between

end_turn()

and the next

start_turn()

gets discarded because the microphone is disabled during that time. It’s a good practice to call

session.flush()

inside

end_turn()

to make sure all audio is processed. Starting with a 10-second timeout for uncommitted audio in PTT mode usually works well. • Client Disconnect Edge Cases: If a client disconnects mid-turn, the agent won’t automatically get a notification. To handle this, setting up a dead-man’s switch timeout is helpful to clean up any pending audio. • Performance and Scaling: Having multiple agents connect to the same LiveKit server doesn’t add much overhead, as long as resources are managed properly. Both agents can use the same API key/secret, but just be sure to keep access control and monitoring in place. For PTT agents that are used sporadically, it’s best to prioritize CPU over memory for audio processing. If you want me to dive deeper into any part or need a hand with something specific, just let me know—I’m here to help!

# LiveKit Support Question - Agent Dispatch Without RoomConfiguration ## Background We're implementing a voice application with two separate LiveKit agents: 1.

cora-voice-agent

- Voice Activity Detection (VAD) mode for live conversations 2.

cora-ptt-agent

- Push-to-Talk mode with manual turn detection ## Current Implementation ### Token Generator (Node.js/TypeScript)

typescript
const token = new AccessToken(apiKey, apiSecret, {
  identity: patientId,
  metadata: JSON.stringify(participantMetadata),
});

const roomName = `${threadId}-${Date.now()}`;

token.addGrant({
  room: roomName,
  roomJoin: true,
  roomCreate: true,
  canPublish: true,
  canSubscribe: true,
  canPublishData: true,
});

_// We removed RoomConfiguration due to import issues_
_// Previously had:_
_// token.roomConfig = new RoomConfiguration({_
_//   agents: [new RoomAgentDispatch({ agentName: agentName })]_
_// });_

return { token: jwt, roomName, agentName, mode };

### Agent Registration (Python)

python
_# cora-voice-agent_
cli.run_app(
    WorkerOptions(
        _entrypoint_fnc_=entrypoint,
        _api_key_=os.getenv("LIVEKIT_API_KEY"),
        _api_secret_=os.getenv("LIVEKIT_API_SECRET"),
        _ws_url_=os.getenv("LIVEKIT_WS_URL"),
        _agent_name_="cora-voice-agent",
    )
)

_# cora-ptt-agent (identical except agent_name)_
agent_name="cora-ptt-agent"

## Problem Without RoomConfiguration in the token, how do we ensure: 1. The correct agent joins the room based on the intended mode? 2. Only ONE agent joins each room (not both)? 3. The agents know which rooms to accept? ## Current Behavior - Agents seem to join rooms but show "Final participant count: 0" - Client connects but no agent interaction occurs - No audio responses in either mode ## Questions 1. Without RoomConfiguration, how should we implement agent dispatch? Should we: - Add a job_request handler to filter based on room metadata? - Use a different mechanism for agent selection? - Implement room name patterns that agents can filter on? 2. Is there a recommended pattern for multiple agents serving different purposes? We want to avoid: - Both agents joining the same room - Neither agent joining a room - Race conditions in agent selection 3. What's the proper way to pass the mode/agent selection from client to agent? Currently we pass

mode

in the token request but it's not used for dispatch. 4. Why might we see "Final participant count: 0" in agent logs? This suggests: - Agent joins the room - Client fails to join or immediately disconnects - Room closes due to no participants ## Environment - LiveKit Cloud - Python agents SDK 1.2.2 - React Native client with livekit-client - Token generation via Medplum bot (Node.js) ## Desired Outcome - User selects "Live Voice" → only cora-voice-agent joins - User selects "Push to Talk" → only cora-ptt-agent joins - Reliable agent dispatch without RoomConfiguration imports

Hey there! If you want to implement agent dispatch in LiveKit without using

RoomConfiguration

in the token, you can go with explicit agent dispatch. Here’s a friendly walkthrough to make sure the right agent joins the room: • Set Agent Name in WorkerOptions: When you create your agent worker, just set the

agent_name

field in

WorkerOptions

. This turns off automatic dispatch and makes sure the agent only joins rooms that are assigned to it directly. • Use AgentDispatchClient: You can use the

AgentDispatchClient

to explicitly send agents to rooms. For example:

const agentDispatchClient = new AgentDispatchClient(LIVEKIT_URL);
const dispatch = await agentDispatchClient.createDispatch(
  roomName,
  agentName,
  { metadata: JSON.stringify({ mode: 'push-to-talk' }) }
);

• Filter Rooms Using Metadata: Pass along metadata when dispatching and have your agent logic filter rooms accordingly. For example:

async function entrypoint(ctx: JobContext) {
  const metadata = JSON.parse(ctx.job.metadata);
  if (metadata.mode === 'push-to-talk') {
    // Process the job
  } else {
    // Ignore the job
  }
}

This way, you get: • Only the right agent joining each room based on the mode. • Agents filtering rooms using metadata, which helps avoid mix-ups or race conditions. • Full control over agent dispatch without needing to rely on

RoomConfiguration

. If you have any questions or want a hand setting this up, just let me know—I’m here to help! Sources: packages/livekit-server-sdk/src/AgentDispatchClient.ts | examples/agent_dispatch.py | Dispatching agents | LiveKit Docs

AttributeError when registering RPC handlers for PTT - correct pattern for ai_callable decorator? Question: We're implementing push-to-talk with manual turn detection using separate agents. Our PTT agent crashes with: AttributeError: 'JobContext' object has no attribute 'ai_callable' We've seen different patterns in docs and examples: Pattern A (causing the error): async def entrypoint(ctx: JobContext): session = AgentSession( turn_detection="manual", # ... other config ) @ctx.ai_callable() # <-- This crashes async def start_turn(participant_id: str = None): await ctx.room.local_participant.set_ microphone_enabled(True) return {"status": "turn_started"} Pattern B (what we think might work): async def entrypoint(ctx: JobContext): assistant = MyAssistant() # Inherits from Agent session = AgentSession(turn_detection="manual") @assistant.ai_callable() # <-- Register on assistant? async def start_turn(participant_id: str = None): # ... Questions: 1. What's the correct object to register @ai_callable() decorators on for RPC handlers? 2. For manual turn detection with PTT, should we be using a different pattern? 3. Is there a working example of PTT with RPC handlers in Python SDK 1.2.2? Our Setup: - livekit-agents==1.2.2 - Python 3.11 - Separate agents: cora-voice-agent (VAD) and cora-ptt-agent (manual) - Using AgentDispatchClient for routing The agent connects to rooms successfully but crashes when trying to register the RPC handlers. Any guidance on the correct pattern would be greatly appreciated!

Here’s a friendly guide on how to properly register RPC handlers and set up manual turn detection for Push-to-Talk (PTT) using the LiveKit Python SDK 1.2.2: • Correct Object for @ai_callable: You’ll want to use

@ctx.room.local_participant.register_rpc_method()

to register your RPC methods. Here’s a quick example:

@ctx.room.local_participant.register_rpc_method("start_turn")
async def start_turn(data: rtc.RpcInvocationData):
    session.interrupt()
    session.clear_user_turn()
    session.input.set_audio_enabled(True)

• Manual Turn Detection: To turn off automatic turn detection, just set

turn_detection="manual"

when you create your

AgentSession

. Then, you can handle turns yourself by implementing RPC methods like

start_turn

and

end_turn

.

session = AgentSession(turn_detection="manual")

@ctx.room.local_participant.register_rpc_method("end_turn")
async def end_turn(data: rtc.RpcInvocationData):
    session.input.set_audio_enabled(False)
    session.commit_user_turn()

• Example Code: If you want to see it all in action, there’s a full example of PTT with manual turn detection over at the LiveKit Agents GitHub repository. If you have any questions or need a hand with your setup, just let me know—I’m here to help! Sources: Turn detection and interruptions | LiveKit Docs | Turn detection and interruptions | LiveKit Docs