To enable the frontend to trigger actions in Agents, you can use the Remote Procedure Calls (RPC) feature in LiveKit. Here's a high-level overview of the process: 1. In your agent implementation, create a function that uses RPC to communicate with the frontend. For example:

@function_tool()
async def get_user_location(context: RunContext, high_accuracy: bool):
    try:
        room = get_job_context().room
        participant_identity = next(iter(room.remote_participants))
        response = await room.local_participant.perform_rpc(
            destination_identity=participant_identity,
            method="getUserLocation",
            payload=json.dumps({"highAccuracy": high_accuracy}),
            response_timeout=10.0 if high_accuracy else 5.0,
        )
        return response
    except Exception:
        raise ToolError("Unable to retrieve user location")

2. On the frontend, register an RPC method handler that will be called when the agent invokes the function. For example:

import { RpcError, RpcInvocationData } from 'livekit-client';

localParticipant.registerRpcMethod(
    'getUserLocation',
    async (data: RpcInvocationData) => {
        try {
            let params = JSON.parse(data.payload);
            const position: GeolocationPosition = await new Promise((resolve, reject) => {
                navigator.geolocation.getCurrentPosition(resolve, reject, {
                    enableHighAccuracy: params.highAccuracy ?? false,
                    timeout: data.responseTimeout,
                });
            });
            return JSON.stringify({
                latitude: position.coords.latitude,
                longitude: position.coords.longitude,
            });
        } catch (error) {
            throw new RpcError(1, "Could not retrieve user location");
        }
    }
);

This setup allows the frontend to handle specific actions or UI updates triggered by the agent. Let me know if you need further assistance! Sources: Tool definition and use | LiveKit Docs