how to display stt and tts transcriptions on front...
# ask-ais
stocky-tailor-95498
08/13/2025, 9:02 AMhow to display stt and tts transcriptions on frontend
t
tall-airline-16756
08/13/2025, 9:02 AMThanks for reaching out, Nakshatra. I'm an AI assistant and I'm here to help. Give me a moment while I research this topic.
tall-airline-16756
08/13/2025, 9:03 AMHey there! If you want to show STT and TTS transcriptions on the frontend using LiveKit, here’s a friendly step-by-step guide to help you out:
• Frontend Integration:
◦ First, use the LiveKit SDK to listen for
RoomEvent.TranscriptionReceivedfinalTranscriptionSegments
stocky-tailor-95498
08/13/2025, 9:05 AMand this goes in page.tsx?
t
tall-airline-16756
08/13/2025, 9:06 AMAbsolutely! You can pop the provided React component into your
page.tsxuseMaybeRoomContextlivekit-client@livekit/components-reacts
stocky-tailor-95498
08/13/2025, 9:07 AMi have an existing script, need to add transcriptions to it:
'use client';
import { LiveKitRoom, AudioConference, useRoomContext, StartAudio } from '@livekit/components-react';
import { useCallback, useEffect, useState } from 'react';
// Generates a random user ID for storage only (not sent to backend)
function generateRandomUserId() {
return `user-${Math.floor(Math.random() * 100000)}`;
}
export default function AudioExamplePage() {
const [mounted, setMounted] = useState(false);
const [userIdentity, setUserIdentity] = useState<string>('');
const endpoint = process.env.NEXT_PUBLIC_LK_TOKEN_ENDPOINT!;
const [isStarting, setIsStarting] = useState(false);
const [isInRoom, setIsInRoom] = useState(false);
const [token, setToken] = useState<string | null>(null);
const [serverUrlState, setServerUrlState] = useState<string | null>(null);
const [roomNameState, setRoomNameState] = useState<string | null>(null);
const [error, setError] = useState<string | null>(null);
// Mounting logic and storing random identity locally
useEffect(() => {
setMounted(true);
const stored = localStorage.getItem('lk-user-id');
if (stored) {
setUserIdentity(stored);
} else {
const uid = generateRandomUserId();
localStorage.setItem('lk-user-id', uid);
setUserIdentity(uid);
}
}, []);
// Start session by calling second token generator endpoint
const handleStart = useCallback(async () => {
if (!endpoint) _return_;
setError(null);
setIsStarting(true);
try {
const res = await fetch(endpoint, { method: 'GET', cache: 'no-store' });
if (!res.ok) throw new Error(
Token fetch failed: ${res.status} ${res.statusText}t
tall-airline-16756
08/13/2025, 9:07 AMHey there! If you want to add transcriptions to your existing script, you can either weave the transcription logic right into the
ContentGridimport { RoomEvent, TranscriptionSegment } from 'livekit-client';AudioExamplePageconst [transcriptions, setTranscriptions] = useState<{ [id: string]: TranscriptionSegment }>({});RoomEvent.TranscriptionReceiveduseEffect Copy code
useEffect(() => {
const room = useRoomContext();
if (!room) return;
const handleTranscription = (segments: TranscriptionSegment[]) => {
setTranscriptions(prev => {
const updated = { ...prev };
segments.forEach(segment => {
updated[segment.id] = segment;
});
return updated;
});
};
room.on(RoomEvent.TranscriptionReceived, handleTranscription);
return () => room.off(RoomEvent.TranscriptionReceived, handleTranscription);
}, []);ContentGrid Copy code
<section>
<h2>Live Transcriptions</h2>
<ul>
{Object.values(transcriptions)
.sort((a, b) => a.firstReceivedTime - b.firstReceivedTime)
.map(segment => (
<li key={segment.id}>{segment.text}</li>
))}
</ul>
</section>s
stocky-tailor-95498
08/13/2025, 9:34 AMi did everything as follows:
'use client';
import { LiveKitRoom, AudioConference, useRoomContext, StartAudio } from '@livekit/components-react';
import { useCallback, useEffect, useState } from 'react';
import { RoomEvent, TranscriptionSegment } from 'livekit-client';
// Generates a random user ID for storage only (not sent to backend)
function generateRandomUserId() {
return `user-${Math.floor(Math.random() * 100000)}`;
}
// ✅ New component to listen for transcriptions inside LiveKit context
function TranscriptionListener({ setTranscriptions }: { setTranscriptions: React.Dispatch<React.SetStateAction<{ [id: string]: TranscriptionSegment }>> }) {
const room = useRoomContext();
useEffect(() => {
if (!room) _return_;
const handleTranscription = (segments: TranscriptionSegment[]) => {
setTranscriptions(prev => {
const updated = { ...prev };
segments.forEach(segment => {
updated[segment.id] = segment;
});
return updated;
});
};
room.on(RoomEvent.TranscriptionReceived, handleTranscription);
return () => {
room.off(RoomEvent.TranscriptionReceived, handleTranscription);
};
}, [room, setTranscriptions]);
return null; // This component only listens; no UI
}
export default function AudioExamplePage() {
const [mounted, setMounted] = useState(false);
const [userIdentity, setUserIdentity] = useState<string>('');
const endpoint = process.env.NEXT_PUBLIC_LK_TOKEN_ENDPOINT!;
const [isStarting, setIsStarting] = useState(false);
const [isInRoom, setIsInRoom] = useState(false);
const [token, setToken] = useState<string | null>(null);
const [serverUrlState, setServerUrlState] = useState<string | null>(null);
const [roomNameState, setRoomNameState] = useState<string | null>(null);
const [error, setError] = useState<string | null>(null);
const [transcriptions, setTranscriptions] = useState<{ [id: string]: TranscriptionSegment }>({});
// Mounting logic and storing random identity locally
useEffect(() => {
setMounted(true);
const stored = localStorage.getItem('lk-user-id');
if (stored) {
setUserIdentity(stored);
} else {
const uid = generateRandomUserId();
localStorage.setItem('lk-user-id', uid);
setUserIdentity(uid);
}
}, []);
// Start session by calling second token generator endpoint
const handleStart = useCallback(async () => {
if (!endpoint) _return_;
setError(null);
setIsStarting(true);
try {
const res = await fetch(endpoint, { method: 'GET', cache: 'no-store' });
if (!res.ok) throw new Error(
Token fetch failed: ${res.status} ${res.statusText}t
tall-airline-16756
08/13/2025, 9:34 AMHey there! It seems like you’ve got the transcription logic set up pretty well, but let’s double-check a few things to make sure those transcriptions actually show up:
1. Verify Transcription Events: Make sure the
RoomEvent.TranscriptionReceivedhandleTranscription Copy code
const handleTranscription = (segments: TranscriptionSegment[]) => {
console.log('Transcription segments received:', segments);
setTranscriptions(prev => {
const updated = { ...prev };
segments.forEach(segment => {
updated[segment.id] = segment;
});
return updated;
});
};ContentGridtranscriptionsconsole.log('Current transcriptions:', transcriptions);useRoomContexts
stocky-tailor-95498
08/13/2025, 9:36 AMyes, i havent added the changes to the python backend:
from dotenv import load_dotenv
from livekit import agents
from livekit.agents import AgentSession, Agent, RoomInputOptions
from livekit.plugins import (
openai,
cartesia,
deepgram,
noise_cancellation,
silero,
)
from livekit.plugins.turn_detector.multilingual import MultilingualModel
from dotenv import load_dotenv
load_dotenv(dotenv_path=".env.local")
class Translator(Agent):
def __init__(self) -> None:
super().__init__(instructions= "Translate English speech to Hindi only, say nothing else",)
async def entrypoint(ctx: agents.JobContext):
session = AgentSession(
stt=deepgram.STT(),
llm=openai.LLM(model="gpt-4o-mini"),
tts=cartesia.TTS(
model="sonic-2",
voice="28ca2041-5dda-42df-8123-f58ea9c3da00",
language="hi"
),
vad=silero.VAD.load(),
turn_detection=MultilingualModel(),
use_tts_aligned_transcription=True,
)
await session.start(
room=ctx.room,
agent=Translator(),
room_input_options=RoomInputOptions(
noise_cancellation=noise_cancellation.BVC(),
),
)
await ctx.connect()
await session.generate_reply(
instructions="Greet 'Hello I'm a translator'"
)
if name == "__main__":
agents.cli.run_app(agents.WorkerOptions(entrypoint_fnc=entrypoint))
t
tall-airline-16756
08/13/2025, 9:37 AMHey there! To get transcriptions working on the frontend, you’ll want to make sure your Python backend is set up to forward those transcription events properly. Based on the script you shared, here’s a friendly step-by-step guide:
1. Enable Transcription Forwarding: You’ll need to update your
AgentSessionSTTSegmentsForwarder Copy code
from livekit.agents.transcription import STTSegmentsForwarder
async def entrypoint(ctx: agents.JobContext):
stt_forwarder = STTSegmentsForwarder()
session = AgentSession(
stt=deepgram.STT(),
llm=openai.LLM(model="gpt-4o-mini"),
tts=cartesia.TTS(
model="sonic-2",
voice="28ca2041-5dda-42df-8123-f58ea9c3da00",
language="hi"
),
vad=silero.VAD.load(),
turn_detection=MultilingualModel(),
use_tts_aligned_transcription=True,
)
# Forward transcriptions
session.add_transcription_forwarder(stt_forwarder)
await session.start(
room=ctx.room,
agent=Translator(),
room_input_options=RoomInputOptions(
noise_cancellation=noise_cancellation.BVC(),
),
)
await ctx.connect()
await session.generate_reply(
instructions="Greet 'Hello I'm a translator'"
) Copy code
import logging
logging.basicConfig(level=<http://logging.INFO|logging.INFO>)
async def entrypoint(ctx: agents.JobContext):
stt_forwarder = STTSegmentsForwarder()
session = AgentSession(
stt=deepgram.STT(),
llm=openai.LLM(model="gpt-4o-mini"),
tts=cartesia.TTS(
model="sonic-2",
voice="28ca2041-5dda-42df-8123-f58ea9c3da00",
language="hi"
),
vad=silero.VAD.load(),
turn_detection=MultilingualModel(),
use_tts_aligned_transcription=True,
)
session.add_transcription_forwarder(stt_forwarder)
async def log_transcriptions():
async for event in stt_forwarder.stream():
<http://logging.info|logging.info>("Transcription event: %s", event)
await session.start(
room=ctx.room,
agent=Translator(),
room_input_options=RoomInputOptions(
noise_cancellation=noise_cancellation.BVC(),
),
)
await ctx.connect()
await session.generate_reply(
instructions="Greet 'Hello I'm a translator'"
)
await log_transcriptions()s
stocky-tailor-95498
08/13/2025, 9:41 AMsttsegmentforwarder is from 0.x versions, i am using1.x
t
tall-airline-16756
08/13/2025, 9:41 AMHey there! Just a quick update about LiveKit 1.x — the way transcription forwarding works has gotten a bit simpler. You don’t need to use
STTSegmentsForwarderAgentSessiondeepgram.STT()use_tts_aligned_transcriptionTrueRoomEvent.TranscriptionReceived Copy code
import logging
logging.basicConfig(level=<http://logging.INFO|logging.INFO>)
async def entrypoint(ctx: agents.JobContext):
session = AgentSession(
stt=deepgram.STT(),
llm=openai.LLM(model="gpt-4o-mini"),
tts=cartesia.TTS(
model="sonic-2",
voice="28ca2041-5dda-42df-8123-f58ea9c3da00",
language="hi"
),
vad=silero.VAD.load(),
turn_detection=MultilingualModel(),
use_tts_aligned_transcription=True,
)
async def log_transcriptions():
async for event in session.transcription_stream():
<http://logging.info|logging.info>("Transcription event: %s", event)
await session.start(
room=ctx.room,
agent=Translator(),
room_input_options=RoomInputOptions(
noise_cancellation=noise_cancellation.BVC(),
),
)
await ctx.connect()
await session.generate_reply(
instructions="Greet 'Hello I'm a translator'"
)
await log_transcriptions()s
stocky-tailor-95498
08/13/2025, 9:43 AMAgentSession.__init__() got an unexpected keyword argument 'use_tts_aligned_transcription'
t
tall-airline-16756
08/13/2025, 9:44 AMHey there! It looks like the error is telling us that the
use_tts_aligned_transcriptionAgentSessionuse_tts_aligned_transcriptionAgentSessionAgentSessiondeepgram.STT()s
stocky-tailor-95498
08/13/2025, 9:46 AMAgentSession' object has no attribute 'transcription_stream'
t
tall-airline-16756
08/13/2025, 9:46 AMHey there! It looks like the error is telling us that the
AgentSessiontranscription_streamdeepgram.STT()AgentSessionRoomEvent.TranscriptionReceived Copy code
import logging
logging.basicConfig(level=<http://logging.INFO|logging.INFO>)
async def entrypoint(ctx: agents.JobContext):
session = AgentSession(
stt=deepgram.STT(),
llm=openai.LLM(model="gpt-4o-mini"),
tts=cartesia.TTS(
model="sonic-2",
voice="28ca2041-5dda-42df-8123-f58ea9c3da00",
language="hi"
),
vad=silero.VAD.load(),
turn_detection=MultilingualModel(),
)
await session.start(
room=ctx.room,
agent=Translator(),
room_input_options=RoomInputOptions(
noise_cancellation=noise_cancellation.BVC(),
),
)
await ctx.connect()
<http://logging.info|logging.info>("Session started successfully")