I am trying to do this: Join room with audio subsc...
# server-api
i
I am trying to do this: Join room with audio subscription enabled. Capture audio tracks from interviewer (RemoteParticipant). Transcribe audio in real-time using OpenAI Whisper. Generate answers using OpenAI's GPT model. Synthesize voice using OpenAI TTS API. Play back the generated voice in the room using
AudioSource
. logs- INFOinterview botSuccessfully connected to room INFOinterview botConnected to room: interview_67dba28d5c177168f3dc56e2_1745033914 INFOinterview botLocal participant identity: interview_67dba28d5c177168f3dc56e2_1745033914 INFOinterview botNumber of remote participants: 0 INFOinterview botAttempting to publish audio track (attempt 1/3) INFOinterview botTrack ID: TR_unknown INFOinterview botTrack kind: 1 INFOinterview botTrack name: candidate_audio WARNINGinterview botFailed to publish track, retrying... Error: room error engine: internal error: track publication timed out, no response received from the server
Copy code
import asyncio
import logging
import numpy as np
import soundfile as sf
from io import BytesIO
from livekit import rtc
import openai
import aiohttp

# Setup credentials
TOKEN = ""
URL = ""
openai.api_key = ""

async def transcribe_audio(audio_bytes):
    audio_file = BytesIO(audio_bytes)
    audio_file.name = "speech.wav"

    return await openai.Audio.atranscribe("whisper-1", file=audio_file)

async def generate_response(prompt):
    response = await openai.ChatCompletion.acreate(
        model="gpt-4",
        messages=[
            {"role": "system", "content": "You are a professional candidate in a technical interview."},
            {"role": "user", "content": prompt}
        ],
    )
    return response.choices[0].message.content.strip()

async def synthesize_audio(text):
    response = await openai.audio.atranslate.create(
        model="tts-1",
        voice="alloy",
        input=text,
        response_format="pcm"
    )
    return await response.read()

async def main():
    logging.basicConfig(level=<http://logging.INFO|logging.INFO>)
    logger = logging.getLogger("interview-bot")

    # Initialize variables
    room = None
    local_track = None
    audio_source = None

    try:
        # Initialize room and audio components
        room = rtc.Room()
        
        # Create audio source with proper configuration
        audio_source = rtc.AudioSource(
            sample_rate=48000,
            num_channels=1
        )
        
        # Create local audio track
        local_track = rtc.LocalAudioTrack.create_audio_track(
            "candidate_audio",
            audio_source
        )
        
        if not local_track:
            raise Exception("Failed to create local audio track")

        # Connect to room with timeout
        <http://logger.info|logger.info>("Connecting to room...")
        try:
            await asyncio.wait_for(room.connect(URL, TOKEN), timeout=10)
            <http://logger.info|logger.info>("Successfully connected to room")
            
            # Log room and participant details
            <http://logger.info|logger.info>(f"Connected to room: {room.name}")
            <http://logger.info|logger.info>(f"Local participant identity: {room.local_participant.identity}")
            <http://logger.info|logger.info>(f"Number of remote participants: {len(room.remote_participants)}")
            
            # Wait for room to be fully connected
            await asyncio.sleep(3)
            
        except asyncio.TimeoutError:
            logger.error("Connection to room timed out")
            raise
        except Exception as e:
            logger.error(f"Failed to connect to room: {str(e)}")
            raise

        # Publish track with retry logic
        max_retries = 3
        for attempt in range(max_retries):
            try:
                <http://logger.info|logger.info>(f"Attempting to publish audio track (attempt {attempt + 1}/{max_retries})")
                
                # Log track details before publishing
                <http://logger.info|logger.info>(f"Track ID: {local_track.sid}")
                <http://logger.info|logger.info>(f"Track kind: {local_track.kind}")
                <http://logger.info|logger.info>(f"Track name: {local_track.name}")
                
                # Verify room is still connected
                if not room.isconnected:
                    raise Exception("Room is not connected")
                
                # Add a small delay before publishing
                await asyncio.sleep(1)
                
                # Attempt to publish
                await room.local_participant.publish_track(local_track)
                <http://logger.info|logger.info>("Successfully published audio track")
                
                # Verify publication
                publication = room.local_participant.get_track_publication(local_track.sid)
                if not publication:
                    raise Exception("Failed to verify track publication")
                
                <http://logger.info|logger.info>(f"Track published successfully with ID: {publication.sid}")
                break
                
            except Exception as e:
                if attempt == max_retries - 1:
                    logger.error(f"Failed to publish track after {max_retries} attempts: {str(e)}")
                    raise
                logger.warning(f"Failed to publish track, retrying... Error: {str(e)}")
                await asyncio.sleep(2)

        @room.on("track_subscribed")
        def on_track_subscribed(track, publication, participant):
            if track.kind == rtc.TrackKind.KIND_AUDIO:
                <http://logger.info|logger.info>(f":headphones: Subscribed to audio from {participant.identity}")
                asyncio.create_task(handle_audio_stream(track, audio_source))

        async def handle_audio_stream(track, audio_source):
            audio_stream = rtc.AudioStream(track)
            buffer = BytesIO()
            sample_rate = 48000
            chunk_size = sample_rate * 5  # 5 seconds

            async for frame in audio_stream:
                pcm_data = frame.data.tobytes()
                buffer.write(pcm_data)

                if buffer.tell() >= chunk_size * 2:
                    audio_bytes = buffer.getvalue()
                    buffer.seek(0)
                    buffer.truncate(0)

                    try:
                        <http://logger.info|logger.info>(":studio_microphone: Transcribing...")
                        transcript_data = await transcribe_audio(audio_bytes)
                        transcript = transcript_data["text"]
                        <http://logger.info|logger.info>(f":speech_balloon: Question: {transcript}")

                        response_text = await generate_response(transcript)
                        <http://logger.info|logger.info>(f":robot_face: Answer: {response_text}")

                        <http://logger.info|logger.info>(":loud_sound: Synthesizing speech...")
                        pcm_response = await synthesize_audio(response_text)

                        # Play it back into the room
                        response_np = np.frombuffer(pcm_response, dtype=np.int16)
                        audio_source.capture_frame(response_np)

                    except Exception as e:
                        logger.exception(":x: Error in pipeline: %s", e)

        <http://logger.info|logger.info>(":robot_face: Candidate bot joined the room.")
        while True:
            await asyncio.sleep(1)

    except Exception as e:
        logger.error(f"Fatal error: {str(e)}")
        raise
    finally:
        # Cleanup
        try:
            if room:
                await room.disconnect()
            if local_track and room and room.local_participant:
                # Unpublish the track if it was published
                try:
                    publication = room.local_participant.get_track_publication(local_track.sid)
                    if publication:
                        await room.local_participant.unpublish_track(local_track)
                except Exception as e:
                    logger.warning(f"Error during track cleanup: {str(e)}")
        except Exception as e:
            logger.error(f"Error during cleanup: {str(e)}")

if __name__ == "__main__":
    asyncio.run(main())
r
Have you looked at this example? https://github.com/livekit-examples/python-agents-examples/blob/main/basics/playing_audio.py I am curious why you are not using the Agents library. Would that be a lot easier?
i
Ok will look into it
Thanks