OpenAI Audio - Netra Docs

OpenAI provides speech-to-text (Whisper, GPT-4o Transcribe) and text-to-speech (TTS-1, GPT-4o Mini TTS) capabilities alongside its LLM offerings. Netra helps you trace audio operations, monitor transcription accuracy, and track voice synthesis performance.

Installation

pip install openai netra-sdk

npm install openai netra-sdk

Usage

Initialize Netra before using OpenAI audio APIs:

import os

from netra import Netra

Netra.init(
    app_name="openai-audio-service",
    headers=f"x-api-key={os.environ.get('NETRA_API_KEY')}"
)

import Netra from 'netra-sdk';

await Netra.init({
  appName: 'openai-audio-service',
  headers: `x-api-key=${process.env.NETRA_API_KEY}`
});

Examples

Speech-to-Text with Whisper

Track transcription operations using Netra decorators:

import os
from openai import OpenAI
from netra.decorators import task, workflow

client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))

@task()
def transcribe_audio(audio_path: str) -> str:
    """Transcribe audio using Whisper."""
    with open(audio_path, "rb") as audio_file:
        response = client.audio.transcriptions.create(
            model="whisper-1",
            file=audio_file,
            response_format="text"
        )
    
    return response

@task()
def transcribe_with_gpt4o(audio_path: str) -> str:
    """Transcribe audio using GPT-4o Transcribe for higher accuracy."""
    with open(audio_path, "rb") as audio_file:
        response = client.audio.transcriptions.create(
            model="gpt-4o-transcribe",
            file=audio_file,
            response_format="text"
        )
    
    return response

@workflow()
def process_audio(audio_path: str) -> dict:
    """Process audio with multiple transcription models."""
    whisper_result = transcribe_audio(audio_path)
    gpt4o_result = transcribe_with_gpt4o(audio_path)
    
    return {
        "whisper": whisper_result,
        "gpt4o_transcribe": gpt4o_result
    }

# Usage
result = process_audio("./audio/meeting.mp3")

import OpenAI from 'openai';
import { task, workflow } from 'netra-sdk';
import * as fs from 'fs';

const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });

class TranscriptionService {
  @task()
  async transcribeAudio(audioPath: string): Promise<string> {
    const file = fs.createReadStream(audioPath);
    const response = await openai.audio.transcriptions.create({
      model: 'whisper-1',
      file: file,
      response_format: 'text'
    });

    return response;
  }

  @task()
  async transcribeWithGpt4o(audioPath: string): Promise<string> {
    const file = fs.createReadStream(audioPath);
    const response = await openai.audio.transcriptions.create({
      model: 'gpt-4o-transcribe',
      file: file,
      response_format: 'text'
    });

    return response;
  }

  @workflow()
  async processAudio(audioPath: string): Promise<Record<string, string>> {
    const whisperResult = await this.transcribeAudio(audioPath);
    const gpt4oResult = await this.transcribeWithGpt4o(audioPath);

    return {
      whisper: whisperResult,
      gpt4o_transcribe: gpt4oResult
    };
  }
}

// Usage
const service = new TranscriptionService();
const result = await service.processAudio('./audio/meeting.mp3');

Text-to-Speech

Track TTS generation using Netra decorators:

import os
from pathlib import Path
from openai import OpenAI
from netra.decorators import task, workflow

client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))

@task()
def generate_speech(text: str, voice: str = "alloy") -> bytes:
    """Generate speech using GPT-4o Mini TTS."""
    response = client.audio.speech.create(
        model="gpt-4o-mini-tts",
        voice=voice,
        input=text
    )
    
    return response.content

@task()
def generate_hd_speech(text: str, voice: str = "nova") -> bytes:
    """Generate high-quality speech using TTS-1-HD."""
    response = client.audio.speech.create(
        model="tts-1-hd",
        voice=voice,
        input=text
    )
    
    return response.content

@workflow()
def create_audio_content(texts: list[str], voice: str = "alloy") -> list[bytes]:
    """Generate speech for multiple text segments."""
    audio_segments = []
    
    for text in texts:
        audio = generate_speech(text, voice)
        audio_segments.append(audio)
    
    return audio_segments

# Usage
audio = generate_speech("Hello, welcome to the demo.", "nova")
Path("output.mp3").write_bytes(audio)

import OpenAI from 'openai';
import { task, workflow } from 'netra-sdk';
import * as fs from 'fs';

const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });

class TTSService {
  @task()
  async generateSpeech(text: string, voice: string = 'alloy'): Promise<Buffer> {
    const response = await openai.audio.speech.create({
      model: 'tts-1',
      voice: voice as any,
      input: text
    });

    return Buffer.from(await response.arrayBuffer());
  }

  @task()
  async generateHdSpeech(text: string, voice: string = 'nova'): Promise<Buffer> {
    const response = await openai.audio.speech.create({
      model: 'tts-1-hd',
      voice: voice as any,
      input: text
    });

    return Buffer.from(await response.arrayBuffer());
  }

  @workflow()
  async createAudioContent(texts: string[], voice: string = 'alloy'): Promise<Buffer[]> {
    const audioSegments: Buffer[] = [];

    for (const text of texts) {
      const audio = await this.generateSpeech(text, voice);
      audioSegments.push(audio);
    }

    return audioSegments;
  }
}

// Usage
const ttsService = new TTSService();
const audio = await ttsService.generateSpeech('Hello, welcome to the demo.', 'nova');
fs.writeFileSync('output.mp3', audio);

Manual Span Creation with Action Tracking

For detailed control over tracing:

import os
import time

from openai import OpenAI

from netra import SpanWrapper, ActionModel, UsageModel

client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))

def transcribe_with_tracking(audio_path: str) -> str:
    """Transcribe audio with detailed tracking."""
    span = SpanWrapper("openai-stt")
    span.start()
    
    try:
        start_time = time.time_ns()
        
        with open(audio_path, "rb") as audio_file:
            audio_data = audio_file.read()
        
        audio_size_bytes = len(audio_data)
        span.set_attribute("audio_file", audio_path)
        span.set_attribute("audio_size_bytes", audio_size_bytes)
        span.set_attribute("model", "whisper-1")
        
        with open(audio_path, "rb") as audio_file:
            response = client.audio.transcriptions.create(
                model="whisper-1",
                file=audio_file,
                response_format="text"
            )
        
        end_time = time.time_ns()
        duration_ms = (end_time - start_time) / 1_000_000
        
        action = ActionModel(
            start_time=str(start_time),
            action="API",
            action_type="STT_TRANSCRIPTION",
            metadata={
                "provider": "openai",
                "model": "whisper-1",
                "audio_size_bytes": str(audio_size_bytes),
                "transcript_length": str(len(response)),
                "duration_ms": str(duration_ms)
            },
            success=True
        )
        span.set_action([action])
        
        usage = UsageModel(
            model="whisper-1",
            usage_type="audio_seconds",
            units_used=audio_size_bytes / 16000,
            cost_in_usd=0.006
        )
        span.set_usage([usage])
        
        span.set_attribute("transcript_length", len(response))
        span.set_status({"code": 1, "message": "Success"})
        span.end()
        
        return response
        
    except Exception as e:
        span.set_error(e)
        span.set_status({"code": 2, "message": "Error"})
        span.end()
        raise

# Usage
transcript = transcribe_with_tracking("./audio/sample.mp3")

import OpenAI from 'openai';
import { SpanWrapper, ActionModel } from 'netra-sdk';
import * as fs from 'fs';

const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });

async function transcribeWithTracking(audioPath: string): Promise<string> {
  const span = new SpanWrapper('openai-stt');
  span.start();

  try {
    const startTime = Date.now();
    const audioBuffer = fs.readFileSync(audioPath);
    const audioSizeBytes = audioBuffer.length;

    span.setAttribute('audio_file', audioPath);
    span.setAttribute('audio_size_bytes', audioSizeBytes);
    span.setAttribute('model', 'whisper-1');

    const file = fs.createReadStream(audioPath);
    const response = await openai.audio.transcriptions.create({
      model: 'whisper-1',
      file: file,
      response_format: 'text'
    });

    const duration = Date.now() - startTime;

    const action: ActionModel = {
      start_time: (startTime * 1000000).toString(),
      action: 'API',
      action_type: 'STT_TRANSCRIPTION',
      metadata: {
        provider: 'openai',
        model: 'whisper-1',
        audio_size_bytes: audioSizeBytes.toString(),
        transcript_length: response.length.toString(),
        duration_ms: duration.toString()
      },
      success: true
    };
    span.setAction([action]);

    span.setUsage({
      model: 'whisper-1',
      usage_type: 'audio_seconds',
      units_used: audioSizeBytes / 16000,
      cost_in_usd: 0.006
    });

    span.setAttribute('transcript_length', response.length);
    span.setStatus({ code: 1, message: 'Success' });
    span.end();

    return response;
  } catch (error) {
    span.setError(error as Error);
    span.setStatus({ code: 2, message: 'Error' });
    span.end();
    throw error;
  }
}

// Usage
const transcript = await transcribeWithTracking('./audio/sample.mp3');

Supported Models

Speech-to-Text

Model	Description
`gpt-4o-transcribe`	High-accuracy transcription powered by GPT-4o
`gpt-4o-mini-transcribe`	Cost-effective transcription with GPT-4o Mini
`whisper-1`	General-purpose speech recognition

Text-to-Speech

Model	Description
`gpt-4o-mini-tts`	Next-generation TTS with expressive voices and style instructions
`tts-1`	Standard quality, optimized for speed
`tts-1-hd`	High-definition audio quality

Next Steps

Quick Start Guide - Complete setup and configuration
Auto Instrumentation - Automatic tracing for supported libraries
Decorators - Add custom tracing with @workflow, @agent, and @task decorators
Session Tracking - Track user sessions and conversations
OpenAI Audio Documentation - Official OpenAI audio and speech documentation

​Installation

​Usage

​Examples

​Speech-to-Text with Whisper

​Text-to-Speech

​Manual Span Creation with Action Tracking

​Supported Models

​Speech-to-Text

​Text-to-Speech

​Next Steps

Installation

Usage

Examples

Speech-to-Text with Whisper

Text-to-Speech

Manual Span Creation with Action Tracking

Supported Models

Speech-to-Text

Text-to-Speech

Next Steps