Self-Hosting Whisper V3 on Bare Metal: High-Throughput Speech-to-Text APIs

# Install system dependencies (ffmpeg is required for audio processing)
sudo apt update && sudo apt install -y ffmpeg curl

# Install Astral UV (Fast python package manager)
curl -LsSf https://astral.sh/uv/install.sh | sh
source $HOME/.cargo/env

# Create and activate environment
uv venv whisper_env
source whisper_env/bin/activate

# Install faster-whisper and FastAPI
uv pip install faster-whisper fastapi "uvicorn[standard]" python-multipart

from faster_whisper import WhisperModel

# Use large-v3-turbo for low-latency English, or large-v3 for max accuracy
# compute_type="int8" enables the quantization hack
model = WhisperModel("large-v3-turbo", device="cuda", compute_type="int8")

print("Model loaded successfully into VRAM!")

# Continue in transcribe.py
audio_file = "patient_interview.wav"

# vad_filter=True skips dead air, preventing hallucinations and speeding up processing
segments, info = model.transcribe(
    audio_file,
    beam_size=5,
    vad_filter=True,
    vad_parameters=dict(min_silence_duration_ms=500),
    language="en" # Hardcoding language saves ~50ms of auto-detect latency
)

print(f"Detected language: {info.language} with probability {info.language_probability}")

for segment in segments:
    print(f"[{segment.start:.2f}s -> {segment.end:.2f}s] {segment.text}")

# server.py
import os
import tempfile
from fastapi import FastAPI, UploadFile, File, HTTPException
from fastapi.responses import JSONResponse
from faster_whisper import WhisperModel

app = FastAPI(title="Private Whisper API")

# Load model globally on startup (Warm state)
model = WhisperModel("large-v3-turbo", device="cuda", compute_type="int8")

@app.post("/v1/audio/transcriptions")
async def transcribe_audio(file: UploadFile = File(...)):
    if not file.filename.endswith((".wav", ".mp3", ".m4a")):
        raise HTTPException(status_code=400, detail="Invalid audio format")

    # Save uploaded file temporarily
    suffix = os.path.splitext(file.filename)[1]
    with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
        tmp.write(await file.read())
        tmp_path = tmp.name

    try:
        segments, info = model.transcribe(tmp_path, beam_size=5, vad_filter=True)
        
        out_segments = []
        full_text = ""
        for s in segments:
            out_segments.append({"start": s.start, "end": s.end, "text": s.text})
            full_text += s.text + " "

        return JSONResponse({
            "text": full_text.strip(),
            "language": info.language,
            "segments": out_segments
        })
    finally:
        os.unlink(tmp_path) # Clean up temp file

# Run this server using: uvicorn server:app --host 0.0.0.0 --port 8000