Streaming

Streaming delivers generated content in real time — the "typing effect" you see in chat interfaces. SeaWhale AI supports streaming for every model.

What is streaming?

Instead of waiting for the model to finish and returning everything at once, streaming pushes the output to the client in small chunks as it is generated. This:

• Improves the user experience — users see progress immediately instead of waiting
• Suits chat interfaces — reproduces the typing effect of a real conversation
• Reduces perceived latency — the application feels faster and more responsive

Enabling streaming

Set stream: true in your request.

Complete examples:

Python

import requests
import json

question = "How would you build the tallest building in the world?"

# SeaWhale AI API configuration
url = "https://api.seawhaleai.com/v2/chat/completions"
headers = {
  "Authorization": "<API_KEY>",  # Replace with your API key
  "Content-Type": "application/json"
}

# Request payload — note stream=True
payload = {
  "model": "gpt-4o",
  "messages": [{"role": "user", "content": question}],
  "stream": True  # Enable streaming
}

# Use a buffer to process the stream
buffer = ""
with requests.post(url, headers=headers, json=payload, stream=True) as r:
  for chunk in r.iter_content(chunk_size=1024, decode_unicode=True):
    buffer += chunk
    while True:
      try:
        # Look for a complete SSE (Server-Sent Events) line
        line_end = buffer.find('\n')
        if line_end == -1:
          break

        line = buffer[:line_end].strip()
        buffer = buffer[line_end + 1:]

        # Parse lines that start with "data: "
        if line.startswith('data: '):
          data = line[6:]
          if data == '[DONE]':  # End-of-stream marker
            break

          try:
            data_obj = json.loads(data)
            content = data_obj["choices"][0]["delta"].get("content")
            if content:
              print(content, end="", flush=True)  # Print as it arrives
          except json.JSONDecodeError:
            pass
      except Exception:
        break

TypeScript

const question = 'How would you build the tallest building in the world?'

// Send a streaming request
const response = await fetch('https://api.seawhaleai.com/v2/chat/completions', {
  method: 'POST',
  headers: {
    Authorization: '<API_KEY>', // Replace with your API key
    'Content-Type': 'application/json',
  },
  body: JSON.stringify({
    model: 'gpt-4o',
    messages: [{role: 'user', content: question}],
    stream: true, // Enable streaming
  }),
})

// Get a reader for the response stream
const reader = response.body?.getReader()
if (!reader) {
  throw new Error('Unable to read the response body')
}

const decoder = new TextDecoder()
let buffer = ''

try {
  while (true) {
    const {done, value} = await reader.read()
    if (done) break

    // Append the new chunk to the buffer
    buffer += decoder.decode(value, {stream: true})

    // Process complete lines in the buffer
    while (true) {
      const lineEnd = buffer.indexOf('\n')
      if (lineEnd === -1) break

      const line = buffer.slice(0, lineEnd).trim()
      buffer = buffer.slice(lineEnd + 1)

      if (line.startsWith('data: ')) {
        const data = line.slice(6)
        if (data === '[DONE]') break // End-of-stream marker

        try {
          const parsed = JSON.parse(data)
          const content = parsed.choices[0].delta.content
          if (content) {
            console.log(content) // Print as it arrives
          }
        } catch (e) {
          // Ignore invalid JSON
        }
      }
    }
  }
} finally {
  reader.cancel() // Clean up
}

How streaming works

Data format: streaming uses the SSE (Server-Sent Events) protocol, where every data line begins with data: .

End marker: receiving data: [DONE] means the stream is finished.

Incremental content: each chunk contains a delta object whose content field holds the newly generated fragment.

Cancelling a stream

You can abort the connection at any time to cancel a streaming request. With providers that support it, this stops generation — and billing — immediately.

How to implement cancellation:

Python

import requests
from threading import Event, Thread

def stream_with_cancellation(prompt: str, cancel_event: Event):
    """Streaming request that supports cancellation"""
    with requests.Session() as session:
        response = session.post(
            "https://api.seawhaleai.com/v2/chat/completions",
            headers={"Authorization": "<API_KEY>"},
            json={
                "model": "gpt-4o",
                "messages": [{"role": "user", "content": prompt}],
                "stream": True
            },
            stream=True
        )

        try:
            for line in response.iter_lines():
                # Check whether cancellation was requested
                if cancel_event.is_set():
                    response.close()
                    print("\nStream cancelled")
                    return
                if line:
                    print(line.decode(), end="", flush=True)
        finally:
            response.close()

# Example usage
cancel_event = Event()
stream_thread = Thread(
    target=lambda: stream_with_cancellation("Write a story", cancel_event)
)
stream_thread.start()

# Cancel the stream when needed
# cancel_event.set()

TypeScript

// Create an AbortController to cancel the request
const controller = new AbortController()

try {
  const response = await fetch('https://api.seawhaleai.com/v2/chat/completions', {
    method: 'POST',
    headers: {
      Authorization: '<API_KEY>',
      'Content-Type': 'application/json',
    },
    body: JSON.stringify({
      model: 'gpt-4o',
      messages: [{role: 'user', content: 'Write a story'}],
      stream: true,
    }),
    signal: controller.signal, // Pass the abort signal
  })

  // Process the stream...
  const reader = response.body?.getReader()
  // ... stream handling omitted
} catch (error) {
  if (error.name === 'AbortError') {
    console.log('Stream cancelled')
  } else {
    throw error
  }
}

// Cancel the stream when needed
// controller.abort();

TIP

Important:

• Cancellation only applies to streaming requests (stream: true)
• Some providers do not support immediate cancellation and will keep generating until finished
• Cancelling promptly saves token costs

Error handling with streams

Two kinds of errors can occur during streaming, and SeaWhale AI handles them differently depending on when they happen.

Case 1: errors before the stream starts

If an error occurs before any content has been streamed to the client, you receive a standard HTTP error response with the appropriate status code.

Error format:

{
  "error": {
    "code": 400,
    "message": "The specified model is invalid"
  }
}

Common HTTP status codes:

• 400 — Bad request (invalid parameters)
• 401 — Unauthorized (invalid API key)
• 402 — Payment required (insufficient balance)
• 429 — Too many requests (rate limited)
• 502 — Bad gateway (provider error)
• 503 — Service unavailable (no provider available)

Case 2: errors mid-stream

If an error occurs after content has already started streaming, the HTTP status code has already been sent (200 OK) and cannot be changed. The error is delivered as an SSE event instead.

Error format:

data: {
  "id": "cmpl-abc123",
  "object": "chat.completion.chunk",
  "created": 1234567890,
  "model": "gpt-3.5-turbo",
  "error": {
    "code": "server_error",
    "message": "The provider disconnected unexpectedly"
  },
  "choices": [{
    "index": 0,
    "delta": {"content": ""},
    "finish_reason": "error"
  }]
}

How to recognize a mid-stream error:

• The error detail sits in the top-level error field of the response
• finish_reason in the choices array is "error", which terminates the stream correctly
• The HTTP status code is still 200 OK (the headers were already sent)
• The stream ends immediately after the error event

Complete error handling example

The code below handles both kinds of errors correctly:

Python

import requests
import json

def stream_with_error_handling(prompt):
    response = requests.post(
        'https://api.seawhaleai.com/v2/chat/completions',
        headers={'Authorization': '<API_KEY>'},
        json={
            'model': 'gpt-4o',
            'messages': [{'role': 'user', 'content': prompt}],
            'stream': True
        },
        stream=True
    )

    # Step 1: check the initial HTTP status (errors before the stream starts)
    if response.status_code != 200:
        error_data = response.json()
        print(f"Error: {error_data['error']['message']}")
        return

    # Step 2: process the stream and catch mid-stream errors
    for line in response.iter_lines():
        if line:
            line_text = line.decode('utf-8')
            if line_text.startswith('data: '):
                data = line_text[6:]
                if data == '[DONE]':
                    break

                try:
                    parsed = json.loads(data)

                    # Check for a mid-stream error
                    if 'error' in parsed:
                        print(f"\nStream error: {parsed['error']['message']}")
                        # Confirm the stream terminated because of the error
                        if parsed.get('choices', [{}])[0].get('finish_reason') == 'error':
                            print("Stream terminated due to an error")
                        break

                    # Normal content handling
                    content = parsed['choices'][0]['delta'].get('content')
                    if content:
                        print(content, end='', flush=True)

                except json.JSONDecodeError:
                    pass  # Ignore JSON parse errors

TypeScript

async function streamWithErrorHandling(prompt: string) {
  const response = await fetch(
    'https://api.seawhaleai.com/v2/chat/completions',
    {
      method: 'POST',
      headers: {
        'Authorization': '<API_KEY>',
        'Content-Type': 'application/json',
      },
      body: JSON.stringify({
        model: 'gpt-4o',
        messages: [{ role: 'user', content: prompt }],
        stream: true,
      }),
    }
  );

  // Step 1: check the initial HTTP status (errors before the stream starts)
  if (!response.ok) {
    const error = await response.json();
    console.error(`Error: ${error.error.message}`);
    return;
  }

  const reader = response.body?.getReader();
  if (!reader) throw new Error('No response body');

  const decoder = new TextDecoder();
  let buffer = '';

  try {
    while (true) {
      const { done, value } = await reader.read();
      if (done) break;

      buffer += decoder.decode(value, { stream: true });

      while (true) {
        const lineEnd = buffer.indexOf('\n');
        if (lineEnd === -1) break;

        const line = buffer.slice(0, lineEnd).trim();
        buffer = buffer.slice(lineEnd + 1);

        if (line.startsWith('data: ')) {
          const data = line.slice(6);
          if (data === '[DONE]') return;

          try {
            const parsed = JSON.parse(data);

            // Check for a mid-stream error
            if (parsed.error) {
              console.error(`Stream error: ${parsed.error.message}`);
              // Confirm the stream terminated because of the error
              if (parsed.choices?.[0]?.finish_reason === 'error') {
                console.log('Stream terminated due to an error');
              }
              return;
            }

            // Normal content handling
            const content = parsed.choices[0].delta.content;
            if (content) {
              console.log(content);
            }
          } catch (e) {
            // Ignore JSON parse errors
          }
        }
      }
    }
  } finally {
    reader.cancel();
  }
}