// sliding_window_summarizer.js
// Node 18+ (fetch available). Pseudocode placeholders for model calls; replace with your real API calls.

class ContextItem {
  constructor({ type, id, timestamp = Date.now(), text = null, image = null, summary = null }) {
    // type: 'text' | 'image' | 'summary'
    this.type = type;
    this.id = id; // unique id per item
    this.timestamp = timestamp;
    this.text = text;
    this.image = image; // { url, base64, metadata }
    this.summary = summary; // short text summary for evicted images
  }
}

class SlidingWindowStore {
  constructor({ maxItems = 50, maxImagesKeep = 5, summarizeBatchSize = 5 }) {
    this.maxItems = maxItems; // total items (text + image + summary)
    this.maxImagesKeep = maxImagesKeep; // keep this many latest images in full
    this.summarizeBatchSize = summarizeBatchSize; // group size to summarize together
    this.items = []; // oldest first
  }

  addText(id, text) {
    this.items.push(new ContextItem({ type: 'text', id, text }));
    this._enforceLimits();
  }

  addImage(id, imageObj) {
    // imageObj: { url?, base64?, meta? }
    this.items.push(new ContextItem({ type: 'image', id, image: imageObj }));
    this._enforceLimits();
  }

  // internal: ensure limits and trigger summarization/eviction
  _enforceLimits() {
    // Count images
    const images = this.items.filter(i => i.type === 'image');
    if (images.length <= this.maxImagesKeep && this.items.length <= this.maxItems) return;

    // Evict old images beyond the most recent maxImagesKeep
    const imagesToEvict = images.slice(0, Math.max(0, images.length - this.maxImagesKeep));
    if (imagesToEvict.length > 0) {
      // Optionally group them into batches and summarize
      const batches = [];
      for (let i = 0; i < imagesToEvict.length; i += this.summarizeBatchSize) {
        batches.push(imagesToEvict.slice(i, i + this.summarizeBatchSize));
      }

      batches.forEach(batch => {
        this._summarizeAndReplaceBatch(batch);
      });
    }

    // If still over maxItems, drop the oldest non-summary text items (or drop summaries last)
    while (this.items.length > this.maxItems) {
      // Prefer to remove oldest text items first
      const idx = this.items.findIndex(it => it.type === 'text');
      if (idx >= 0) {
        this.items.splice(idx, 1);
      } else {
        // fallback: drop oldest item
        this.items.shift();
      }
    }
  }

  // Create a summary for a batch of image items, replace them with a single summary item
  async _summarizeAndReplaceBatch(batch) {
    if (batch.length === 0) return;

    // Create input for summarizer: combine captions/metadata or fetch image pixels if needed
    const summarizerInput = batch.map(it => {
      return {
        id: it.id,
        meta: it.image?.meta ?? null,
        url: it.image?.url ?? null,
        base64: it.image?.base64 ? '[base64 image trimmed]' : null // avoid huge in-memory transfer in logs
      };
    });

    // Call a vision-capable summarization model
    const summaryText = await visionSummarizeImages(summarizerInput);

    // Build a compact summary item (timestamp uses newest in batch)
    const summaryItem = new ContextItem({
      type: 'summary',
      id: `summary-${batch[0].id}-${Date.now()}`,
      timestamp: Math.max(...batch.map(b => b.timestamp)),
      text: `Summary of ${batch.length} images: ${summaryText}`
    });

    // Remove batch items from store and insert summary at the position of the first evicted item
    const firstIndex = this.items.findIndex(it => it.id === batch[0].id);
    // remove each by id
    const ids = new Set(batch.map(b => b.id));
    this.items = this.items.filter(it => !ids.has(it.id));
    // insert summary at firstIndex (if found) otherwise push to end
    if (firstIndex >= 0 && firstIndex <= this.items.length) {
      this.items.splice(firstIndex, 0, summaryItem);
    } else {
      this.items.push(summaryItem);
    }
  }

  // Build plugin context payload: include full recent images, include summaries for evicted ones
  buildPluginContext() {
    // Convert internal items into the plugin's expected message list
    // Example: map text -> chat message, image -> ImageContent, summary -> chat text
    return this.items.map(it => {
      if (it.type === 'text') return { role: 'user', content: { type: 'text', text: it.text } };
      if (it.type === 'image') {
        return {
          role: 'user',
          content: {
            type: 'image',
            // use URL or base64 depending on what the plugin accepts
            image: it.image
          }
        };
      }
      if (it.type === 'summary') {
        return { role: 'system', content: { type: 'text', text: it.text } };
      }
      return null;
    }).filter(Boolean);
  }
}

/* ---------- Placeholder: vision summarizer ---------- */
/* Replace this function with your call to a vision-capable model that accepts images and returns a compact textual summary.
   The function should accept an array of objects: [{id, url?, base64?, meta?}, ...] and return a short string summary.
*/
async function visionSummarizeImages(images) {
  // Minimal example: combine available captions/metadata; if not available call a model API
  const simpleCombined = images.map(i => (i.meta?.caption ?? `image ${i.id}`)).join('; ');
  // TODO: call your vision model (e.g., OpenAI multimodal endpoint) with the images
  // const response = await fetch('YOUR_VISION_SUMMARY_API', {...});
  // return response.summary;

  // For demo return a trimmed combined caption
  return simpleCombined.length > 300 ? simpleCombined.slice(0, 300) + '...' : simpleCombined;
}

/* ---------- Usage Example ---------- */
async function demo() {
  const store = new SlidingWindowStore({ maxItems: 40, maxImagesKeep: 4, summarizeBatchSize: 4 });

  // Add some text and images as they come in
  store.addText('t1', 'User: Here are the first images');
  store.addImage('img1', { url: '<https://example.com/img1.jpg>', meta: { caption: 'kitchen, person' } });
  store.addImage('img2', { url: '<https://example.com/img2.jpg>', meta: { caption: 'kitchen, dog' } });
  store.addImage('img3', { url: '<https://example.com/img3.jpg>', meta: { caption: 'living room' } });
  store.addImage('img4', { url: '<https://example.com/img4.jpg>', meta: { caption: 'backyard' } });
  store.addImage('img5', { url: '<https://example.com/img5.jpg>', meta: { caption: 'front door' } });

  // After adding, _enforceLimits will run and may summarize older images
  // If summarization is async and you want to await completion, ensure calls are awaited in your production flow.
  // For demo, we wait a tick to let async summarization finish
  await new Promise(r => setTimeout(r, 200));

  // Build the payload to send to OpenAI Realtime plugin
  const pluginContext = store.buildPluginContext();
  console.log(JSON.stringify(pluginContext, null, 2));

  // Send pluginContext to your Realtime plugin API as needed
  // e.g., realtimeClient.send({ messages: pluginContext })
}

demo().catch(console.error);

visionSummarizeImages

# sliding_window_summarizer.py
# Python 3.9+. Replace vision_summarize_images with your multimodal model call.

from dataclasses import dataclass, field
from typing import List, Optional, Dict, Any
import time
import asyncio
import uuid

@dataclass
class ContextItem:
    type: str  # 'text' | 'image' | 'summary'
    id: str
    timestamp: float = field(default_factory=lambda: time.time())
    text: Optional[str] = None
    image: Optional[Dict[str, Any]] = None  # { 'url':..., 'base64':..., 'meta':... }
    summary: Optional[str] = None

class SlidingWindowStore:
    def __init__(self, max_items: int = 50, max_images_keep: int = 5, summarize_batch_size: int = 5):
        self.max_items = max_items
        self.max_images_keep = max_images_keep
        self.summarize_batch_size = summarize_batch_size
        self.items: List[ContextItem] = []  # oldest first
        self._summarize_lock = asyncio.Lock()

    def add_text(self, text: str, id: Optional[str] = None):
        self.items.append(ContextItem(type='text', id=id or str(uuid.uuid4()), text=text))
        asyncio.create_task(self._enforce_limits_async())

    def add_image(self, image_obj: Dict[str, Any], id: Optional[str] = None):
        self.items.append(ContextItem(type='image', id=id or str(uuid.uuid4()), image=image_obj))
        asyncio.create_task(self._enforce_limits_async())

    async def _enforce_limits_async(self):
        # avoid concurrent summarization runs
        async with self._summarize_lock:
            images = [it for it in self.items if it.type == 'image']
            if len(images) <= self.max_images_keep and len(self.items) <= self.max_items:
                return

            # Evict older images beyond the most recent max_images_keep
            images_to_evict = images[:max(0, len(images) - self.max_images_keep)]
            if images_to_evict:
                # batch them
                batches = [images_to_evict[i:i+self.summarize_batch_size]
                           for i in range(0, len(images_to_evict), self.summarize_batch_size)]
                for batch in batches:
                    await self._summarize_and_replace_batch(batch)

            # If still over max_items, remove oldest text items first
            while len(self.items) > self.max_items:
                idx = next((i for i, it in enumerate(self.items) if it.type == 'text'), None)
                if idx is not None:
                    self.items.pop(idx)
                else:
                    # fallback: pop oldest
                    self.items.pop(0)

    async def _summarize_and_replace_batch(self, batch: List[ContextItem]):
        if not batch:
            return
        # Build input for summarizer
        summarizer_input = []
        for it in batch:
            summarizer_input.append({
                'id': it.id,
                'meta': it.image.get('meta') if it.image else None,
                'url': it.image.get('url') if it.image else None,
                'base64': '[base64 trimmed]' if it.image and it.image.get('base64') else None
            })

        # Call multimodal summarizer (replace with your model call)
        summary_text = await vision_summarize_images(summarizer_input)

        # Create summary item (timestamp uses newest in batch)
        newest_ts = max(it.timestamp for it in batch)
        summary_item = ContextItem(
            type='summary',
            id=f"summary-{batch[0].id}-{int(time.time()*1000)}",
            timestamp=newest_ts,
            text=f"Summary of {len(batch)} images: {summary_text}"
        )

        # Remove batch items from store and insert summary at first batch position
        ids = {it.id for it in batch}
        first_index = next((i for i, it in enumerate(self.items) if it.id == batch[0].id), None)
        self.items = [it for it in self.items if it.id not in ids]
        if first_index is not None and 0 <= first_index <= len(self.items):
            self.items.insert(first_index, summary_item)
        else:
            self.items.append(summary_item)

    def build_plugin_context(self) -> List[Dict[str, Any]]:
        payload = []
        for it in self.items:
            if it.type == 'text':
                payload.append({'role': 'user', 'content': {'type': 'text', 'text': it.text}})
            elif it.type == 'image':
                payload.append({'role': 'user', 'content': {'type': 'image', 'image': it.image}})
            elif it.type == 'summary':
                payload.append({'role': 'system', 'content': {'type': 'text', 'text': it.text}})
        return payload

# Placeholder summarizer: replace with your vision-capable model call.
async def vision_summarize_images(images: List[Dict[str, Any]]) -> str:
    # Simple combine captions/metadata, or call your model/SDK here.
    combined = '; '.join((img.get('meta', {}).get('caption') or f"image {img.get('id')}") for img in images)
    return (combined[:500] + '...') if len(combined) > 500 else combined

# Example usage
async def demo():
    store = SlidingWindowStore(max_items=40, max_images_keep=4, summarize_batch_size=4)

    store.add_text("User: initial message")
    store.add_image({'url': '<https://example.com/img1.jpg>', 'meta': {'caption': 'kitchen, person'}})
    store.add_image({'url': '<https://example.com/img2.jpg>', 'meta': {'caption': 'kitchen, dog'}})
    store.add_image({'url': '<https://example.com/img3.jpg>', 'meta': {'caption': 'living room'}})
    store.add_image({'url': '<https://example.com/img4.jpg>', 'meta': {'caption': 'backyard'}})
    store.add_image({'url': '<https://example.com/img5.jpg>', 'meta': {'caption': 'front door'}})

    # Wait for async summarization to complete in demo
    await asyncio.sleep(0.2)

    plugin_context = store.build_plugin_context()
    print(plugin_context)
    # Send plugin_context to your Realtime plugin client (LiveKit/OpenAI) as required.

if __name__ == '__main__':
    asyncio.run(demo())