When a User Shares Media in a DM

What Meta Actually Sends You

Platform-by-Platform Breakdown

1. Instagram DM

{
  "message": {
    "mid": "...",
    "attachments": [
      {
        "type": "ig_reel",        // or "image", "video", "ig_reel", "share"
        "payload": {
          "url": "https://cdn.instagram.com/...",
          "title": "Reel title (if any)"
        }
      }
    ]
  }
}

• The CDN URL in payload.url is ephemeral — it expires in minutes. Download immediately upon webhook receipt.
• For private/restricted accounts, you may only get a URL to the post page, not the raw media.
• You need the instagram_business_manage_messages permission to read attachment payloads.

2. Facebook Messenger

{
  "message": {
    "attachments": [
      {
        "type": "video",          // "image", "video", "audio", "file"
        "payload": {
          "url": "https://video.xx.fbcdn.net/..."
        }
      }
    ]
  }
}

3. WhatsApp Business Cloud API

{
  "messages": [{
    "type": "image",              // "image", "video", "audio", "document", "sticker"
    "image": {
      "id": "3857693717630",
      "mime_type": "image/jpeg",
      "sha256": "...",
      "caption": "Check this out"
    }
  }]
}

1. GET https://graph.facebook.com/v19.0/{media-id} → returns a temporary download URL
2. GET {download_url} with Authorization: Bearer {token} → download the actual file

The Processing Pipeline

Webhook received
       │
       ▼
[1] Extract attachment type + URL/media-ID
       │
       ▼
[2] Download media immediately (CDN URLs expire fast)
       │
       ├─ Image → Vision LLM (Claude, GPT-4V, Gemini Vision)
       ├─ Video → Frame extraction → Vision LLM per frame
       ├─ Audio → Whisper / Deepgram → transcript → LLM
       ├─ Post URL → Scrape OG tags / oEmbed → LLM
       └─ Reel URL → yt-dlp download → frames + audio
       │
       ▼
[3] Build context + user message → Claude
       │
       ▼
[4] Send reply via Send API

Step-by-Step: Analysing Each Media Type

Images

import anthropic, base64, httpx

def analyse_image(image_url: str, user_caption: str) -> str:
    image_data = base64.b64encode(httpx.get(image_url).content).decode()
    
    client = anthropic.Anthropic()
    response = client.messages.create(
        model="claude-opus-4-5",
        max_tokens=1024,
        messages=[{
            "role": "user",
            "content": [
                {
                    "type": "image",
                    "source": {
                        "type": "base64",
                        "media_type": "image/jpeg",
                        "data": image_data
                    }
                },
                {
                    "type": "text",
                    "text": f"The user shared this image and said: '{user_caption}'. Respond helpfully as Orris."
                }
            ]
        }]
    )
    return response.content[0].text

Videos / Reels

import cv2, tempfile, httpx

def extract_frames(video_url: str, num_frames: int = 8) -> list[bytes]:
    # Download video
    video_bytes = httpx.get(video_url).content
    with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
        f.write(video_bytes)
        tmp_path = f.name
    
    # Extract evenly-spaced frames
    cap = cv2.VideoCapture(tmp_path)
    total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
    frames = []
    for i in range(num_frames):
        cap.set(cv2.CAP_PROP_POS_FRAMES, int(i * total / num_frames))
        ret, frame = cap.read()
        if ret:
            _, buf = cv2.imencode(".jpg", frame)
            frames.append(buf.tobytes())
    cap.release()
    return frames

ffmpeg -i reel.mp4 -vn -acodec mp3 audio.mp3

Shared Post URLs (Instagram/Facebook)

GET https://graph.facebook.com/v19.0/instagram_oembed
    ?url=https://www.instagram.com/reel/ABC123/
    &access_token={token}

import requests
from bs4 import BeautifulSoup

def scrape_og(url: str) -> dict:
    html = requests.get(url, headers={"User-Agent": "facebookexternalhit/1.1"}).text
    soup = BeautifulSoup(html, "html.parser")
    return {
        tag["property"]: tag["content"]
        for tag in soup.find_all("meta", property=True)
        if tag.get("content")
    }

Audio Messages (WhatsApp)

import openai

def transcribe_audio(audio_path: str) -> str:
    with open(audio_path, "rb") as f:
        transcript = openai.audio.transcriptions.create(
            model="whisper-1",
            file=f
        )
    return transcript.text

Putting It Together: A Unified Handler

async def handle_attachment(attachment: dict, user_text: str) -> str:
    atype = attachment["type"]
    url   = attachment["payload"].get("url", "")

    if atype == "image":
        return analyse_image(url, user_text)

    elif atype in ("video", "ig_reel"):
        frames    = extract_frames(url)
        transcript = extract_and_transcribe_audio(url)  # optional
        return analyse_video_frames(frames, transcript, user_text)

    elif atype == "share":
        og = scrape_og(url)
        thumbnail = og.get("og:image")
        description = og.get("og:description", "")
        # Optionally fetch and analyse thumbnail
        return ask_claude_about_post(og, thumbnail, user_text)

    elif atype == "audio":
        local_path = download_whatsapp_media(attachment["id"])
        transcript = transcribe_audio(local_path)
        return ask_claude(f"The user sent a voice note: '{transcript}'. {user_text}")

    else:
        return "I received your attachment but can't process this type yet."

Key Constraints to Keep in Mind

• Download on webhook receipt — don't store URLs, store the files or base64.
• For private Instagram posts shared in DMs, you'll only get a link back to the post — you can't pull the raw video unless the account owner granted your app permissions.
• Meta's terms prohibit storing user media beyond what's needed to fulfil the request. Build your pipeline to be stateless — process and discard, don't cache media.
• For reels with music, you'll get the audio but the music may be copyright-restricted. Transcription is fine; redistribution is not.

Recommended Tool Stack

Bright Data

Where it helps:

import requests

def scrape_with_brightdata(post_url: str) -> dict:
    proxy = {
        "http":  "http://USERNAME:PASSWORD@brd.superproxy.io:22225",
        "https": "http://USERNAME:PASSWORD@brd.superproxy.io:22225"
    }
    resp = requests.get(post_url, proxies=proxy, verify=False)
    # parse OG tags / JSON-LD from the response
    return parse_og_tags(resp.text)

POST https://api.brightdata.com/datasets/v3/trigger
{
  "dataset_id": "gd_lk5ns7kz21pck8jpis",   // Instagram posts dataset
  "inputs": [{"url": "https://www.instagram.com/reel/ABC123/"}]
}

Azure Credits

Where it helps:

• Image captioning (imageAnalysis/caption)
• OCR — extracting text from images/screenshots in reels
• Object detection, face detection, brand/logo detection
• Dense captions — describing multiple regions in one image

from azure.ai.vision.imageanalysis import ImageAnalysisClient
from azure.ai.vision.imageanalysis.models import VisualFeatures
from azure.core.credentials import AzureKeyCredential

client = ImageAnalysisClient(endpoint=AZURE_ENDPOINT, credential=AzureKeyCredential(AZURE_KEY))

result = client.analyze_from_url(
    image_url=cdn_url,
    visual_features=[VisualFeatures.CAPTION, VisualFeatures.TEXT, VisualFeatures.OBJECTS]
)
# result.caption.text → "a person cooking pasta in a pan"
# result.read.blocks → OCR text from any on-screen text

from openai import AzureOpenAI

client = AzureOpenAI(
    azure_endpoint=AZURE_OPENAI_ENDPOINT,
    api_key=AZURE_OPENAI_KEY,
    api_version="2024-02-01"
)

with open("reel_audio.mp3", "rb") as f:
    transcript = client.audio.transcriptions.create(
        model="whisper",         # your Azure deployment name
        file=f
    )

from azure.storage.blob import BlobServiceClient

blob_client = BlobServiceClient.from_connection_string(CONN_STR)
container   = blob_client.get_container_client("media-buffer")

# On webhook receipt — fast
media_bytes = httpx.get(cdn_url).content
container.upload_blob(name=message_id, data=media_bytes, overwrite=True)

# In background worker — slow OK
blob = container.download_blob(message_id).readall()

from azure.ai.contentsafety import ContentSafetyClient
from azure.ai.contentsafety.models import AnalyzeImageOptions, ImageData

safety_client = ContentSafetyClient(endpoint, AzureKeyCredential(key))
result = safety_client.analyze_image(AnalyzeImageOptions(image=ImageData(content=image_bytes)))
# Check result.hate_result, result.violence_result, etc.

Revised Full Pipeline with Both

Webhook (Azure Function)
        │
        ▼
[Bright Data] Fetch post/reel metadata + media URL if CDN URL is missing/expired
        │
        ▼
[Azure Blob] Buffer raw media immediately (decouple from webhook timeout)
        │
        ├── [Azure Content Safety] Screen for harmful content
        │
        ├── [Azure Vision] Caption + OCR + object detect (cheap, fast)
        │
        ├── [Azure Whisper] Transcribe audio track
        │
        └── [Claude] Receive enriched context: caption + transcript + OCR + objects
                 │
                 ▼
        [Meta Send API] Reply to user

Cost Impact