DESIGN
YOUTUBE

// Step 1: Client initiates upload session
POST /upload/initiate
Body: {"filename": "video.mp4", "size": 2147483648, "sha256": "abc123..."}
Response: {"uploadId": "up_xyz", "chunkSize": 5242880}  ← 5 MB chunks

// Step 2: Client checks for dedup (same SHA-256 already exists)
POST /upload/check?hash=abc123
Response: {"exists": false}  ← proceed with upload
// If exists: {"exists": true, "videoId": "abc123"} → DONE, no upload needed!

// Step 3: Upload chunks (can be parallelised)
PUT /upload/up_xyz/chunk/0  Body: [bytes 0–5MB]       → 200 OK
PUT /upload/up_xyz/chunk/1  Body: [bytes 5MB–10MB]    → 200 OK
PUT /upload/up_xyz/chunk/2  Body: [bytes 10MB–15MB]   → 500 (network drop)

// Step 4: Resume from last successful chunk
GET /upload/up_xyz/status   → {"lastChunk": 1}
PUT /upload/up_xyz/chunk/2  Body: [bytes 10MB–15MB]   → 200 OK  ← retry

// Step 5: Finalize — triggers transcoding pipeline
POST /upload/up_xyz/complete → {"videoId": "abc456", "status": "processing"}

// Naive: one worker transcodes full 60-minute video
// 60-min video at 1x realtime = 60 min transcoding time ← too slow

// YouTube's approach: temporal parallelism
// 1. Split raw video into 1-minute segments
60-minute video → 60 × 1-minute segments

// 2. Dispatch each segment to a separate worker (60 workers)
Worker 01: segment_01 → transcode to all 6 quality levels
Worker 02: segment_02 → transcode to all 6 quality levels
Worker 03: segment_03 → transcode to all 6 quality levels
...
Worker 60: segment_60 → transcode to all 6 quality levels

// 3. All workers run simultaneously → done in ~1 minute (60× speedup)
// 4. Concatenate segments → complete HLS playlist per quality level

// Per-resolution breakdown (for each segment):
Worker A: 360p  (fast — ~5 sec/segment)   ← first available, serve to poor connections immediately
Worker B: 480p  (~8 sec/segment)
Worker C: 720p  (~15 sec/segment)
Worker D: 1080p (~25 sec/segment)
Worker E: 1440p (~40 sec/segment)
Worker F: 4K    (~60 sec/segment)

#EXTM3U
#EXT-X-STREAM-INF:BANDWIDTH=500000,RESOLUTION=640x360
https://cdn.youtube.com/v/abc123/360p.m3u8

#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=1280x720
https://cdn.youtube.com/v/abc123/720p.m3u8

#EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=1920x1080
https://cdn.youtube.com/v/abc123/1080p.m3u8

# Player logic (every 2 seconds):
# measured bandwidth > 5 Mbps  → switch to 1080p
# measured bandwidth 2.5–5 Mbps → switch to 720p
# measured bandwidth < 500 Kbps  → switch to 360p
# Switches happen at segment boundaries → no buffering

// New video uploaded by channel with 10M subscribers:
// Don't wait for cache misses — proactively push to edges

// Tier 1: Subscriber count-based pre-warm
if (channel.subscriberCount > 1_000_000) {
    cdn.prefetch(videoUrl, tier: "edge", regions: "all");   // push to all edges
} else if (channel.subscriberCount > 100_000) {
    cdn.prefetch(videoUrl, tier: "regional");              // push to regional only
}

// Tier 2: Virality-based dynamic warm (triggered by view velocity)
if (viewVelocity > 10_000 // views/minute) {
    cdn.prefetch(videoUrl, tier: "edge", regions: "all");
}

// Pre-warm only first 5 segments (first 10–30 seconds)
// Reason: most users watch the start; remaining segments warmed on demand
// "Seek-ahead": when user is at segment N, pre-fetch N+1 to N+5

// On every view event:
// 1. INCR random Redis shard (instant, non-blocking)
INCR view:{videoId}:shard_{random(10)}

// 2. Publish to Kafka (async, doesn't block response)
kafka.publish("view-events", {videoId, userId, ip, timestamp, country})

// View count displayed to user:
GET /api/views/{videoId}
  → SUM MGET view:{videoId}:shard_0 ... view:{videoId}:shard_9  ← Redis (fast)

// Analytics dashboard (historical, per-country, per-hour):
SELECT COUNT(*) FROM view_events WHERE video_id = X AND timestamp > T
  → ClickHouse query  ← accurate, supports complex aggregations

// Spam prevention:
// Check: SETNX view_dedup:{videoId}:{ip}:{hour} 1  EX 3600
// If key already exists → don't count this view (same IP, same hour)

// Storage layout:
raw/{videoId}/original.mp4                   ← deleted after transcoding (save cost)
processed/{videoId}/master.m3u8               ← HLS master playlist
processed/{videoId}/360p/seg_001.ts           ← 2s video segments
processed/{videoId}/1080p/seg_001.ts
thumbnails/{videoId}/thumb_1.jpg             ← multiple choices for uploader

// Lifecycle policies (automated tiering by popularity):
Hot   (>100 views/month):  Standard storage + CDN — fast and expensive
Warm  (10–100 views/month): Nearline — slightly slower, 50% cheaper
Cold  (<10 views/month):   Coldline — retrieval delay, 80% cheaper
Archive (<1 view/month):   Archive — hours to retrieve, 95% cheaper

// ~80% of all YouTube videos have fewer than 1K total views ever
// Tiering long-tail to cold storage saves enormous cost

// Replication:
// GCS multi-region: automatic 3x replication across AZs
// Cross-region: popular videos replicated to US, EU, APAC buckets

// Elasticsearch index: "videos"
{
  "videoId": "abc123",
  "title": "How to make pasta",            ← full-text search
  "description": "Step by step guide...",    ← full-text search
  "tags": ["cooking", "pasta", "recipe"],  ← exact match
  "transcript": "Today we're making...",     ← auto-generated captions
  "viewCount": 1500000,                      ← boost popular results
  "uploadDate": "2024-01-15",               ← recency signal
  "channelSubscribers": 5000000             ← authority signal
}

// Keep ES in sync with MySQL:
// MySQL change → Debezium CDC → Kafka topic "db-changes" → ES consumer
// Async — ES may lag MySQL by seconds, acceptable for search freshness

// Autocomplete: ES "search_as_you_type" field type on title
// Returns suggestions after 2 characters