TWITTER/X
FEED

// User U follows 500 people. Naive read-time approach:
SELECT tweet_id FROM tweets WHERE user_id IN (followee_1, followee_2, ... followee_500)
ORDER BY created_at DESC LIMIT 200;

// Cost: 500 DB queries (or 1 massive IN query) per timeline load
// 320,000 timeline reads/sec × 500 queries = 160,000,000 queries/sec
// A well-tuned MySQL handles ~100,000 QPS → need 1,600 DB nodes
// Merge latency: 500 streams × network roundtrip → 200–500ms → SLA violation

// Conclusion: pure read-time fan-out is IMPOSSIBLE at Twitter scale
// We must pre-compute (at least partially) the home timeline

function getHomeTimeline(userId, limit=200):

    // 1. Pre-computed portion (fan-out-on-write tweets)
    precomputed = redis.ZREVRANGE("timeline:" + userId, 0, limit * 2, WITHSCORES)
    //    O(log N) — fast, covers all normal users the person follows

    // 2. Celebrity injection (fan-out-on-read portion)
    celebrities = socialGraph.getCelebrityFollowees(userId)
    //    Typically <50 celebrities per user (manageable)

    celeb_tweets = []
    for celeb in celebrities:
        recent = tweetCache.getRecentTweets(celeb.userId, n=50)
        celeb_tweets.extend(recent)
    //    50 celebrities × 50 tweets = 2,500 tweet fetches (cached in Redis)

    // 3. Merge by timestamp + deduplicate retweets
    merged = mergeSortedByTimestamp(precomputed, celeb_tweets)
    return merged[:limit]

    // Total latency: ~5–20ms (all Redis operations)

POST /tweet → [Tweet Service]
  → INSERT INTO tweets (tweet_id=snowflake(), user_id, content, ...) ✓ durable
  → HSET tweet:{id} userId content likeCount ...                    ✓ cached
  → Kafka.publish("tweet-created", {tweetId, userId, followerCount})

[Fanout Worker] consumes from Kafka:
  if followerCount < 10_000:
      followers = socialGraph.getFollowers(userId)    // read from graph DB
      for followerId in followers:
          redis.ZADD("timeline:"+followerId, timestamp, tweetId)
          redis.ZREMRANGEBYRANK("timeline:"+followerId, 0, -1001)  // keep top 1000
  else:
      // Celebrity: just ensure tweet is in tweet cache, no fanout
      // Timeline reads will inject this lazily
      redis.HSET("tweet:"+tweetId, ...)

// Key pattern: timeline:{userId}
// Type: Sorted Set — score = tweet timestamp (epoch ms)
// Members: tweet_id strings

redis.ZADD("timeline:123", 1700000500000, "tweet_abc")
redis.ZADD("timeline:123", 1700000400000, "tweet_def")

// Read top 200:
tweetIds = redis.ZREVRANGE("timeline:123", 0, 199)   // O(log N + 200)

// Batch hydrate (pipeline, single roundtrip):
tweets = redis.PIPELINE { tweetIds.map(id => HGETALL("tweet:"+id)) }

// Trim timeline to 1000 entries (memory bound):
redis.ZREMRANGEBYRANK("timeline:123", 0, -1001)  // keep newest 1000

// Memory: 300M users × 1000 IDs × 8 bytes = 2.4 TB → Redis cluster

// A viral tweet receives 5M likes in 10 minutes
// = 8,333 likes/sec at peak

// Naive: UPDATE tweets SET like_count = like_count + 1 WHERE tweet_id = ?
// Problem: 8,333 concurrent UPDATE ops on SAME row = row-level lock contention
// MySQL handles ~10K single-row updates/sec → this saturates the primary
// And we have thousands of tweets being liked simultaneously

// Solution: decouple write from increment

// IMAGE UPLOAD:
Client → CDN upload endpoint → S3 (raw bucket)
Tweet stores: media_ids: ["s3://tweets-raw/2024/01/img_abc.jpg"]
CDN serves: https://pbs.twimg.com/media/img_abc.jpg
// CDN cache hit rate: 99%+ for viral content
// Without CDN: 100M impressions × 500KB = 50TB bandwidth from S3 → $$$

// VIDEO UPLOAD (async transcoding):
Client → S3 raw → Lambda trigger → Transcoding worker
  Transcodes to HLS (HTTP Live Streaming) at multiple bitrates:
    240p, 480p, 720p, 1080p
  Output → S3 transcoded bucket → CDN
// HLS: browser fetches small segments (2-10sec), adapts bitrate to bandwidth

// Kafka stream: every tweet → extract hashtags
Tweet: "Just watched #Oppenheimer, amazing! #movies"
Extract: ["#Oppenheimer", "#movies"]

// Flink/Storm: count per hashtag in 1-hour sliding window
// Min-heap: maintain top-30 hashtags globally + per-region

// Store in Redis sorted set:
redis.ZINCRBY("trending:global", 1, "#Oppenheimer")
redis.ZINCRBY("trending:US", 1, "#Oppenheimer")

// Read top 10:
redis.ZREVRANGE("trending:global", 0, 9, WITHSCORES)

// Refresh: recalculate every 5 minutes
// Geographic trending: separate sorted set per region