Design
WhatsApp

// 1. HTTP Upgrade handshake
GET /ws HTTP/1.1
Upgrade: websocket
Connection: Upgrade
Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==

// Server responds:
HTTP/1.1 101 Switching Protocols
Upgrade: websocket
Connection: Upgrade

// 2. TCP connection stays open — frames flow bidirectionally
Client → Server: {"type":"message","to":456,"content":"Hey!"}
Server → Client: {"type":"message","from":123,"content":"What's up?"}
Server → Client: {"type":"ack","msgId":7890,"status":"delivered"}

// 3. Heartbeat every 30s — keeps connection alive through NAT
Server → Client: PING
Client → Server: PONG

// 4. On disconnect: client reconnects, fetches offline messages via REST
GET /messages/offline?since="last_message_id"

// Bob is offline: message stored in Cassandra (already done — Step 2 above)
// Also: store message_id in inbox:{bobId} sorted set
ZADD inbox:bob {message_id} {message_id}

// Bob reconnects (WebSocket upgrade):
// 1. REST call to fetch missed messages
GET /messages/offline?userId=bob&since={lastReadMessageId}

// 2. Server queries Cassandra for all conversations Bob participates in
// 3. Returns all messages with message_id > lastReadMessageId
// 4. Push to Bob's new WebSocket connection
// 5. Update last_read_message_id = latest received
// 6. Mark each message as delivered → route receipts back to senders

-- Option A: counters on message row
ALTER TABLE messages ADD delivered_count INT DEFAULT 0;
ALTER TABLE messages ADD read_count INT DEFAULT 0;
-- ✓✓ shown when delivered_count = group_size
-- ✓✓ blue when read_count = group_size

-- Option B: per-recipient receipts table (more granular, scales better)
CREATE TABLE message_receipts (
    message_id    BIGINT,
    recipient_id  BIGINT,
    status        VARCHAR,  -- 'delivered' | 'read'
    updated_at    TIMESTAMP,
    PRIMARY KEY (message_id, recipient_id)
);
-- Query: SELECT COUNT(*) FROM message_receipts
--        WHERE message_id = X AND status = 'read'
-- Compare to group size → determine if all have read

// Write: every 30s heartbeat via WebSocket
SETEX presence:{userId} 45 "online"   // 45s TTL > 30s heartbeat interval

// Read: check if user is online
String val = redis.get("presence:" + userId);
if (val != null) return "Online";
else return "Last seen at " + db.getLastSeen(userId);

// Scaling the writes:
// 100M active users × 1 write/30s = 3.3M writes/sec
// Redis Cluster: shard by hash(userId) across 10+ nodes
// Each node handles ~330K writes/sec → achievable

// Subscriber-based presence notifications (avoids fan-out to all contacts):
// Bob opens chat with Alice → subscribe to presence:{aliceId}
// Alice comes online → notify only active subscribers (open chat windows)
// NOT: notify all 300 of Alice's contacts (expensive, most don't care)

public void handleGroupMessage(GroupMessageEvent e) {
    // 1. Message already stored in Cassandra by Chat Server
    UUID groupId = e.groupId;
    long messageId = e.messageId;

    // 2. Fetch all group members (paginated from Cassandra/Redis)
    List<Long> members = groupService.getMembers(groupId);

    for (long memberId : members) {
        if (memberId == e.senderId) continue;  // skip sender

        String serverAddr = redis.get("session:" + memberId);

        if (serverAddr != null) {
            // Online: route to their Chat Server
            chatRouter.deliver(serverAddr, memberId, messageId);
        } else {
            // Offline: store in inbox for later delivery
            redis.zadd("inbox:" + memberId, messageId, messageId);
        }
    }
}

// Step 1: Client requests pre-signed upload URL
POST /media/upload/presign
Response: {uploadUrl: "https://s3.../media/uuid.jpg?X-Amz-Signature=...", mediaId: "uuid"}

// Step 2: Client uploads directly to S3 (NOT through Chat Server)
PUT https://s3.amazonaws.com/wa-media/uuid.jpg
Content-Type: image/jpeg
Body: [encrypted image bytes]

// Step 3: Client sends message with media reference
WS: {type:"message", to:456, mediaId:"uuid", mediaType:"image", thumbnail:"base64..."}

// Step 4: Recipient downloads from CDN (not Chat Server)
GET https://cdn.wa.me/media/uuid.jpg  ← edge-cached, fast

// Benefits:
// Chat servers handle only ~200 byte WS frames (never MB of media)
// S3 + CDN handle bandwidth independently
// E2E encryption: client encrypts before upload, only recipient can decrypt