feat: implement Redis Streams support with stream checkpoints and update history

- Added Redis Streams operations to the message bus interface and implementation.
- Introduced StreamCheckpoint model to track last processed stream entry per document.
- Implemented UpsertStreamCheckpoint and GetStreamCheckpoint methods in the Postgres store.
- Created document_update_history table for storing update payloads for recovery and replay.
- Developed update persist worker to handle Redis Stream updates and persist them to Postgres.
- Enhanced Docker Compose configuration for Redis with persistence.
- Updated frontend API to support fetching document state with optional share token.
- Added connection stability monitoring in the Yjs document hook.

backend/internal/store/postgres.go

@@ -53,6 +53,15 @@ type Store interface {
 	GetDocumentVersion(ctx context.Context, versionID uuid.UUID) (*models.DocumentVersion, error)
 	GetLatestDocumentVersion(ctx context.Context, documentID uuid.UUID) (*models.DocumentVersion, error)
 
+	// Stream checkpoint operations
+	UpsertStreamCheckpoint(ctx context.Context, documentID uuid.UUID, streamID string, seq int64) error
+	GetStreamCheckpoint(ctx context.Context, documentID uuid.UUID) (*models.StreamCheckpoint, error)
+
+	// Update history (WAL) operations
+	InsertUpdateHistoryBatch(ctx context.Context, entries []UpdateHistoryEntry) error
+	ListUpdateHistoryAfterSeq(ctx context.Context, documentID uuid.UUID, afterSeq int64, limit int) ([]UpdateHistoryEntry, error)
+	DeleteUpdateHistoryUpToSeq(ctx context.Context, documentID uuid.UUID, maxSeq int64) error
+
 	Close() error
 }
 

backend/internal/store/stream_checkpoint.go

@@ -0,0 +1,46 @@
+package store
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/M1ngdaXie/realtime-collab/internal/models"
+	"github.com/google/uuid"
+)
+
+// UpsertStreamCheckpoint creates or updates the stream checkpoint for a document
+func (s *PostgresStore) UpsertStreamCheckpoint(ctx context.Context, documentID uuid.UUID, streamID string, seq int64) error {
+	query := `
+		INSERT INTO stream_checkpoints (document_id, last_stream_id, last_seq, updated_at)
+		VALUES ($1, $2, $3, NOW())
+		ON CONFLICT (document_id)
+		DO UPDATE SET last_stream_id = EXCLUDED.last_stream_id,
+		             last_seq = EXCLUDED.last_seq,
+		             updated_at = NOW()
+	`
+
+	if _, err := s.db.ExecContext(ctx, query, documentID, streamID, seq); err != nil {
+		return fmt.Errorf("failed to upsert stream checkpoint: %w", err)
+	}
+	return nil
+}
+
+// GetStreamCheckpoint retrieves the stream checkpoint for a document
+func (s *PostgresStore) GetStreamCheckpoint(ctx context.Context, documentID uuid.UUID) (*models.StreamCheckpoint, error) {
+	query := `
+		SELECT document_id, last_stream_id, last_seq, updated_at
+		FROM stream_checkpoints
+		WHERE document_id = $1
+	`
+
+	var checkpoint models.StreamCheckpoint
+	if err := s.db.QueryRowContext(ctx, query, documentID).Scan(
+		&checkpoint.DocumentID,
+		&checkpoint.LastStreamID,
+		&checkpoint.LastSeq,
+		&checkpoint.UpdatedAt,
+	); err != nil {
+		return nil, fmt.Errorf("failed to get stream checkpoint: %w", err)
+	}
+	return &checkpoint, nil
+}

backend/internal/store/testutil.go

@@ -71,10 +71,14 @@ func SetupTestDB(t *testing.T) (*PostgresStore, func()) {
 	// Run migrations
 	scriptsDir := filepath.Join("..", "..", "scripts")
 	migrations := []string{
-		"init.sql",
-		"001_add_users_and_sessions.sql",
-		"002_add_document_shares.sql",
-		"003_add_public_sharing.sql",
+		"000_extensions.sql",
+		"001_init_schema.sql",
+		"002_add_users_and_sessions.sql",
+		"003_add_document_shares.sql",
+		"004_add_public_sharing.sql",
+		"005_add_share_link_permission.sql",
+		"010_add_stream_checkpoints.sql",
+		"011_add_update_history.sql",
 	}
 
 	for _, migration := range migrations {
@@ -107,6 +111,8 @@ func SetupTestDB(t *testing.T) (*PostgresStore, func()) {
 func TruncateAllTables(ctx context.Context, store *PostgresStore) error {
 	tables := []string{
 		"document_updates",
+		"document_update_history",
+		"stream_checkpoints",
 		"document_shares",
 		"sessions",
 		"documents",

backend/internal/store/update_history.go

@@ -0,0 +1,115 @@
+package store
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"github.com/google/uuid"
+)
+
+// UpdateHistoryEntry represents a persisted update from Redis Streams
+// used for recovery and replay.
+type UpdateHistoryEntry struct {
+	DocumentID uuid.UUID
+	StreamID   string
+	Seq        int64
+	Payload    []byte
+	MsgType    string
+	ServerID   string
+	CreatedAt  time.Time
+}
+
+// InsertUpdateHistoryBatch inserts update history entries in a single batch.
+// Uses ON CONFLICT DO NOTHING to make inserts idempotent.
+func (s *PostgresStore) InsertUpdateHistoryBatch(ctx context.Context, entries []UpdateHistoryEntry) error {
+	if len(entries) == 0 {
+		return nil
+	}
+
+	var sb strings.Builder
+	sb.WriteString("INSERT INTO document_update_history (document_id, stream_id, seq, payload, msg_type, server_id, created_at) VALUES ")
+
+	args := make([]interface{}, 0, len(entries)*7)
+	for i, e := range entries {
+		if i > 0 {
+			sb.WriteString(",")
+		}
+		base := i*7 + 1
+		sb.WriteString(fmt.Sprintf("($%d,$%d,$%d,$%d,$%d,$%d,$%d)", base, base+1, base+2, base+3, base+4, base+5, base+6))
+		msgType := sanitizeTextForDB(e.MsgType)
+		serverID := sanitizeTextForDB(e.ServerID)
+		args = append(args, e.DocumentID, e.StreamID, e.Seq, e.Payload, nullIfEmpty(msgType), nullIfEmpty(serverID), e.CreatedAt)
+	}
+	// Idempotent insert
+	sb.WriteString(" ON CONFLICT (document_id, stream_id) DO NOTHING")
+
+	if _, err := s.db.ExecContext(ctx, sb.String(), args...); err != nil {
+		return fmt.Errorf("failed to insert update history batch: %w", err)
+	}
+	return nil
+}
+
+// ListUpdateHistoryAfterSeq returns updates with seq greater than afterSeq, ordered by seq.
+func (s *PostgresStore) ListUpdateHistoryAfterSeq(ctx context.Context, documentID uuid.UUID, afterSeq int64, limit int) ([]UpdateHistoryEntry, error) {
+	if limit <= 0 {
+		limit = 1000
+	}
+	query := `
+		SELECT document_id, stream_id, seq, payload, COALESCE(msg_type, ''), COALESCE(server_id, ''), created_at
+		FROM document_update_history
+		WHERE document_id = $1 AND seq > $2
+		ORDER BY seq ASC
+		LIMIT $3
+	`
+
+	rows, err := s.db.QueryContext(ctx, query, documentID, afterSeq, limit)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list update history: %w", err)
+	}
+	defer rows.Close()
+
+	var results []UpdateHistoryEntry
+	for rows.Next() {
+		var e UpdateHistoryEntry
+		if err := rows.Scan(&e.DocumentID, &e.StreamID, &e.Seq, &e.Payload, &e.MsgType, &e.ServerID, &e.CreatedAt); err != nil {
+			return nil, fmt.Errorf("failed to scan update history: %w", err)
+		}
+		results = append(results, e)
+	}
+	return results, nil
+}
+
+// DeleteUpdateHistoryUpToSeq deletes updates with seq <= maxSeq for a document.
+func (s *PostgresStore) DeleteUpdateHistoryUpToSeq(ctx context.Context, documentID uuid.UUID, maxSeq int64) error {
+	query := `
+		DELETE FROM document_update_history
+		WHERE document_id = $1 AND seq <= $2
+	`
+	if _, err := s.db.ExecContext(ctx, query, documentID, maxSeq); err != nil {
+		return fmt.Errorf("failed to delete update history: %w", err)
+	}
+	return nil
+}
+
+func nullIfEmpty(s string) interface{} {
+	if s == "" {
+		return nil
+	}
+	return s
+}
+
+func sanitizeTextForDB(s string) string {
+	if s == "" {
+		return ""
+	}
+	if strings.IndexByte(s, 0) >= 0 {
+		return ""
+	}
+	if !utf8.ValidString(s) {
+		return ""
+	}
+	return s
+}