debug(kafka): add restart flake diagnostics

test/kafka/e2e/offset_management_test.go

@@ -39,7 +39,7 @@ func TestOffsetManagement(t *testing.T) {
 	})
 
 	t.Run("ConsumerGroupResumption", func(t *testing.T) {
-		testConsumerGroupResumption(t, addr, topic2, groupID+"2")
+		testConsumerGroupResumption(t, gateway, addr, topic2, groupID+"2")
 	})
 }
 
@@ -78,7 +78,7 @@ func testBasicOffsetCommitFetch(t *testing.T, addr, topic, groupID string) {
 	t.Logf("SUCCESS: Offset management test completed - consumed %d + %d messages", len(consumed1), len(consumed2))
 }
 
-func testConsumerGroupResumption(t *testing.T, addr, topic, groupID string) {
+func testConsumerGroupResumption(t *testing.T, gateway *testutil.GatewayTestServer, addr, topic, groupID string) {
 	client := testutil.NewKafkaGoClient(t, addr)
 	msgGen := testutil.NewMessageGenerator()
 
@@ -97,10 +97,18 @@ func testConsumerGroupResumption(t *testing.T, addr, topic, groupID string) {
 	for i, msg := range consumed1 {
 		t.Logf("  Message %d: offset=%d, partition=%d, value=%s", i, msg.Offset, msg.Partition, string(msg.Value))
 	}
+	gateway.LogConsumerGroupSnapshot(groupID)
 
 	// Simulate consumer restart by consuming remaining messages with same group ID
 	t.Logf("=== Phase 3: Second consumer (simulated restart) - consuming remaining messages with same group %s ===", groupID)
-	consumed2, err := client.ConsumeWithGroup(topic, groupID, 2)
+	consumed2, err := client.ConsumeWithGroupDebug(topic, groupID, 2, func(info testutil.ConsumeGroupRetryDebug) {
+		t.Logf("Consumer restart attempt %d/%d for group %s failed before receiving any messages: %v",
+			info.Attempt, info.MaxAttempts, info.GroupID, info.Err)
+		gateway.LogConsumerGroupSnapshot(groupID)
+	})
+	if err != nil {
+		gateway.LogConsumerGroupSnapshot(groupID)
+	}
 	testutil.AssertNoError(t, err, "Failed to consume after restart")
 	t.Logf("Second consumer consumed %d messages:", len(consumed2))
 	for i, msg := range consumed2 {

test/kafka/internal/testutil/clients.go

@@ -31,6 +31,15 @@ type SaramaClient struct {
 	t          *testing.T
 }
 
+type ConsumeGroupRetryDebug struct {
+	Attempt       int
+	MaxAttempts   int
+	Topic         string
+	GroupID       string
+	ExpectedCount int
+	Err           error
+}
+
 // NewKafkaGoClient creates a new kafka-go test client
 func NewKafkaGoClient(t *testing.T, brokerAddr string) *KafkaGoClient {
 	return &KafkaGoClient{
@@ -142,6 +151,14 @@ func (k *KafkaGoClient) ConsumeMessages(topicName string, expectedCount int) ([]
 // member's LeaveGroup / session cleanup and can surface as an i/o timeout on
 // the first FetchMessage.
 func (k *KafkaGoClient) ConsumeWithGroup(topicName, groupID string, expectedCount int) ([]kafka.Message, error) {
+	return k.consumeWithGroup(topicName, groupID, expectedCount, nil)
+}
+
+func (k *KafkaGoClient) ConsumeWithGroupDebug(topicName, groupID string, expectedCount int, onRetry func(ConsumeGroupRetryDebug)) ([]kafka.Message, error) {
+	return k.consumeWithGroup(topicName, groupID, expectedCount, onRetry)
+}
+
+func (k *KafkaGoClient) consumeWithGroup(topicName, groupID string, expectedCount int, onRetry func(ConsumeGroupRetryDebug)) ([]kafka.Message, error) {
 	k.t.Helper()
 
 	const maxJoinAttempts = 5
@@ -158,6 +175,16 @@ func (k *KafkaGoClient) ConsumeWithGroup(topicName, groupID string, expectedCoun
 		if progressed {
 			return messages, err
 		}
+		if onRetry != nil {
+			onRetry(ConsumeGroupRetryDebug{
+				Attempt:       attempt,
+				MaxAttempts:   maxJoinAttempts,
+				Topic:         topicName,
+				GroupID:       groupID,
+				ExpectedCount: expectedCount,
+				Err:           err,
+			})
+		}
 		if attempt == maxJoinAttempts {
 			break
 		}

test/kafka/internal/testutil/gateway.go

@@ -2,6 +2,7 @@ package testutil
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net"
 	"os"
@@ -111,6 +112,31 @@ func (g *GatewayTestServer) CleanupAndClose() {
 	}
 }
 
+// LogConsumerGroupSnapshot dumps the gateway's current view of a consumer group.
+func (g *GatewayTestServer) LogConsumerGroupSnapshot(groupID string) {
+	g.t.Helper()
+
+	handler := g.GetHandler()
+	if handler == nil {
+		g.t.Logf("Consumer group snapshot for %s unavailable: handler is nil", groupID)
+		return
+	}
+
+	snapshot := handler.DebugGroupSnapshot(groupID)
+	if snapshot == nil {
+		g.t.Logf("Consumer group snapshot for %s unavailable", groupID)
+		return
+	}
+
+	payload, err := json.MarshalIndent(snapshot, "", "  ")
+	if err != nil {
+		g.t.Logf("Consumer group snapshot for %s could not be marshaled: %v", groupID, err)
+		return
+	}
+
+	g.t.Logf("Consumer group snapshot for %s:\n%s", groupID, string(payload))
+}
+
 // SMQAvailabilityMode indicates whether SeaweedMQ is available for testing
 type SMQAvailabilityMode int