log read stateless

test/kafka/kafka-client-loadtest/config/loadtest.yaml

@@ -51,7 +51,7 @@ consumers:
   group_prefix: "loadtest-group"  # Consumer group prefix
   auto_offset_reset: "earliest"  # earliest, latest
   enable_auto_commit: true
-  auto_commit_interval_ms: 1000
+  auto_commit_interval_ms: 100  # Reduced from 1000ms to 100ms to minimize duplicate window
   session_timeout_ms: 30000
   heartbeat_interval_ms: 3000
   max_poll_records: 500

test/kafka/kafka-client-loadtest/docker-compose.yml

@@ -252,7 +252,7 @@ services:
       - TOPIC_COUNT=${TOPIC_COUNT:-5}
       - PARTITIONS_PER_TOPIC=${PARTITIONS_PER_TOPIC:-3}
       - TEST_MODE=${TEST_MODE:-comprehensive}
-      - SCHEMAS_ENABLED=true
+      - SCHEMAS_ENABLED=${SCHEMAS_ENABLED:-true}
       - VALUE_TYPE=${VALUE_TYPE:-avro}
     profiles:
       - loadtest

test/kafka/kafka-client-loadtest/internal/consumer/consumer.go

@@ -619,11 +619,19 @@ func (h *ConsumerGroupHandler) ConsumeClaim(session sarama.ConsumerGroupSession,
 
 				// Mark message as processed
 				session.MarkMessage(message, "")
+
+				// Commit offset immediately every 10 messages to reduce duplicate window
+				// This supplements auto-commit and ensures offsets are committed more frequently
+				if msgCount%10 == 0 {
+					session.Commit()
+				}
 			}
 
 		case <-session.Context().Done():
-			log.Printf("Consumer %d: Session context cancelled for %s[%d]",
+			log.Printf("Consumer %d: Session context cancelled for %s[%d], committing final offsets",
 				h.consumer.id, claim.Topic(), claim.Partition())
+			// Commit all remaining marked offsets before shutting down
+			session.Commit()
 			return nil
 		}
 	}

test/kafka/kafka-client-loadtest/test-no-schema.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Test without schema registry to isolate missing messages issue
+
+# Clean old data
+find test-results -name "*.jsonl" -delete 2>/dev/null || true
+
+# Run test without schemas
+TEST_MODE=comprehensive \
+TEST_DURATION=1m \
+PRODUCER_COUNT=2 \
+CONSUMER_COUNT=2 \
+MESSAGE_RATE=50 \
+MESSAGE_SIZE=512 \
+VALUE_TYPE=json \
+SCHEMAS_ENABLED=false \
+docker compose --profile loadtest up --abort-on-container-exit kafka-client-loadtest
+
+echo ""
+echo "═══════════════════════════════════════════════════════"
+echo "Analyzing results..."
+if [ -f test-results/produced.jsonl ] && [ -f test-results/consumed.jsonl ]; then
+    produced=$(wc -l < test-results/produced.jsonl)
+    consumed=$(wc -l < test-results/consumed.jsonl)
+    echo "Produced: $produced"
+    echo "Consumed: $consumed"
+    
+    # Check for missing messages
+    jq -r '"\(.topic)[\(.partition)]@\(.offset)"' test-results/produced.jsonl | sort > /tmp/produced.txt
+    jq -r '"\(.topic)[\(.partition)]@\(.offset)"' test-results/consumed.jsonl | sort > /tmp/consumed.txt
+    missing=$(comm -23 /tmp/produced.txt /tmp/consumed.txt | wc -l)
+    echo "Missing: $missing"
+    
+    if [ $missing -eq 0 ]; then
+        echo "✓ NO MISSING MESSAGES!"
+    else
+        echo "✗ Still have missing messages"
+        echo "Sample missing:"
+        comm -23 /tmp/produced.txt /tmp/consumed.txt | head -10
+    fi
+else
+    echo "✗ Result files not found"
+fi
+echo "═══════════════════════════════════════════════════════"