address comments

This commit is contained in:
chrislu
2025-08-26 22:32:42 -07:00
parent 7eb587f956
commit e6c50f9ba9
3 changed files with 218 additions and 21 deletions
+21 -15
View File
@@ -104,9 +104,9 @@ func TestS3IAMDistributedTests(t *testing.T) {
t.Run("distributed_concurrent_operations", func(t *testing.T) {
// Test concurrent operations across distributed instances
// Reduced concurrency (3x2=6 total ops) for CI stability
const numGoroutines = 3
const numOperationsPerGoroutine = 2
// Increased concurrency to effectively detect race conditions and system stability
const numGoroutines = 10 // More goroutines to stress test concurrency
const numOperationsPerGoroutine = 5 // More operations per goroutine for better coverage
var wg sync.WaitGroup
errors := make(chan error, numGoroutines*numOperationsPerGoroutine)
@@ -215,25 +215,31 @@ func TestS3IAMDistributedTests(t *testing.T) {
}
}
// STRICT CONCURRENCY TESTING: Use appropriate thresholds for the operation count
// For totalOperations=6, error thresholds need to be adjusted to avoid unreachable conditions
// RIGOROUS CONCURRENCY TESTING: Use strict thresholds for effective race condition detection
// For totalOperations=50, we can set much more realistic and strict error thresholds
// Serious errors (race conditions, deadlocks) should be zero for reliable testing
maxSeriousErrors := 0 // Allow 0 serious errors max - any serious error indicates system issues
maxSeriousErrors := 0 // Zero tolerance for serious errors - any indicates system issues
if len(seriousErrors) > maxSeriousErrors {
t.Errorf("❌ %d serious error(s) detected, indicating potential concurrency bugs. First error: %v",
len(seriousErrors), seriousErrors[0])
}
// For total errors, allow more flexibility with small operation counts
maxTotalErrors := 2 // Allow 2 total errors max (33.3%) for small test size
if len(errorList) > maxTotalErrors {
t.Errorf("❌ Too many total errors: %d (%.1f%%) - max allowed: %d. This suggests system instability under concurrent load.",
len(errorList), errorRate*100, maxTotalErrors)
// For total errors, use strict thresholds that can actually detect system instability
maxTotalErrorsStrict := 2 // Allow max 2 total errors (4% rate) - very strict
maxTotalErrorsRelaxed := 5 // Allow max 5 total errors (10% rate) - fallback for CI environments
if len(errorList) > maxTotalErrorsRelaxed {
t.Errorf("❌ Too many total errors: %d (%.1f%%) - exceeds relaxed threshold of %d (%.1f%%). System is unstable under concurrent load.",
len(errorList), errorRate*100, maxTotalErrorsRelaxed, float64(maxTotalErrorsRelaxed)/float64(totalOperations)*100)
} else if len(errorList) > maxTotalErrorsStrict {
t.Logf("⚠️ Concurrent operations completed with %d errors (%.1f%%) - exceeds strict threshold but within relaxed limits. Consider investigating.",
len(errorList), errorRate*100)
} else if len(errorList) > 0 {
t.Logf("⚠️ Concurrent operations completed with %d errors (%.1f%%) - within acceptable range for testing", len(errorList), errorRate*100)
t.Logf(" Concurrent operations completed with %d errors (%.1f%%) - within strict thresholds, excellent stability!",
len(errorList), errorRate*100)
} else {
t.Logf(" All concurrent operations completed successfully - excellent concurrency handling!")
t.Logf("🎉 All %d concurrent operations completed successfully - perfect concurrency handling!", totalOperations)
}
})
}