package fuse_dlm import ( "fmt" "os" "path/filepath" "sync" "sync/atomic" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // TestDLMConcurrentWritersSameFile verifies that two mounts writing to the same // file concurrently produce valid (non-corrupted) data. With DLM enabled, the // writes are serialized — one blocks until the other completes. // // Note: cross-mount read consistency depends on FUSE kernel cache invalidation // and filer metadata subscription, which are asynchronous. This test verifies // write integrity, not instant read convergence. func TestDLMConcurrentWritersSameFile(t *testing.T) { if testing.Short() { t.Skip("skipping DLM integration test in short mode") } cluster := startDLMTestCluster(t) t.Cleanup(cluster.Stop) const iterations = 5 for iter := 0; iter < iterations; iter++ { fileName := fmt.Sprintf("concurrent_write_%d.txt", iter) payloadA := []byte(fmt.Sprintf("mount0-iteration-%d-payload-AAAA", iter)) payloadB := []byte(fmt.Sprintf("mount1-iteration-%d-payload-BBBB", iter)) var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done() err := os.WriteFile(filepath.Join(cluster.mountPoints[0], fileName), payloadA, 0644) assert.NoError(t, err, "mount0 write iteration %d", iter) }() go func() { defer wg.Done() err := os.WriteFile(filepath.Join(cluster.mountPoints[1], fileName), payloadB, 0644) assert.NoError(t, err, "mount1 write iteration %d", iter) }() wg.Wait() // Verify file is readable and contains one of the expected payloads // (read from mount0 — its own view is authoritative for write success). content, err := os.ReadFile(filepath.Join(cluster.mountPoints[0], fileName)) require.NoError(t, err, "read from mount0 iteration %d", iter) validPayload := string(content) == string(payloadA) || string(content) == string(payloadB) assert.True(t, validPayload, "iteration %d: content must be one of the expected payloads, got: %q", iter, content) } } // TestDLMRepeatedOpenWriteClose verifies that repeated open/write/close cycles // from both mounts all succeed without errors. func TestDLMRepeatedOpenWriteClose(t *testing.T) { if testing.Short() { t.Skip("skipping DLM integration test in short mode") } cluster := startDLMTestCluster(t) t.Cleanup(cluster.Stop) const cycles = 20 fileName := "repeated_write.txt" var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done() for i := 0; i < cycles; i++ { data := []byte(fmt.Sprintf("mount0-cycle-%d", i)) err := os.WriteFile(filepath.Join(cluster.mountPoints[0], fileName), data, 0644) assert.NoError(t, err, "mount0 cycle %d", i) } }() go func() { defer wg.Done() for i := 0; i < cycles; i++ { data := []byte(fmt.Sprintf("mount1-cycle-%d", i)) err := os.WriteFile(filepath.Join(cluster.mountPoints[1], fileName), data, 0644) assert.NoError(t, err, "mount1 cycle %d", i) } }() wg.Wait() // File must be readable from at least one mount content, err := os.ReadFile(filepath.Join(cluster.mountPoints[0], fileName)) require.NoError(t, err) assert.NotEmpty(t, content, "file must not be empty") } // TestDLMWriteBlocksSecondWriter verifies the core DLM guarantee: while one // mount has a file open for writing, another mount's write-open blocks until // the first mount closes the file. func TestDLMWriteBlocksSecondWriter(t *testing.T) { if testing.Short() { t.Skip("skipping DLM integration test in short mode") } cluster := startDLMTestCluster(t) t.Cleanup(cluster.Stop) fileName := "blocking_test.txt" path0 := filepath.Join(cluster.mountPoints[0], fileName) path1 := filepath.Join(cluster.mountPoints[1], fileName) // Mount 0 opens the file for writing and holds it open f, err := os.OpenFile(path0, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) require.NoError(t, err, "mount0 open") _, err = f.Write([]byte("mount0-holds-lock")) require.NoError(t, err, "mount0 write") // Mount 1 tries to write — should block (we use a goroutine with atomic flag) var mount1Completed atomic.Bool mount1Done := make(chan error, 1) go func() { err := os.WriteFile(path1, []byte("mount1-waited"), 0644) mount1Completed.Store(true) mount1Done <- err }() // Give mount 1 a moment — it should NOT complete while mount 0 holds the file open time.Sleep(3 * time.Second) require.False(t, mount1Completed.Load(), "mount1 write must not complete while mount0 holds the file open") t.Log("mount1 write is blocked as expected while mount0 holds the file") // Mount 0 closes the file — this releases the DLM lock require.NoError(t, f.Close(), "mount0 close") // Mount 1 should now complete select { case err := <-mount1Done: assert.NoError(t, err, "mount1 write after mount0 close") case <-time.After(30 * time.Second): t.Fatal("mount1 write did not complete within 30s after mount0 closed") } } // TestDLMRenameWhileWriteOpen verifies that a rename is coordinated with DLM: // while mount0 has a file open for writing (re-opened after creation), // mount1 cannot rename it until mount0 closes the file. func TestDLMRenameWhileWriteOpen(t *testing.T) { if testing.Short() { t.Skip("skipping DLM integration test in short mode") } cluster := startDLMTestCluster(t) t.Cleanup(cluster.Stop) origName := "rename_source.txt" newName := "rename_dest.txt" // Create and close the file first so it's flushed to the filer and // visible on both mounts. require.NoError(t, os.WriteFile( filepath.Join(cluster.mountPoints[0], origName), []byte("initial-content"), 0644)) time.Sleep(2 * time.Second) // metadata propagation // Verify mount1 can see the file _, err := os.Stat(filepath.Join(cluster.mountPoints[1], origName)) require.NoError(t, err, "mount1 should see the file") // Mount 0 re-opens the file for writing and holds it open f, err := os.OpenFile( filepath.Join(cluster.mountPoints[0], origName), os.O_WRONLY|os.O_TRUNC, 0644) require.NoError(t, err, "mount0 reopen") _, err = f.Write([]byte("data-while-holding-lock")) require.NoError(t, err, "mount0 write") // Mount 1 tries to rename — should block because mount0 holds the // DLM lock on the old path var renameCompleted atomic.Bool renameDone := make(chan error, 1) go func() { err := os.Rename( filepath.Join(cluster.mountPoints[1], origName), filepath.Join(cluster.mountPoints[1], newName)) renameCompleted.Store(true) renameDone <- err }() // Rename must NOT complete while mount0 holds the file open time.Sleep(3 * time.Second) require.False(t, renameCompleted.Load(), "rename must not complete while mount0 holds the file open") t.Log("rename is blocked as expected while mount0 holds the file") // Mount 0 closes → releases DLM lock → rename should proceed require.NoError(t, f.Close(), "mount0 close") select { case err := <-renameDone: assert.NoError(t, err, "rename after mount0 close") case <-time.After(30 * time.Second): t.Fatal("rename did not complete within 30s after mount0 closed") } } // Note: Same-mount rename while a file is open for writing is not tested here // because macOS FUSE serializes operations on the same inode, causing a // kernel-level deadlock between the Rename handler's internal flush and the // pending Close. Same-mount coordination is already handled by the per-mount // fhLockTable and FUSE kernel serialization, so DLM is not needed for it. // TestDLMConcurrentRenames verifies that two concurrent renames of the same // file from different mounts don't corrupt metadata. DLM locks on both old // and new paths ensure renames are serialized. func TestDLMConcurrentRenames(t *testing.T) { if testing.Short() { t.Skip("skipping DLM integration test in short mode") } cluster := startDLMTestCluster(t) t.Cleanup(cluster.Stop) // Create a file first origPath := filepath.Join(cluster.mountPoints[0], "rename_race.txt") require.NoError(t, os.WriteFile(origPath, []byte("original-content"), 0644)) time.Sleep(1 * time.Second) // propagation // Both mounts try to rename the same file concurrently var wg sync.WaitGroup var errA, errB error wg.Add(2) go func() { defer wg.Done() errA = os.Rename( filepath.Join(cluster.mountPoints[0], "rename_race.txt"), filepath.Join(cluster.mountPoints[0], "renamed_by_mount0.txt"), ) }() go func() { defer wg.Done() errB = os.Rename( filepath.Join(cluster.mountPoints[1], "rename_race.txt"), filepath.Join(cluster.mountPoints[1], "renamed_by_mount1.txt"), ) }() wg.Wait() // At least one rename should succeed; the other may fail with ENOENT // since the source was already moved. succeeded := 0 if errA == nil { succeeded++ t.Logf("mount0 rename succeeded") } else { t.Logf("mount0 rename failed: %v", errA) } if errB == nil { succeeded++ t.Logf("mount1 rename succeeded") } else { t.Logf("mount1 rename failed: %v", errB) } assert.GreaterOrEqual(t, succeeded, 1, "at least one rename must succeed") }