diff --git a/weed/storage/volume_checking.go b/weed/storage/volume_checking.go index 52cff0e740..1bb1157346 100644 --- a/weed/storage/volume_checking.go +++ b/weed/storage/volume_checking.go @@ -1,6 +1,7 @@ package storage import ( + "errors" "fmt" "io" "os" @@ -10,6 +11,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage/backend" "github.com/seaweedfs/seaweedfs/weed/storage/idx" "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/needle_map" "github.com/seaweedfs/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/storage/types" "github.com/seaweedfs/seaweedfs/weed/util" @@ -166,6 +168,32 @@ func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) (lastAppendAtNs uin return 0, nil } + // A crash between appending a needle to the .dat and appending its entry to + // the .idx leaves a .dat tail no index entry covers. Re-index the complete + // needles in it and cut an incomplete remainder before the check below, + // which would otherwise load the volume read-only. Only a volume that is + // about to load writable is repaired; a read-only one is left as it is. + // A repair that fails for any other reason than an unexplained tail (a + // read, write, truncate or sync error) fails the check, so the volume + // loads read-only instead of writable on a repair that may not be on disk. + if !v.noWriteOrDelete && !v.noWriteCanDelete { + reindexed, _, err := repairUnindexedDatTail(v, indexFile, indexSize, tailEntryPos) + if err != nil { + if !errors.Is(err, errTailNotTorn) { + return 0, fmt.Errorf("CheckVolumeDataIntegrity %s: repair unindexed .dat tail: %v", indexFile.Name(), err) + } + glog.Warningf("volume %d: unindexed .dat tail not repaired: %v", v.Id, err) + } + if reindexed > 0 { + if indexSize, err = verifyIndexFileIntegrity(indexFile); err != nil { + return 0, fmt.Errorf("verifyIndexFileIntegrity %s after re-indexing failed: %v", indexFile.Name(), err) + } + if tailEntryPos, err = findDatTailEntryOffset(v, indexFile, indexSize); err != nil { + return 0, fmt.Errorf("CheckVolumeDataIntegrity %s: %v", indexFile.Name(), err) + } + } + } + if lastAppendAtNs, err = doCheckAndFixVolumeData(v, indexFile, tailEntryPos); err != nil { return lastAppendAtNs, fmt.Errorf("CheckVolumeDataIntegrity %s failed: %v", indexFile.Name(), err) } @@ -219,6 +247,247 @@ func needleDiskEnd(offset types.Offset, size types.Size, version needle.Version) return offset.ToActualOffset() + needle.GetActualSize(types.Size(size.Raw()), version) } +// errTailNotTorn marks a .dat tail that is not the result of an interrupted +// append; the volume keeps loading read-only. +var errTailNotTorn = errors.New("not a torn tail") + +// repairUnindexedDatTail re-indexes the complete needles a crash left past the +// last indexed needle of a local version 3 volume, cuts an incomplete +// remainder, and returns how many needles it indexed and how many bytes it cut. +// +// A write appends the needle to the .dat, then its entry to the .idx, then +// answers the client. A process killed between the first two steps leaves a +// needle, or part of one, past the end of the needle at the highest indexed +// offset. After a power loss the order on disk is whatever the page cache +// flushed: a complete needle can survive whose .idx entry did not, although +// the client was answered. So a complete needle is never dropped: every +// complete needle in the tail (its header, body, checksum, timestamp and +// padding inside the file, its CRC valid, its append timestamp later than the +// needle before it) gets the .idx entry the volume server would have written +// (a size 0 record as resolveEmptyTailRecords reads it), in .dat order, and +// the .idx is synced. Only what follows the last complete +// needle and is itself incomplete is cut: fewer bytes than a header, a record +// running past the end of the file, or a last record whose bytes are damaged +// (its CRC fails or its body does not parse); the .dat is synced after the +// cut. A record that cannot be read is not cut: a read error is returned and +// nothing is changed. +// +// Nothing is changed when the tail is anything else: the .idx entry at the +// highest offset does not describe the needle there (id, size, CRC), a record +// inside the tail is damaged while more data follows it, a header carries id 0 +// or a negative size, append timestamps go backwards, the .dat is beyond what +// 4-byte offsets can address, the needle version is not 3, or the .idx size is +// not a whole number of entries. The integrity check that follows then marks +// the volume read-only exactly as before. +func repairUnindexedDatTail(v *Volume, indexFile *os.File, indexSize int64, tailEntryPos int64) (int, int64, error) { + version := v.Version() + if version != needle.Version3 || v.DataBackend == nil { + return 0, 0, nil + } + datSize, _, err := v.DataBackend.GetStat() + if err != nil { + return 0, 0, fmt.Errorf("stat %s: %v", v.FileName(".dat"), err) + } + if datSize <= int64(super_block.SuperBlockSize) || datSize > int64(types.MaxPossibleVolumeSize) { + // past MaxPossibleVolumeSize the .idx offsets wrap: an entry's offset + // does not name its needle's position on its own + return 0, 0, nil + } + entry, err := readIndexEntryAtOffset(indexFile, tailEntryPos) + if err != nil { + return 0, 0, fmt.Errorf("read index entry at %d: %v", tailEntryPos, err) + } + key, offset, size := idx.IdxFileEntry(entry) + if offset.IsZero() { + return 0, 0, nil + } + start := needleDiskEnd(offset, size, version) + if start >= datSize { + // the .dat ends at the last needle (clean), or the index references + // bytes the .dat does not have: not a torn tail, not ours to fix + return 0, 0, nil + } + // The backend rounds its size up to the needle alignment; the walk needs + // the bytes that are really there. + fileSize := datSize + if st, statErr := os.Stat(v.DataBackend.Name()); statErr == nil { + fileSize = st.Size() + } + + // Anchor: the needle at the highest indexed offset must be the one its + // entry describes, whole and with a valid CRC. + onDiskSize := size + if size.IsDeleted() { + onDiskSize = 0 + } + anchor, err := readTailNeedle(v.DataBackend, version, offset.ToActualOffset(), onDiskSize) + if err != nil { + return 0, 0, fmt.Errorf("last indexed needle %v at %d: %v: %w", key, offset.ToActualOffset(), err, errTailNotTorn) + } + if anchor.Id != key { + return 0, 0, fmt.Errorf("needle at %d is id %v, its index entry says id %v: %w", offset.ToActualOffset(), anchor.Id, key, errTailNotTorn) + } + + entries, cutAt, err := walkUnindexedDatTail(v.DataBackend, version, start, fileSize, anchor.AppendAtNs) + if err != nil { + return 0, 0, err + } + reindexed := len(entries) / types.NeedleMapEntrySize + if reindexed > 0 { + if err := resolveEmptyTailRecords(indexFile, entries); err != nil { + return 0, 0, err + } + if err := appendIndexEntries(indexFile, entries, indexSize); err != nil { + return 0, 0, err + } + } + var cut int64 + if cutAt < fileSize { + if err := v.DataBackend.Truncate(cutAt); err != nil { + return reindexed, 0, fmt.Errorf("truncate %s to %d: %v", v.FileName(".dat"), cutAt, err) + } + if err := v.DataBackend.Sync(); err != nil { + return reindexed, fileSize - cutAt, fmt.Errorf("sync %s after truncating to %d: %v", v.FileName(".dat"), cutAt, err) + } + cut = fileSize - cutAt + } + glog.Warningf("volume %d: repaired the unindexed tail of %s: re-indexed %d needle(s), cut %d bytes (%d -> %d bytes)", + v.Id, v.FileName(".dat"), reindexed, cut, fileSize, fileSize-cut) + return reindexed, cut, nil +} + +// indexAppender is the part of the .idx file the repair writes through. +type indexAppender interface { + io.WriterAt + Truncate(size int64) error + Sync() error + Name() string +} + +// appendIndexEntries writes entries at the end (at) of the .idx and syncs it. +// A failed write can leave part of the entries behind, which would make the +// next load reject the .idx as a partial entry; they are removed again. +func appendIndexEntries(indexFile indexAppender, entries []byte, at int64) error { + if _, err := indexFile.WriteAt(entries, at); err != nil { + if truncErr := indexFile.Truncate(at); truncErr != nil { + return fmt.Errorf("append %d bytes to %s: %v; truncate back to %d: %v", len(entries), indexFile.Name(), err, at, truncErr) + } + return fmt.Errorf("append %d bytes to %s: %v", len(entries), indexFile.Name(), err) + } + if err := indexFile.Sync(); err != nil { + return fmt.Errorf("sync %s: %v", indexFile.Name(), err) + } + return nil +} + +// readTailNeedle reads and verifies the whole record of a needle of the given +// on-disk size at offset: header, body, CRC and timestamp. +func readTailNeedle(datFile backend.BackendStorageFile, version needle.Version, at int64, size types.Size) (*needle.Needle, error) { + blob, err := needle.ReadNeedleBlob(datFile, at, size, version) + if err != nil { + return nil, err + } + n := new(needle.Needle) + if err := n.ReadBytes(blob, at, size, version); err != nil { + return nil, err + } + if size > 0 && len(n.Data) == 0 { + // A needle with no data is written with size 0, so a record with a + // body whose data size reads 0 is damaged. The CRC covers only the + // data, and ReadBytes does not check it for empty data. + return nil, fmt.Errorf("needle %v at %d has size %d but no data: %w", n.Id, at, size, needle.ErrorCorrupted) + } + return n, nil +} + +// resolveEmptyTailRecords turns the size 0 entries among the re-indexed ones +// into what the volume server indexed them as. A delete and a write with no +// data both append a size 0 record; a delete is indexed as a tombstone, an +// empty write with size 0. A delete is only appended for a needle that is +// live, so a size 0 record of a needle that is not live (absent or deleted in +// the .idx and in the tail before it) is an empty write and keeps size 0. One +// of a live needle is indexed as a tombstone, as weed fix and the backup's +// index regeneration read a size 0 record. +func resolveEmptyTailRecords(indexFile io.ReaderAt, entries []byte) error { + live := make(map[types.NeedleId]bool) + for i := 0; i < len(entries); i += types.NeedleMapEntrySize { + if key, _, size := idx.IdxFileEntry(entries[i : i+types.NeedleMapEntrySize]); size == 0 { + live[key] = false + } + } + if len(live) == 0 { + return nil + } + if err := idx.WalkIndexFile(indexFile, 0, func(key types.NeedleId, offset types.Offset, size types.Size) error { + if _, ok := live[key]; ok { + live[key] = !offset.IsZero() && !size.IsDeleted() + } + return nil + }); err != nil { + return fmt.Errorf("scan index: %v", err) + } + for i := 0; i < len(entries); i += types.NeedleMapEntrySize { + key, offset, size := idx.IdxFileEntry(entries[i : i+types.NeedleMapEntrySize]) + wasLive, tracked := live[key] + if !tracked { + continue + } + if size == 0 && wasLive { + copy(entries[i:], needle_map.ToBytes(key, offset, types.TombstoneFileSize)) + live[key] = false + continue + } + live[key] = true + } + return nil +} + +// walkUnindexedDatTail walks the needles from start to fileSize and returns +// the .idx entries of the complete ones, and the offset from which the rest is +// an incomplete last record (fileSize when there is none). It fails with +// errTailNotTorn when the tail is damaged in a way an interrupted append does +// not explain. +func walkUnindexedDatTail(datFile backend.BackendStorageFile, version needle.Version, start, fileSize int64, lastAppendAtNs uint64) ([]byte, int64, error) { + var entries []byte + at := start + for at < fileSize { + if fileSize-at < types.NeedleHeaderSize { + return entries, at, nil // a partial header + } + header, _, _, err := needle.ReadNeedleHeader(datFile, version, at) + if err != nil || header == nil { + return nil, 0, fmt.Errorf("read needle header at %d: %v", at, err) + } + if header.Id == 0 || header.Size < 0 { + return nil, 0, fmt.Errorf("record at %d has id %v size %d: %w", at, header.Id, header.Size, errTailNotTorn) + } + end := at + needle.GetActualSize(header.Size, version) + if end > fileSize { + return entries, at, nil // the last record runs past the end of the file + } + n, err := readTailNeedle(datFile, version, at, header.Size) + if err != nil { + if !errors.Is(err, needle.ErrorCorrupted) { + // a read error says nothing about the bytes: leave them + return nil, 0, fmt.Errorf("read record %v at %d: %w", header.Id, at, err) + } + if end == fileSize { + return entries, at, nil // the last record is damaged: it never completed + } + return nil, 0, fmt.Errorf("record %v at %d is damaged and %d bytes follow it: %v: %w", header.Id, at, fileSize-end, err, errTailNotTorn) + } + if n.AppendAtNs <= lastAppendAtNs { + return nil, 0, fmt.Errorf("record %v at %d was appended at %d, not after %d: %w", n.Id, at, n.AppendAtNs, lastAppendAtNs, errTailNotTorn) + } + lastAppendAtNs = n.AppendAtNs + // a size 0 record is a delete or an empty write: resolveEmptyTailRecords + // tells them apart + entries = append(entries, needle_map.ToBytes(n.Id, types.ToOffset(at), header.Size)...) + at = end + } + return entries, fileSize, nil +} + func doCheckAndFixVolumeData(v *Volume, indexFile *os.File, indexOffset int64) (lastAppendAtNs uint64, err error) { var lastIdxEntry []byte if lastIdxEntry, err = readIndexEntryAtOffset(indexFile, indexOffset); err != nil { diff --git a/weed/storage/volume_checking_tail_repair_test.go b/weed/storage/volume_checking_tail_repair_test.go new file mode 100644 index 0000000000..19a5f6a4ea --- /dev/null +++ b/weed/storage/volume_checking_tail_repair_test.go @@ -0,0 +1,561 @@ +package storage + +import ( + "bytes" + "errors" + "os" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/storage/backend" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "github.com/seaweedfs/seaweedfs/weed/storage/types" +) + +// A torn tail is what a crash between the two appends of a write leaves: the +// needle is in the .dat, its entry never reached the .idx. After a process +// crash the client was never answered; after a power loss it may have been. +// These tests write needles (and deletes) normally, drop the last .idx +// entries, cut or damage the .dat as a crash would, and reload the volume +// through the real load path. + +type tornOp struct { + id uint64 + del bool + empty bool // a write with no data, which appends a size 0 record like a delete +} + +type tornFixture struct { + dir string + datName string + idxName string + starts []int64 // .dat offset of each op's record + end int64 // .dat size after every op + written map[uint64]*needle.Needle +} + +func newTornTailTestVolume(t *testing.T, dir string) *Volume { + t.Helper() + v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0) + if err != nil { + t.Fatalf("load volume: %v", err) + } + return v +} + +// buildTornFixture runs ops against a fresh volume (each op appends one .dat +// record and one .idx entry), closes it and drops the last unindexed entries. +func buildTornFixture(t *testing.T, ops []tornOp, unindexed int) *tornFixture { + t.Helper() + f := &tornFixture{dir: t.TempDir(), written: map[uint64]*needle.Needle{}} + v := newTornTailTestVolume(t, f.dir) + for _, op := range ops { + at, _, err := v.DataBackend.GetStat() + if err != nil { + t.Fatal(err) + } + f.starts = append(f.starts, at) + if op.del { + if _, err := v.doDeleteRequest(newEmptyNeedle(op.id)); err != nil { + t.Fatalf("delete needle %d: %v", op.id, err) + } + continue + } + n := newRandomNeedle(op.id) + if op.empty { + n = newEmptyNeedle(op.id) + } + if _, _, _, err := v.writeNeedle2(n, true, false, false); err != nil { + t.Fatalf("write needle %d: %v", op.id, err) + } + f.written[op.id] = n + } + f.end, _, _ = v.DataBackend.GetStat() + f.datName, f.idxName = v.FileName(".dat"), v.FileName(".idx") + v.Close() + if unindexed > 0 { + if err := os.Truncate(f.idxName, fileSize(t, f.idxName)-int64(unindexed)*types.NeedleMapEntrySize); err != nil { + t.Fatalf("drop %d .idx entries: %v", unindexed, err) + } + } + return f +} + +// midRecord is an offset inside the body of op i's record: past its header, +// short of its end. A fixed offset is not: a needle with a few bytes of data +// is a 40-byte record, which a cut at +40 leaves whole. +func (f *tornFixture) midRecord(i int) int64 { + end := f.end + if i+1 < len(f.starts) { + end = f.starts[i+1] + } + return midRecordBody(f.starts[i], end) +} + +func midRecordBody(start, end int64) int64 { + return start + types.NeedleHeaderSize + (end-start-types.NeedleHeaderSize)/2 +} + +func (f *tornFixture) truncateDat(t *testing.T, size int64) { + t.Helper() + if err := os.Truncate(f.datName, size); err != nil { + t.Fatal(err) + } +} + +// flipDataByte damages the data of the record at the given offset, so its +// CRC no longer matches. +func (f *tornFixture) flipDataByte(t *testing.T, at int64) { + t.Helper() + file, err := os.OpenFile(f.datName, os.O_RDWR, 0644) + if err != nil { + t.Fatal(err) + } + defer file.Close() + b := make([]byte, 1) + pos := at + types.NeedleHeaderSize + 4 // past the 4-byte DataSize field + if _, err := file.ReadAt(b, pos); err != nil { + t.Fatal(err) + } + b[0] ^= 0xff + if _, err := file.WriteAt(b, pos); err != nil { + t.Fatal(err) + } +} + +// zeroDataSize zeroes the DataSize field and the first data byte (read as +// the flags once DataSize is 0) of the record at the given offset: the header +// still claims a body, the body parses as an empty needle. +func (f *tornFixture) zeroDataSize(t *testing.T, at int64) { + t.Helper() + file, err := os.OpenFile(f.datName, os.O_RDWR, 0644) + if err != nil { + t.Fatal(err) + } + defer file.Close() + if _, err := file.WriteAt(make([]byte, 5), at+types.NeedleHeaderSize); err != nil { + t.Fatal(err) + } +} + +func assertReadable(t *testing.T, v *Volume, want *needle.Needle) { + t.Helper() + got := &needle.Needle{Id: want.Id} + if _, err := v.readNeedle(got, nil, nil); err != nil { + t.Fatalf("read needle %d: %v", want.Id, err) + } + if !bytes.Equal(got.Data, want.Data) { + t.Fatalf("needle %d reads back different data", want.Id) + } +} + +func assertNotReadable(t *testing.T, v *Volume, id uint64) { + t.Helper() + if _, err := v.readNeedle(&needle.Needle{Id: types.Uint64ToNeedleId(id)}, nil, nil); err == nil { + t.Fatalf("needle %d reads back; it should be deleted or absent", id) + } +} + +func writes(ids ...uint64) []tornOp { + ops := make([]tornOp, len(ids)) + for i, id := range ids { + ops[i] = tornOp{id: id} + } + return ops +} + +func TestVolumeLoadRepairsUnindexedDatTail(t *testing.T) { + cases := []struct { + name string + ops []tornOp + unindexed int + damage func(t *testing.T, f *tornFixture) + wantSize func(f *tornFixture) int64 // .dat size after the repair + reindexed int + readable []uint64 + absent []uint64 + }{ + {name: "one complete unindexed needle is re-indexed", + ops: writes(1, 2, 3, 4), unindexed: 1, + wantSize: func(f *tornFixture) int64 { return f.end }, reindexed: 1, + readable: []uint64{1, 2, 3, 4}}, + {name: "two complete unindexed needles are re-indexed", + ops: writes(1, 2, 3, 4, 5), unindexed: 2, + wantSize: func(f *tornFixture) int64 { return f.end }, reindexed: 2, + readable: []uint64{1, 2, 3, 4, 5}}, + {name: "an unindexed delete is re-indexed as a tombstone", + ops: append(writes(1, 2, 3), tornOp{id: 2, del: true}), unindexed: 1, + wantSize: func(f *tornFixture) int64 { return f.end }, reindexed: 1, + readable: []uint64{1, 3}, absent: []uint64{2}}, + {name: "a partial header is cut", + ops: writes(1, 2, 3, 4), unindexed: 1, + damage: func(t *testing.T, f *tornFixture) { f.truncateDat(t, f.starts[3]+7) }, + wantSize: func(f *tornFixture) int64 { return f.starts[3] }, + readable: []uint64{1, 2, 3}, absent: []uint64{4}}, + {name: "a partial body is cut", + ops: writes(1, 2, 3, 4), unindexed: 1, + damage: func(t *testing.T, f *tornFixture) { f.truncateDat(t, f.midRecord(3)) }, + wantSize: func(f *tornFixture) int64 { return f.starts[3] }, + readable: []uint64{1, 2, 3}, absent: []uint64{4}}, + {name: "a complete needle is re-indexed and the partial one after it cut", + ops: writes(1, 2, 3, 4, 5), unindexed: 2, + damage: func(t *testing.T, f *tornFixture) { f.truncateDat(t, f.midRecord(4)) }, + wantSize: func(f *tornFixture) int64 { return f.starts[4] }, reindexed: 1, + readable: []uint64{1, 2, 3, 4}, absent: []uint64{5}}, + {name: "a last needle with a bad CRC is cut", + ops: writes(1, 2, 3, 4), unindexed: 1, + damage: func(t *testing.T, f *tornFixture) { f.flipDataByte(t, f.starts[3]) }, + wantSize: func(f *tornFixture) int64 { return f.starts[3] }, + readable: []uint64{1, 2, 3}, absent: []uint64{4}}, + {name: "a last needle whose data size field reads 0 is cut", + ops: writes(1, 2, 3, 4), unindexed: 1, + damage: func(t *testing.T, f *tornFixture) { f.zeroDataSize(t, f.starts[3]) }, + wantSize: func(f *tornFixture) int64 { return f.starts[3] }, + readable: []uint64{1, 2, 3}, absent: []uint64{4}}, + {name: "an unindexed empty write of a new id stays an empty needle", + ops: append(writes(1, 2), tornOp{id: 3, empty: true}), unindexed: 1, + wantSize: func(f *tornFixture) int64 { return f.end }, reindexed: 1, + readable: []uint64{1, 2, 3}}, + {name: "an unindexed empty write after a delete of the same id stays an empty needle", + ops: append(append(writes(1, 2), tornOp{id: 2, del: true}), tornOp{id: 2, empty: true}), unindexed: 1, + wantSize: func(f *tornFixture) int64 { return f.end }, reindexed: 1, + readable: []uint64{1, 2}}, + {name: "an unindexed write and delete of the same id are re-indexed in order", + ops: append(writes(1, 2, 3), tornOp{id: 3, del: true}), unindexed: 2, + wantSize: func(f *tornFixture) int64 { return f.end }, reindexed: 2, + readable: []uint64{1, 2}, absent: []uint64{3}}, + {name: "an unindexed empty write and its delete are re-indexed in order", + ops: append(writes(1, 2), tornOp{id: 3, empty: true}, tornOp{id: 3, del: true}), unindexed: 2, + wantSize: func(f *tornFixture) int64 { return f.end }, reindexed: 2, + readable: []uint64{1, 2}, absent: []uint64{3}}, + {name: "a partial needle after an indexed tombstone is cut", + ops: append(append(writes(1, 2, 3), tornOp{id: 2, del: true}), tornOp{id: 4}), unindexed: 1, + damage: func(t *testing.T, f *tornFixture) { f.truncateDat(t, f.midRecord(4)) }, + wantSize: func(f *tornFixture) int64 { return f.starts[4] }, + readable: []uint64{1, 3}, absent: []uint64{2, 4}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := buildTornFixture(t, tc.ops, tc.unindexed) + if tc.damage != nil { + tc.damage(t, f) + } + idxBefore := fileSize(t, f.idxName) + + v := newTornTailTestVolume(t, f.dir) + defer v.Close() + if v.noWriteOrDelete || v.noWriteCanDelete { + t.Fatal("volume with an unindexed .dat tail loaded read-only") + } + if got, want := fileSize(t, f.datName), tc.wantSize(f); got != want { + t.Fatalf(".dat is %d bytes after load, want %d", got, want) + } + if got, want := fileSize(t, f.idxName), idxBefore+int64(tc.reindexed)*types.NeedleMapEntrySize; got != want { + t.Fatalf(".idx is %d bytes after load, want %d (%d entries re-indexed)", got, want, tc.reindexed) + } + for _, id := range tc.readable { + assertReadable(t, v, f.written[id]) + } + for _, id := range tc.absent { + assertNotReadable(t, v, id) + } + extra := newRandomNeedle(100) + if _, _, _, err := v.writeNeedle2(extra, true, false, false); err != nil { + t.Fatalf("write after repair: %v", err) + } + assertReadable(t, v, extra) + v.Close() + + // the repaired volume reloads clean, and keeps what it indexed + v2 := newTornTailTestVolume(t, f.dir) + defer v2.Close() + if v2.noWriteOrDelete { + t.Fatal("repaired volume loaded read-only on the next load") + } + for _, id := range tc.readable { + assertReadable(t, v2, f.written[id]) + } + assertReadable(t, v2, extra) + }) + } +} + +// Every tail an interrupted append does not explain is left exactly as it is, +// and the volume loads read-only as before. +func TestVolumeLoadKeepsUnexplainedTailReadOnly(t *testing.T) { + cases := []struct { + name string + ops []tornOp + unindexed int + damage func(t *testing.T, f *tornFixture) + }{ + {name: "the last indexed needle's id differs from its entry", + ops: writes(1, 2, 3, 4), unindexed: 1, + damage: func(t *testing.T, f *tornFixture) { + file, err := os.OpenFile(f.datName, os.O_RDWR, 0644) + if err != nil { + t.Fatal(err) + } + defer file.Close() + if _, err := file.WriteAt([]byte{0xff}, f.starts[2]+types.CookieSize+types.NeedleIdSize-1); err != nil { + t.Fatal(err) + } + }}, + {name: "the last indexed needle fails its CRC", + ops: writes(1, 2, 3, 4), unindexed: 1, + damage: func(t *testing.T, f *tornFixture) { f.flipDataByte(t, f.starts[2]) }}, + {name: "a zeroed data size field followed by more data", + ops: writes(1, 2, 3, 4, 5), unindexed: 2, + damage: func(t *testing.T, f *tornFixture) { f.zeroDataSize(t, f.starts[3]) }}, + {name: "a bad CRC followed by more data", + ops: writes(1, 2, 3, 4, 5), unindexed: 2, + damage: func(t *testing.T, f *tornFixture) { f.flipDataByte(t, f.starts[3]) }}, + {name: "zero bytes past the last needle (id 0)", + ops: writes(1, 2, 3), + damage: func(t *testing.T, f *tornFixture) { + file, err := os.OpenFile(f.datName, os.O_RDWR|os.O_APPEND, 0644) + if err != nil { + t.Fatal(err) + } + defer file.Close() + if _, err := file.Write(make([]byte, 64)); err != nil { + t.Fatal(err) + } + }}, + {name: "the index ends past the data", + ops: writes(1, 2, 3), + damage: func(t *testing.T, f *tornFixture) { f.truncateDat(t, f.end-10) }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := buildTornFixture(t, tc.ops, tc.unindexed) + tc.damage(t, f) + datBefore, idxBefore := fileSize(t, f.datName), fileSize(t, f.idxName) + + v := newTornTailTestVolume(t, f.dir) + defer v.Close() + if !v.noWriteOrDelete { + t.Fatal("volume with an unexplained tail loaded writable") + } + if got := fileSize(t, f.datName); got != datBefore { + t.Fatalf(".dat changed from %d to %d bytes", datBefore, got) + } + if got := fileSize(t, f.idxName); got != idxBefore { + t.Fatalf(".idx changed from %d to %d bytes", idxBefore, got) + } + }) + } +} + +func TestVolumeLoadLeavesCleanVolumeUntouched(t *testing.T) { + f := buildTornFixture(t, append(writes(1, 2, 3), tornOp{id: 2, del: true}), 0) + datBefore, idxBefore := fileSize(t, f.datName), fileSize(t, f.idxName) + + v := newTornTailTestVolume(t, f.dir) + defer v.Close() + if v.noWriteOrDelete || v.noWriteCanDelete { + t.Fatal("clean volume loaded read-only") + } + if got := fileSize(t, f.datName); got != datBefore { + t.Fatalf("clean .dat changed from %d to %d bytes", datBefore, got) + } + if got := fileSize(t, f.idxName); got != idxBefore { + t.Fatalf("clean .idx changed from %d to %d bytes", idxBefore, got) + } + assertReadable(t, v, f.written[1]) + assertReadable(t, v, f.written[3]) + assertNotReadable(t, v, 2) +} + +// A volume persisted read-only is not repaired: the tail stays for whoever +// marked it. +func TestVolumeLoadLeavesReadOnlyVolumeTailAlone(t *testing.T) { + dir := t.TempDir() + v := newTornTailTestVolume(t, dir) + for _, id := range []uint64{1, 2, 99} { + if _, _, _, err := v.writeNeedle2(newRandomNeedle(id), true, false, false); err != nil { + t.Fatal(err) + } + } + if err := v.PersistReadOnly(true, false); err != nil { + t.Fatal(err) + } + datName, idxName := v.FileName(".dat"), v.FileName(".idx") + v.Close() + if err := os.Truncate(idxName, fileSize(t, idxName)-types.NeedleMapEntrySize); err != nil { + t.Fatal(err) + } + datBefore, idxBefore := fileSize(t, datName), fileSize(t, idxName) + + v = newTornTailTestVolume(t, dir) + defer v.Close() + if !v.noWriteOrDelete { + t.Fatal("volume persisted read-only loaded writable") + } + if got := fileSize(t, datName); got != datBefore { + t.Fatalf(".dat of a read-only volume changed from %d to %d bytes", datBefore, got) + } + if got := fileSize(t, idxName); got != idxBefore { + t.Fatalf(".idx of a read-only volume changed from %d to %d bytes", idxBefore, got) + } +} + +// An .idx that ends in a partial entry keeps the existing behaviour: +// the volume does not load writable and neither file is changed. +func TestVolumeLoadLeavesPartialIdxEntryAlone(t *testing.T) { + f := buildTornFixture(t, writes(1, 2, 3), 0) + idxPartial := fileSize(t, f.idxName) - types.NeedleMapEntrySize/2 + if err := os.Truncate(f.idxName, idxPartial); err != nil { + t.Fatal(err) + } + datBefore := fileSize(t, f.datName) + + v2, err := NewVolume(f.dir, f.dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0) + if err == nil { + t.Logf("partial .idx: volume loaded, noWriteOrDelete=%v", v2.noWriteOrDelete) + if !v2.noWriteOrDelete { + v2.Close() + t.Fatal("volume with a partial .idx entry loaded writable") + } + v2.Close() + } else { + t.Logf("partial .idx: volume not loaded: %v", err) + } + if got := fileSize(t, f.datName); got != datBefore { + t.Fatalf(".dat changed from %d to %d bytes", datBefore, got) + } + if got := fileSize(t, f.idxName); got != idxPartial { + t.Fatalf(".idx changed from %d to %d bytes", idxPartial, got) + } +} + +// faultyDatFile fails Sync, or the read of a whole record at one offset, the +// way a failing disk does. +type faultyDatFile struct { + backend.BackendStorageFile + failSync bool + failRecordAt int64 + failRecordSet bool +} + +var errFaultyDisk = errors.New("input/output error") + +func (f *faultyDatFile) Sync() error { + if f.failSync { + return errFaultyDisk + } + return f.BackendStorageFile.Sync() +} + +func (f *faultyDatFile) ReadAt(p []byte, off int64) (int, error) { + if f.failRecordSet && off == f.failRecordAt && len(p) > types.NeedleHeaderSize { + return 0, errFaultyDisk + } + return f.BackendStorageFile.ReadAt(p, off) +} + +// A read error is not evidence of a torn write: the last record is kept. +func TestWalkUnindexedDatTailKeepsRecordOnReadError(t *testing.T) { + f := buildTornFixture(t, writes(1, 2, 3, 4), 1) + file, err := os.Open(f.datName) + if err != nil { + t.Fatal(err) + } + dat := &faultyDatFile{BackendStorageFile: backend.NewDiskFile(file), failRecordAt: f.starts[3], failRecordSet: true} + defer dat.Close() + + records, cutAt, err := walkUnindexedDatTail(dat, needle.Version3, f.starts[3], f.end, 0) + if err == nil { + t.Fatalf("read error on the last record: walk returned %d record(s) and a cut at %d, want an error", len(records), cutAt) + } + if errors.Is(err, errTailNotTorn) { + t.Fatalf("read error reported as an unexplained tail: %v", err) + } + if !errors.Is(err, errFaultyDisk) { + t.Fatalf("walk error %v does not carry the read error", err) + } +} + +// A repair that changed a file but could not sync it fails the integrity +// check, so the volume loads read-only rather than writable on an unsynced +// repair. +func TestCheckVolumeDataIntegrityFailsOnUnsyncedRepair(t *testing.T) { + dir := t.TempDir() + v := newTornTailTestVolume(t, dir) + var starts []int64 + for _, id := range []uint64{1, 2, 3, 4} { + at, _, _ := v.DataBackend.GetStat() + starts = append(starts, at) + if _, _, _, err := v.writeNeedle2(newRandomNeedle(id), true, false, false); err != nil { + t.Fatal(err) + } + } + end, _, _ := v.DataBackend.GetStat() + // persisted read-only so the load below does not repair the tail itself + if err := v.PersistReadOnly(true, false); err != nil { + t.Fatal(err) + } + datName, idxName := v.FileName(".dat"), v.FileName(".idx") + v.Close() + if err := os.Truncate(idxName, fileSize(t, idxName)-types.NeedleMapEntrySize); err != nil { + t.Fatal(err) + } + if err := os.Truncate(datName, midRecordBody(starts[3], end)); err != nil { + t.Fatal(err) + } + + v = newTornTailTestVolume(t, dir) + defer v.Close() + v.noWriteOrDelete = false + v.DataBackend = &faultyDatFile{BackendStorageFile: v.DataBackend, failSync: true} + indexFile, err := os.OpenFile(idxName, os.O_RDWR, 0644) + if err != nil { + t.Fatal(err) + } + defer indexFile.Close() + + if _, err := CheckVolumeDataIntegrity(v, indexFile); err == nil { + t.Fatal("integrity check passed although the repaired .dat was not synced") + } +} + +// partialWriteIndex accepts only the first n bytes of a write, as a full disk +// does. +type partialWriteIndex struct { + data []byte + accept int + truncated bool +} + +func (p *partialWriteIndex) WriteAt(b []byte, off int64) (int, error) { + n := min(len(b), p.accept) + if end := int(off) + n; end > len(p.data) { + p.data = append(p.data, make([]byte, end-len(p.data))...) + } + copy(p.data[off:], b[:n]) + if n < len(b) { + return n, errors.New("no space left on device") + } + return n, nil +} + +func (p *partialWriteIndex) Truncate(size int64) error { + p.data = p.data[:size] + p.truncated = true + return nil +} + +func (p *partialWriteIndex) Sync() error { return nil } +func (p *partialWriteIndex) Name() string { return "test.idx" } + +// A failed append leaves no partial entry behind. +func TestAppendIndexEntriesRemovesPartialWrite(t *testing.T) { + idxFile := &partialWriteIndex{data: make([]byte, 2*types.NeedleMapEntrySize), accept: types.NeedleMapEntrySize + 5} + entries := make([]byte, 2*types.NeedleMapEntrySize) + if err := appendIndexEntries(idxFile, entries, 2*types.NeedleMapEntrySize); err == nil { + t.Fatal("a partial append reported success") + } + if got, want := len(idxFile.data), 2*types.NeedleMapEntrySize; got != want { + t.Fatalf(".idx is %d bytes after a failed append, want %d", got, want) + } +} diff --git a/weed/storage/volume_checking_test.go b/weed/storage/volume_checking_test.go index b077485cab..ce1f093428 100644 --- a/weed/storage/volume_checking_test.go +++ b/weed/storage/volume_checking_test.go @@ -427,9 +427,21 @@ func TestCheckVolumeDataIntegrityVerifiesDeletionTail(t *testing.T) { } defer idxFile.Close() + // A read-only volume keeps its tail: the check must report it. + v.noWriteOrDelete = true if _, err := CheckVolumeDataIntegrity(v, idxFile); err == nil { t.Fatal("CheckVolumeDataIntegrity should detect bytes past a deletion-tombstone tail") } + + // A writable volume would have a torn needle re-indexed or cut, but 64 zero + // bytes are no needle (id 0): they are reported and left in place. + v.noWriteOrDelete = false + if _, err := CheckVolumeDataIntegrity(v, idxFile); err == nil { + t.Fatal("CheckVolumeDataIntegrity should detect zero bytes past a deletion-tombstone tail on a writable volume") + } + if got, _, _ := v.DataBackend.GetStat(); got != datSize+64 { + t.Fatalf(".dat is %d bytes after the check, want %d (nothing cut)", got, datSize+64) + } } // TestVolumeLoadStaysWritableWithKeySortedIndex exercises end to