diff --git a/weed/admin/dash/volume_export.go b/weed/admin/dash/volume_export.go new file mode 100644 index 000000000..58a63d9ef --- /dev/null +++ b/weed/admin/dash/volume_export.go @@ -0,0 +1,340 @@ +package dash + +import ( + "context" + "sort" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/storage" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" +) + +// VolumeListExport is a full-cluster volume report, downloadable from the admin +// UI as JSON. It mirrors the topology the `volume.list` shell command walks +// (data center -> rack -> node -> disk -> volumes / EC shards) and adds derived +// fields the shell command does not print: garbage and fullness ratios, a +// human-readable modified time, remote-tiering keys, per-disk capacity counts, +// and the cluster-wide duplicate-volume-id scan. +type VolumeListExport struct { + GeneratedAt time.Time `json:"generated_at"` + VolumeSizeLimitMB uint64 `json:"volume_size_limit_mb"` + FilterCollection string `json:"filter_collection,omitempty"` + Totals VolumeExportTotals `json:"totals"` + DuplicateVolumeIds []DuplicateVolumeId `json:"duplicate_volume_ids"` + DataCenters []*ExportDataCenter `json:"data_centers"` +} + +// VolumeExportTotals aggregates the included records. TotalSize sums normal +// volume sizes and EC shard sizes (matching volume.list's statistics); the +// file/delete totals cover normal volumes only. +type VolumeExportTotals struct { + VolumeCount int `json:"volume_count"` + EcShardCount int `json:"ec_shard_count"` + TotalSize uint64 `json:"total_size"` + FileCount uint64 `json:"file_count"` + DeletedFileCount uint64 `json:"deleted_file_count"` + DeletedBytes uint64 `json:"deleted_bytes"` +} + +// DuplicateVolumeId flags a volume id that appears under more than one +// collection, where collection.delete or a bare-id operation is dangerous. +type DuplicateVolumeId struct { + VolumeId uint32 `json:"volume_id"` + Collections []string `json:"collections"` +} + +type ExportDataCenter struct { + Id string `json:"id"` + Racks []*ExportRack `json:"racks"` +} + +type ExportRack struct { + Id string `json:"id"` + Nodes []*ExportNode `json:"nodes"` +} + +type ExportNode struct { + Id string `json:"id"` + GrpcPort uint32 `json:"grpc_port,omitempty"` + Disks []*ExportDisk `json:"disks"` +} + +// ExportDisk capacity counts describe the whole physical disk; the Volumes / +// EcShards lists are filtered when a collection filter is active. +type ExportDisk struct { + DiskType string `json:"disk_type"` + DiskId uint32 `json:"disk_id"` + VolumeCount int64 `json:"volume_count"` + MaxVolumeCount int64 `json:"max_volume_count"` + ActiveVolumeCount int64 `json:"active_volume_count"` + FreeVolumeCount int64 `json:"free_volume_count"` + RemoteVolumeCount int64 `json:"remote_volume_count"` + Volumes []*ExportVolume `json:"volumes,omitempty"` + EcShards []*ExportEcShard `json:"ec_shards,omitempty"` +} + +type ExportVolume struct { + Id uint32 `json:"id"` + Collection string `json:"collection"` + Size uint64 `json:"size"` + FileCount uint64 `json:"file_count"` + DeleteCount uint64 `json:"delete_count"` + DeletedByteCount uint64 `json:"deleted_byte_count"` + GarbageRatio float64 `json:"garbage_ratio"` + FullnessRatio float64 `json:"fullness_ratio"` + ReplicaPlacement string `json:"replica_placement"` + Version uint32 `json:"version"` + Ttl string `json:"ttl"` + ReadOnly bool `json:"read_only"` + CompactRevision uint32 `json:"compact_revision"` + ModifiedAtSecond int64 `json:"modified_at_second"` + ModifiedAt string `json:"modified_at,omitempty"` + DiskType string `json:"disk_type"` + DiskId uint32 `json:"disk_id"` + RemoteStorageName string `json:"remote_storage_name,omitempty"` + RemoteStorageKey string `json:"remote_storage_key,omitempty"` +} + +type ExportEcShard struct { + Id uint32 `json:"id"` + Collection string `json:"collection"` + ShardIds []uint32 `json:"shard_ids"` + ShardSizes []int64 `json:"shard_sizes"` + TotalSize int64 `json:"total_size"` + FileCount uint64 `json:"file_count"` + DeleteCount uint64 `json:"delete_count"` + DiskType string `json:"disk_type"` + DiskId uint32 `json:"disk_id"` + ExpireAtSec uint64 `json:"expire_at_sec,omitempty"` +} + +// ExportClusterVolumeList builds a full-cluster volume report from the master +// topology. A non-empty collection limits the listed volumes and EC shards to +// that collection; the duplicate-id scan always covers the whole cluster. +// generatedAt is passed in so the report is deterministic and testable. +func (s *AdminServer) ExportClusterVolumeList(ctx context.Context, collection string, generatedAt time.Time) (*VolumeListExport, error) { + export := &VolumeListExport{ + GeneratedAt: generatedAt, + FilterCollection: collection, + DuplicateVolumeIds: []DuplicateVolumeId{}, + DataCenters: []*ExportDataCenter{}, + } + + err := s.WithMasterClient(func(client master_pb.SeaweedClient) error { + resp, err := client.VolumeList(ctx, &master_pb.VolumeListRequest{}) + if err != nil { + return err + } + export.VolumeSizeLimitMB = resp.VolumeSizeLimitMb + topo := resp.TopologyInfo + if topo == nil { + return nil + } + + volumeSizeLimit := resp.VolumeSizeLimitMb * 1024 * 1024 + export.DuplicateVolumeIds = findDuplicateVolumeIdsForExport(topo) + + dcs := append([]*master_pb.DataCenterInfo(nil), topo.DataCenterInfos...) + sort.Slice(dcs, func(i, j int) bool { return dcs[i].Id < dcs[j].Id }) + for _, dc := range dcs { + exportDc := &ExportDataCenter{Id: dc.Id, Racks: []*ExportRack{}} + racks := append([]*master_pb.RackInfo(nil), dc.RackInfos...) + sort.Slice(racks, func(i, j int) bool { return racks[i].Id < racks[j].Id }) + for _, rack := range racks { + exportRack := &ExportRack{Id: rack.Id, Nodes: []*ExportNode{}} + nodes := append([]*master_pb.DataNodeInfo(nil), rack.DataNodeInfos...) + sort.Slice(nodes, func(i, j int) bool { return nodes[i].Id < nodes[j].Id }) + for _, node := range nodes { + exportNode := &ExportNode{Id: node.Id, GrpcPort: node.GrpcPort, Disks: []*ExportDisk{}} + for _, diskType := range sortedDiskTypes(node.DiskInfos) { + // Split per physical disk like volume.list so DiskId is meaningful. + for _, diskInfo := range node.DiskInfos[diskType].SplitByPhysicalDisk() { + if disk := buildExportDisk(diskInfo, collection, volumeSizeLimit, &export.Totals); disk != nil { + exportNode.Disks = append(exportNode.Disks, disk) + } + } + } + if len(exportNode.Disks) > 0 { + exportRack.Nodes = append(exportRack.Nodes, exportNode) + } + } + if len(exportRack.Nodes) > 0 { + exportDc.Racks = append(exportDc.Racks, exportRack) + } + } + if len(exportDc.Racks) > 0 { + export.DataCenters = append(export.DataCenters, exportDc) + } + } + return nil + }) + if err != nil { + return nil, err + } + return export, nil +} + +func sortedDiskTypes(diskInfos map[string]*master_pb.DiskInfo) []string { + types := make([]string, 0, len(diskInfos)) + for t := range diskInfos { + types = append(types, t) + } + sort.Strings(types) + return types +} + +// buildExportDisk returns nil when the disk holds nothing matching the filter. +func buildExportDisk(diskInfo *master_pb.DiskInfo, collection string, volumeSizeLimit uint64, totals *VolumeExportTotals) *ExportDisk { + diskType := diskInfo.Type + if diskType == "" { + diskType = "hdd" + } + + vols := append([]*master_pb.VolumeInformationMessage(nil), diskInfo.VolumeInfos...) + sort.Slice(vols, func(i, j int) bool { return vols[i].Id < vols[j].Id }) + var volumes []*ExportVolume + for _, vi := range vols { + if collection != "" && !matchesCollection(vi.Collection, collection) { + continue + } + volumes = append(volumes, buildExportVolume(vi, volumeSizeLimit)) + totals.VolumeCount++ + totals.TotalSize += vi.Size + totals.FileCount += vi.FileCount + totals.DeletedFileCount += vi.DeleteCount + totals.DeletedBytes += vi.DeletedByteCount + } + + shards := append([]*master_pb.VolumeEcShardInformationMessage(nil), diskInfo.EcShardInfos...) + sort.Slice(shards, func(i, j int) bool { return shards[i].Id < shards[j].Id }) + var ecShards []*ExportEcShard + for _, eci := range shards { + if collection != "" && !matchesCollection(eci.Collection, collection) { + continue + } + shard := buildExportEcShard(eci) + ecShards = append(ecShards, shard) + totals.EcShardCount++ + totals.TotalSize += uint64(shard.TotalSize) + } + + if len(volumes) == 0 && len(ecShards) == 0 { + return nil + } + + return &ExportDisk{ + DiskType: diskType, + DiskId: diskInfo.DiskId, + VolumeCount: diskInfo.VolumeCount, + MaxVolumeCount: diskInfo.MaxVolumeCount, + ActiveVolumeCount: diskInfo.ActiveVolumeCount, + FreeVolumeCount: diskInfo.FreeVolumeCount, + RemoteVolumeCount: diskInfo.RemoteVolumeCount, + Volumes: volumes, + EcShards: ecShards, + } +} + +func buildExportVolume(m *master_pb.VolumeInformationMessage, volumeSizeLimit uint64) *ExportVolume { + v := &ExportVolume{ + Id: m.Id, + Collection: m.Collection, + Size: m.Size, + FileCount: m.FileCount, + DeleteCount: m.DeleteCount, + DeletedByteCount: m.DeletedByteCount, + Version: m.Version, + ReadOnly: m.ReadOnly, + CompactRevision: m.CompactRevision, + ModifiedAtSecond: m.ModifiedAtSecond, + DiskType: m.DiskType, + DiskId: m.DiskId, + RemoteStorageName: m.RemoteStorageName, + RemoteStorageKey: m.RemoteStorageKey, + } + // Decode replica placement and TTL the way volume.list does. + if vi, err := storage.NewVolumeInfo(m); err == nil { + v.ReplicaPlacement = vi.ReplicaPlacement.String() + v.Ttl = vi.Ttl.String() + } + if m.Size > 0 { + v.GarbageRatio = float64(m.DeletedByteCount) / float64(m.Size) + } + if volumeSizeLimit > 0 { + v.FullnessRatio = float64(m.Size) / float64(volumeSizeLimit) + } + if m.ModifiedAtSecond > 0 { + v.ModifiedAt = time.Unix(m.ModifiedAtSecond, 0).UTC().Format(time.RFC3339) + } + return v +} + +func buildExportEcShard(m *master_pb.VolumeEcShardInformationMessage) *ExportEcShard { + si := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(m) + return &ExportEcShard{ + Id: m.Id, + Collection: m.Collection, + ShardIds: si.IdsUint32(), + ShardSizes: si.SizesInt64(), + TotalSize: int64(si.TotalSize()), + FileCount: m.FileCount, + DeleteCount: m.DeleteCount, + DiskType: m.DiskType, + DiskId: m.DiskId, + ExpireAtSec: m.ExpireAtSec, + } +} + +// findDuplicateVolumeIdsForExport reports volume ids living under more than one +// collection. Volume ids are meant to be globally unique; a duplicate is a +// cluster-health hazard (collection.delete destroys one collection's copy, +// bare-id lookups are ambiguous). This mirrors volume.list's findDuplicateVolumeIds +// rather than reaching into the shell package, per the admin-renderer design. +// First collection per id is tracked alone, and a set is allocated only on the +// first clash, keeping allocations O(duplicates) on million-volume clusters. +func findDuplicateVolumeIdsForExport(topo *master_pb.TopologyInfo) []DuplicateVolumeId { + firstCollectionByVid := make(map[uint32]string) + collectionsByVid := make(map[uint32]map[string]struct{}) + note := func(vid uint32, collection string) { + if collections := collectionsByVid[vid]; collections != nil { + collections[collection] = struct{}{} + return + } + first, seen := firstCollectionByVid[vid] + if !seen { + firstCollectionByVid[vid] = collection + return + } + if first == collection { + return + } + collectionsByVid[vid] = map[string]struct{}{first: {}, collection: {}} + } + for _, dc := range topo.DataCenterInfos { + for _, rack := range dc.RackInfos { + for _, node := range rack.DataNodeInfos { + for _, disk := range node.DiskInfos { + for _, vi := range disk.VolumeInfos { + note(vi.Id, vi.Collection) + } + for _, ec := range disk.EcShardInfos { + note(ec.Id, ec.Collection) + } + } + } + } + } + + duplicates := make([]DuplicateVolumeId, 0, len(collectionsByVid)) + for vid, set := range collectionsByVid { + names := make([]string, 0, len(set)) + for name := range set { + names = append(names, name) + } + sort.Strings(names) + duplicates = append(duplicates, DuplicateVolumeId{VolumeId: vid, Collections: names}) + } + sort.Slice(duplicates, func(i, j int) bool { return duplicates[i].VolumeId < duplicates[j].VolumeId }) + return duplicates +} diff --git a/weed/admin/dash/volume_export_test.go b/weed/admin/dash/volume_export_test.go new file mode 100644 index 000000000..e3ed0cd30 --- /dev/null +++ b/weed/admin/dash/volume_export_test.go @@ -0,0 +1,129 @@ +package dash + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" +) + +func TestBuildExportVolumeDerivedFields(t *testing.T) { + v := buildExportVolume(&master_pb.VolumeInformationMessage{ + Id: 7, + Collection: "photos", + Size: 100, + DeletedByteCount: 25, + ReplicaPlacement: 0, + ModifiedAtSecond: 1700000000, + }, 1000) + + if v.GarbageRatio != 0.25 { + t.Errorf("garbage ratio = %v, want 0.25", v.GarbageRatio) + } + if v.FullnessRatio != 0.1 { + t.Errorf("fullness ratio = %v, want 0.1", v.FullnessRatio) + } + if v.ReplicaPlacement != "000" { + t.Errorf("replica placement = %q, want 000", v.ReplicaPlacement) + } + if v.ModifiedAt == "" { + t.Error("modified_at should be set when modified_at_second > 0") + } +} + +func TestBuildExportDiskFilterAndTotals(t *testing.T) { + disk := &master_pb.DiskInfo{ + Type: "hdd", + VolumeCount: 2, + VolumeInfos: []*master_pb.VolumeInformationMessage{ + {Id: 1, Collection: "keep", Size: 10, FileCount: 3, DeleteCount: 1, DeletedByteCount: 2}, + {Id: 2, Collection: "drop", Size: 99, FileCount: 9}, + }, + EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ + {Id: 3, Collection: "keep", EcIndexBits: 0b111, ShardSizes: []int64{4, 4, 4}, FileCount: 5}, + {Id: 4, Collection: "drop", EcIndexBits: 0b1, ShardSizes: []int64{7}}, + }, + } + + var totals VolumeExportTotals + got := buildExportDisk(disk, "keep", 1000, &totals) + if got == nil { + t.Fatal("expected disk with matching records, got nil") + } + if len(got.Volumes) != 1 || got.Volumes[0].Id != 1 { + t.Errorf("expected only volume 1 after filter, got %+v", got.Volumes) + } + if len(got.EcShards) != 1 || got.EcShards[0].Id != 3 { + t.Errorf("expected only ec shard 3 after filter, got %+v", got.EcShards) + } + if got.VolumeCount != 2 { + t.Errorf("disk VolumeCount should stay physical (2), got %d", got.VolumeCount) + } + if got.EcShards[0].TotalSize != 12 { + t.Errorf("ec shard total size = %d, want 12", got.EcShards[0].TotalSize) + } + + if totals.VolumeCount != 1 || totals.EcShardCount != 1 { + t.Errorf("totals counts = vol %d ec %d, want 1/1", totals.VolumeCount, totals.EcShardCount) + } + // 10 (volume size) + 12 (kept ec shard sizes). + if totals.TotalSize != 22 { + t.Errorf("totals TotalSize = %d, want 22", totals.TotalSize) + } + if totals.FileCount != 3 || totals.DeletedFileCount != 1 || totals.DeletedBytes != 2 { + t.Errorf("totals files = %d del %d delBytes %d, want 3/1/2", totals.FileCount, totals.DeletedFileCount, totals.DeletedBytes) + } +} + +func TestBuildExportDiskPrunesWhenNoMatch(t *testing.T) { + disk := &master_pb.DiskInfo{ + Type: "hdd", + VolumeInfos: []*master_pb.VolumeInformationMessage{{Id: 1, Collection: "other"}}, + } + var totals VolumeExportTotals + if got := buildExportDisk(disk, "keep", 0, &totals); got != nil { + t.Errorf("expected nil for disk with no matching records, got %+v", got) + } +} + +func TestFindDuplicateVolumeIdsForExport(t *testing.T) { + topo := &master_pb.TopologyInfo{ + DataCenterInfos: []*master_pb.DataCenterInfo{{ + RackInfos: []*master_pb.RackInfo{{ + DataNodeInfos: []*master_pb.DataNodeInfo{ + {DiskInfos: map[string]*master_pb.DiskInfo{"hdd": { + VolumeInfos: []*master_pb.VolumeInformationMessage{ + {Id: 5, Collection: "a"}, + {Id: 7, Collection: "shared"}, // replica below, same collection + }, + }}}, + {DiskInfos: map[string]*master_pb.DiskInfo{"hdd": { + VolumeInfos: []*master_pb.VolumeInformationMessage{ + {Id: 5, Collection: "b"}, // 5 now spans a + b -> duplicate + {Id: 7, Collection: "shared"}, // replica, not a duplicate + }, + EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ + {Id: 5, Collection: "c"}, // 5 also as EC in c + }, + }}}, + }, + }}, + }}, + } + + dups := findDuplicateVolumeIdsForExport(topo) + if len(dups) != 1 { + t.Fatalf("expected exactly 1 duplicate, got %+v", dups) + } + if dups[0].VolumeId != 5 { + t.Errorf("duplicate volume id = %d, want 5", dups[0].VolumeId) + } + want := []string{"a", "b", "c"} + if len(dups[0].Collections) != len(want) { + t.Fatalf("collections = %v, want %v", dups[0].Collections, want) + } + for i := range want { + if dups[0].Collections[i] != want[i] { + t.Errorf("collections[%d] = %q, want %q", i, dups[0].Collections[i], want[i]) + } + } +} diff --git a/weed/admin/handlers/admin_handlers.go b/weed/admin/handlers/admin_handlers.go index 4ac8f717f..fee4b198e 100644 --- a/weed/admin/handlers/admin_handlers.go +++ b/weed/admin/handlers/admin_handlers.go @@ -250,6 +250,7 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) { filesApi.HandleFunc("/metadata", h.fileBrowserHandlers.ExportMetadata).Methods(http.MethodGet) volumeApi := api.PathPrefix("/volumes").Subrouter() + volumeApi.HandleFunc("/export", h.clusterHandlers.ExportClusterVolumes).Methods(http.MethodGet) volumeApi.Handle("/{id}/{server}/vacuum", wrapWrite(h.clusterHandlers.VacuumVolume)).Methods(http.MethodPost) pluginApi := api.PathPrefix("/plugin").Subrouter() diff --git a/weed/admin/handlers/cluster_handlers.go b/weed/admin/handlers/cluster_handlers.go index ae454b1a0..fe6734b60 100644 --- a/weed/admin/handlers/cluster_handlers.go +++ b/weed/admin/handlers/cluster_handlers.go @@ -1,14 +1,19 @@ package handlers import ( + "encoding/json" + "fmt" "math" + "mime" "net/http" "strconv" + "time" "github.com/gorilla/mux" "github.com/seaweedfs/seaweedfs/weed/admin/dash" "github.com/seaweedfs/seaweedfs/weed/admin/view/app" "github.com/seaweedfs/seaweedfs/weed/admin/view/layout" + "github.com/seaweedfs/seaweedfs/weed/glog" ) // ClusterHandlers contains all the HTTP handlers for cluster management @@ -88,6 +93,30 @@ func (h *ClusterHandlers) ShowClusterVolumes(w http.ResponseWriter, r *http.Requ } } +// ExportClusterVolumes streams the full-cluster volume list as a downloadable +// JSON report: every volume and EC shard across the topology, with more fields +// than the paginated table (a superset of the volume.list shell command). +func (h *ClusterHandlers) ExportClusterVolumes(w http.ResponseWriter, r *http.Request) { + collection := r.URL.Query().Get("collection") // Optional collection filter + + export, err := h.adminServer.ExportClusterVolumeList(r.Context(), collection, time.Now().UTC()) + if err != nil { + writeJSONError(w, http.StatusInternalServerError, "Failed to export volume list: "+err.Error()) + return + } + + filename := fmt.Sprintf("seaweedfs-volumes-%s.json", export.GeneratedAt.Format("20060102-150405")) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": filename})) + + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + if err := enc.Encode(export); err != nil { + // The response is already streaming, so we can only log a late failure. + glog.Errorf("export volume list: encode failed: %v", err) + } +} + // ShowVolumeDetails renders the volume details page func (h *ClusterHandlers) ShowVolumeDetails(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) diff --git a/weed/admin/static/js/admin.js b/weed/admin/static/js/admin.js index f93d8315e..ad9db59c0 100644 --- a/weed/admin/static/js/admin.js +++ b/weed/admin/static/js/admin.js @@ -736,6 +736,18 @@ function exportVolumes() { downloadCSV(csv, 'seaweedfs-volumes.csv'); } +// Export the full cluster volume list (all volumes + EC shards, more fields than +// the table) as a JSON report served by the admin API. Carries the active +// collection filter from the page URL. +function exportVolumeList() { + let url = basePath('/api/volumes/export'); + const collection = new URLSearchParams(window.location.search).get('collection'); + if (collection) { + url += '?collection=' + encodeURIComponent(collection); + } + window.open(url, '_blank'); +} + // Export collections data as CSV function exportCollections() { const table = document.getElementById('collectionsTable'); diff --git a/weed/admin/view/app/cluster_volumes.templ b/weed/admin/view/app/cluster_volumes.templ index 7cee0474e..a1b4bcb44 100644 --- a/weed/admin/view/app/cluster_volumes.templ +++ b/weed/admin/view/app/cluster_volumes.templ @@ -32,7 +32,10 @@ templ ClusterVolumes(data dash.ClusterVolumesData) { + diff --git a/weed/admin/view/app/cluster_volumes_templ.go b/weed/admin/view/app/cluster_volumes_templ.go index 9fca3108f..50421b01a 100644 --- a/weed/admin/view/app/cluster_volumes_templ.go +++ b/weed/admin/view/app/cluster_volumes_templ.go @@ -111,14 +111,14 @@ func ClusterVolumes(data dash.ClusterVolumesData) templ.Component { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, ">500 per page