admin: show capacity per storage tier and stop counting remote-tiered bytes as local disk usage (#10766)

* admin: show capacity per storage tier and stop counting remote-tiered bytes as local disk usage

A remote-tiered volume reports its cloud object's size, so summing volume
sizes inflated the dashboard's used-vs-capacity numbers (the local .dat is
gone after volume.tier.move). Split the accounting: DiskUsage now only
counts bytes on local disks, with the cloud bytes surfaced separately per
server and per remote storage name.

The dashboard gains a Storage Tiers table breaking volumes and EC shards
down by tier (each local disk type plus each remote storage), using the
per-disk-type statfs numbers already in the VolumeList response. The
volumes page badges remote-tiered volumes with their storage name, and
the EC shards page fills in real per-shard sizes instead of hardcoding 0.

* admin: review fixes for the tier capacity display

- A disk that predates disk_total_bytes now contributes its logical
  bytes to the tier's DiskUsed, so a tier mixing old and new volume
  servers doesn't underreport usage; the usage bar always reflects the
  displayed Disk Used value (the DataSize fallback in UsagePercent is
  gone, and the percent math is overflow-safe).
- getTopologyViaGRPC defaults a zero VolumeSizeLimitMb to 30000 MB like
  GetClusterVolumeServers, keeping slot-based capacities consistent.
- The dashboard volume-servers column reads Usage / Capacity to match
  its cell content, and the hdd disk-type default is shared between the
  volumes-page badge and countUniqueDiskTypes.
This commit is contained in:
Chris Lu
2026-08-15 12:35:16 -07:00
committed by GitHub
parent 1c926e8fac
commit 97a155d14d
13 changed files with 1392 additions and 720 deletions
+5
View File
@@ -44,6 +44,10 @@ type AdminData struct {
// Trends holds at-a-glance sparklines built from the admin's own recent
// cluster snapshots (no Prometheus required).
Trends DashboardTrends `json:"trends"`
// TierStats breaks volumes and EC shards down by storage tier: local
// disk types plus one entry per remote storage holding tiered volumes.
TierStats []TierStats `json:"tier_stats"`
}
// Object Store Users management structures
@@ -213,6 +217,7 @@ func (s *AdminServer) GetAdminData(username string) (AdminData, error) {
TotalEcShards: totalEcShards,
TotalMountClients: totalMountClients,
Trends: s.GetDashboardTrends(),
TierStats: topology.TierStats,
}
return adminData, nil
+22 -4
View File
@@ -125,6 +125,12 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error {
}
if resp.TopologyInfo != nil {
// Get volume size limit from response, default to 30GB if not set
volumeSizeLimitMb := resp.VolumeSizeLimitMb
if volumeSizeLimitMb == 0 {
volumeSizeLimitMb = 30000
}
// Process gRPC response
for _, dc := range resp.TopologyInfo.DataCenterInfos {
dataCenter := DataCenter{
@@ -143,6 +149,7 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error {
var totalVolumes int64
var totalMaxVolumes int64
var totalSize int64
var remoteSize int64
// Prefer the real physical disk capacity the volume server
// reports per disk; the slot-based estimate overstates capacity
// when maxVolumeCount is configured higher than the disk holds.
@@ -154,12 +161,18 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error {
if diskInfo.DiskTotalBytes > 0 {
diskCapacity += int64(diskInfo.DiskTotalBytes)
} else {
diskCapacity += diskInfo.MaxVolumeCount * int64(resp.VolumeSizeLimitMb) * 1024 * 1024
diskCapacity += diskInfo.MaxVolumeCount * int64(volumeSizeLimitMb) * 1024 * 1024
}
// Sum up individual volume information
// A remote-tiered volume reports its cloud object's
// size; keep those bytes out of the local disk usage
// that is compared against diskCapacity.
for _, volInfo := range diskInfo.VolumeInfos {
totalSize += int64(volInfo.Size)
if volInfo.RemoteStorageName != "" {
remoteSize += int64(volInfo.Size)
} else {
totalSize += int64(volInfo.Size)
}
}
// ShardSizes is local to this node, so summing
@@ -193,12 +206,15 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error {
DiskUsage: totalSize,
DiskCapacity: diskCapacity,
LastHeartbeat: time.Now(),
RemoteSize: remoteSize,
}
rackObj.Nodes = append(rackObj.Nodes, vs)
topology.VolumeServers = append(topology.VolumeServers, vs)
topology.TotalVolumes += vs.Volumes
topology.TotalSize += totalSize
// TotalSize is the logical data size, wherever the
// bytes live, so remote-tiered volumes still count.
topology.TotalSize += totalSize + remoteSize
}
dataCenter.Racks = append(dataCenter.Racks, rackObj)
@@ -211,6 +227,8 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error {
// nets out tombstones and counts a chunk once no matter how many
// volume replicas or EC shard holders report it.
topology.TotalChunks = totalCollectionFileCount(resp.TopologyInfo)
topology.TierStats = CollectTierStats(resp.TopologyInfo, volumeSizeLimitMb)
}
return nil
+2 -1
View File
@@ -66,6 +66,7 @@ func (s *AdminServer) GetClusterEcShards(page int, pageSize int, sortBy string,
// Create individual shard entries for each shard this server has
shardBits := ecShardInfo.EcIndexBits
shardsInfo := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(ecShardInfo)
for shardId := 0; shardId < erasure_coding.MaxShardCount; shardId++ {
if (shardBits & (1 << uint(shardId))) != 0 {
// Mark this shard as present for this volume
@@ -75,7 +76,7 @@ func (s *AdminServer) GetClusterEcShards(page int, pageSize int, sortBy string,
VolumeID: volumeId,
ShardID: uint32(shardId),
Collection: ecShardInfo.Collection,
Size: 0, // EC shards don't have individual size in the API response
Size: uint64(shardsInfo.Size(erasure_coding.ShardId(shardId))),
Server: node.Id,
DataCenter: dc.Id,
Rack: rack.Id,
+134
View File
@@ -0,0 +1,134 @@
package dash
import (
"sort"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
)
// TierStats aggregates the volumes and EC shards that live on one storage
// tier: a local disk type ("hdd", "ssd", or a custom tag), or the remote
// storage a tiered volume was uploaded to. A remote-tiered volume reports
// the size of its cloud object, so its bytes belong to the remote tier,
// not to the local disk that holds only its index.
type TierStats struct {
Name string `json:"name"`
IsRemote bool `json:"is_remote"`
VolumeCount int `json:"volume_count"`
EcShardCount int `json:"ec_shard_count"`
DataSize int64 `json:"data_size"`
DiskUsed int64 `json:"disk_used"`
DiskCapacity int64 `json:"disk_capacity"`
MaxVolumes int64 `json:"max_volumes"`
}
// UsagePercent is the tier's local disk usage in percent, clamped to
// [0, 100]. Remote tiers have no capacity and return 0.
func (t TierStats) UsagePercent() int {
if t.IsRemote || t.DiskCapacity <= 0 {
return 0
}
percent := int(float64(t.DiskUsed) / float64(t.DiskCapacity) * 100)
if percent < 0 {
return 0
}
if percent > 100 {
return 100
}
return percent
}
// tierDiskType maps the empty disk type to its display name.
func tierDiskType(diskType string) string {
if diskType == "" {
return "hdd"
}
return diskType
}
// CollectTierStats walks the topology and groups capacity and usage by
// tier. DiskUsed/DiskCapacity come from the statfs numbers the volume
// servers report per disk type; a disk that predates disk_total_bytes
// falls back to the slot-based capacity estimate and to the logical
// bytes it holds, so mixed-version tiers don't underreport usage.
// Remote tiers have no local disk, so only VolumeCount and DataSize are
// meaningful there.
func CollectTierStats(topo *master_pb.TopologyInfo, volumeSizeLimitMb uint64) []TierStats {
if topo == nil {
return nil
}
tiers := make(map[string]*TierStats)
tier := func(name string, isRemote bool) *TierStats {
key := name
if isRemote {
key = "remote\x00" + name
}
t := tiers[key]
if t == nil {
t = &TierStats{Name: name, IsRemote: isRemote}
tiers[key] = t
}
return t
}
for _, dc := range topo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for _, node := range rack.DataNodeInfos {
for _, diskInfo := range node.DiskInfos {
local := tier(tierDiskType(diskInfo.Type), false)
local.MaxVolumes += diskInfo.MaxVolumeCount
hasStatfs := diskInfo.DiskTotalBytes > 0
if hasStatfs {
local.DiskCapacity += int64(diskInfo.DiskTotalBytes)
if diskInfo.DiskTotalBytes > diskInfo.DiskFreeBytes {
local.DiskUsed += int64(diskInfo.DiskTotalBytes - diskInfo.DiskFreeBytes)
}
} else {
local.DiskCapacity += diskInfo.MaxVolumeCount * int64(volumeSizeLimitMb) * 1024 * 1024
}
var diskLocalBytes int64
for _, volInfo := range diskInfo.VolumeInfos {
if volInfo.RemoteStorageName != "" {
remote := tier(volInfo.RemoteStorageName, true)
remote.VolumeCount++
remote.DataSize += int64(volInfo.Size)
} else {
local.VolumeCount++
local.DataSize += int64(volInfo.Size)
diskLocalBytes += int64(volInfo.Size)
}
}
// ShardSizes is local to this node, so summing across
// nodes gives the tier's physical footprint.
for _, ecShardInfo := range diskInfo.EcShardInfos {
local.EcShardCount += erasure_coding.GetShardCount(ecShardInfo)
ecBytes := erasure_coding.EcShardsTotalSize(ecShardInfo)
local.DataSize += ecBytes
diskLocalBytes += ecBytes
}
// Without statfs numbers, approximate this disk's
// footprint with the logical bytes it holds.
if !hasStatfs {
local.DiskUsed += diskLocalBytes
}
}
}
}
}
result := make([]TierStats, 0, len(tiers))
for _, t := range tiers {
result = append(result, *t)
}
sort.Slice(result, func(i, j int) bool {
if result[i].IsRemote != result[j].IsRemote {
return !result[i].IsRemote
}
return result[i].Name < result[j].Name
})
return result
}
+91
View File
@@ -0,0 +1,91 @@
package dash
import (
"reflect"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)
func TestCollectTierStats(t *testing.T) {
topo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{{
Id: "dc1",
RackInfos: []*master_pb.RackInfo{{
Id: "rack1",
DataNodeInfos: []*master_pb.DataNodeInfo{
{
Id: "node1:8080",
DiskInfos: map[string]*master_pb.DiskInfo{
"": {
Type: "",
MaxVolumeCount: 10,
DiskTotalBytes: 1_000_000,
DiskFreeBytes: 400_000,
VolumeInfos: []*master_pb.VolumeInformationMessage{
{Id: 1, Size: 1000},
{Id: 2, Size: 5000, RemoteStorageName: "s3.backup", RemoteStorageKey: "/2.dat"},
},
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{Id: 7, EcIndexBits: 0b111, ShardSizes: []int64{10, 20, 30}},
},
},
},
},
{
Id: "node2:8080",
DiskInfos: map[string]*master_pb.DiskInfo{
"ssd": {
Type: "ssd",
MaxVolumeCount: 5,
// No statfs numbers: capacity falls back to
// slots and usage to logical bytes.
VolumeInfos: []*master_pb.VolumeInformationMessage{
{Id: 3, Size: 2000, DiskType: "ssd"},
{Id: 4, Size: 7000, DiskType: "ssd", RemoteStorageName: "s3.backup", RemoteStorageKey: "/4.dat"},
{Id: 5, Size: 100, DiskType: "ssd", RemoteStorageName: "gcs.archive", RemoteStorageKey: "/5.dat"},
},
},
},
},
{
// An old server without statfs numbers sharing the
// hdd tier with node1: its logical bytes must still
// count toward the tier's DiskUsed.
Id: "node3:8080",
DiskInfos: map[string]*master_pb.DiskInfo{
"": {
Type: "",
MaxVolumeCount: 2,
VolumeInfos: []*master_pb.VolumeInformationMessage{
{Id: 6, Size: 800},
},
},
},
},
},
}},
}},
}
got := CollectTierStats(topo, 30)
want := []TierStats{
{Name: "hdd", VolumeCount: 2, EcShardCount: 3, DataSize: 1860, DiskUsed: 600_800, DiskCapacity: 1_000_000 + 2*30*1024*1024, MaxVolumes: 12},
{Name: "ssd", VolumeCount: 1, DataSize: 2000, DiskUsed: 2000, DiskCapacity: 5 * 30 * 1024 * 1024, MaxVolumes: 5},
{Name: "gcs.archive", IsRemote: true, VolumeCount: 1, DataSize: 100},
{Name: "s3.backup", IsRemote: true, VolumeCount: 2, DataSize: 12000},
}
if !reflect.DeepEqual(got, want) {
t.Errorf("CollectTierStats mismatch:\n got: %+v\nwant: %+v", got, want)
}
}
func TestCollectTierStatsEmpty(t *testing.T) {
if got := CollectTierStats(nil, 30); got != nil {
t.Errorf("expected nil for nil topology, got %+v", got)
}
if got := CollectTierStats(&master_pb.TopologyInfo{}, 30); len(got) != 0 {
t.Errorf("expected no tiers for empty topology, got %+v", got)
}
}
+8
View File
@@ -19,6 +19,9 @@ type ClusterTopology struct {
TotalChunks int64 `json:"total_chunks"`
TotalSize int64 `json:"total_size"`
UpdatedAt time.Time `json:"updated_at"`
// TierStats breaks volumes and EC shards down by storage tier: local
// disk types plus one entry per remote storage holding tiered volumes.
TierStats []TierStats `json:"tier_stats"`
}
type MasterNode struct {
@@ -52,6 +55,11 @@ type VolumeServer struct {
EcVolumes int `json:"ec_volumes"` // Number of EC volumes this server has shards for
EcShards int `json:"ec_shards"` // Total number of EC shards on this server
EcShardDetails []VolumeServerEcInfo `json:"ec_shard_details"` // Detailed EC shard information
// RemoteSize is the bytes this server's remote-tiered volumes hold in
// cloud storage. Those bytes are excluded from DiskUsage, which only
// counts what occupies local disks.
RemoteSize int64 `json:"remote_size"`
}
func (vs *VolumeServer) GetDisplayAddress() string {
+16 -3
View File
@@ -122,7 +122,11 @@ func (s *AdminServer) GetClusterVolumes(page int, pageSize int, sortBy string, s
diskTypeMap := make(map[string]bool)
collectionMap := make(map[string]bool)
versionMap := make(map[string]bool)
hasRemoteVolumes := false
for _, volume := range volumes {
if volume.RemoteStorageName != "" {
hasRemoteVolumes = true
}
if volume.DataCenter != "" {
dataCenterMap[volume.DataCenter] = true
}
@@ -190,7 +194,9 @@ func (s *AdminServer) GetClusterVolumes(page int, pageSize int, sortBy string, s
// Determine conditional display flags and extract single values
showDataCenterColumn := dataCenterCount > 1
showRackColumn := rackCount > 1
showDiskTypeColumn := diskTypeCount > 1
// Remote-tiered volumes surface in the disk type column, so show it
// whenever any volume lives on a remote tier.
showDiskTypeColumn := diskTypeCount > 1 || hasRemoteVolumes
showCollectionColumn := collectionCount > 1 && collection == "" // Hide column when filtering by collection
showVersionColumn := versionCount > 1
@@ -482,10 +488,17 @@ func (s *AdminServer) GetClusterVolumeServers() (*ClusterVolumeServersData, erro
vs.DiskCapacity += int64(diskInfo.MaxVolumeCount) * int64(volumeSizeLimitMB) * 1024 * 1024
}
// Count regular volumes and calculate disk usage
// Count regular volumes and calculate disk usage.
// A remote-tiered volume reports its cloud object's
// size; keep those bytes out of the local disk usage
// that is compared against DiskCapacity.
for _, volInfo := range diskInfo.VolumeInfos {
vs.Volumes++
vs.DiskUsage += int64(volInfo.Size)
if volInfo.RemoteStorageName != "" {
vs.RemoteSize += int64(volInfo.Size)
} else {
vs.DiskUsage += int64(volInfo.Size)
}
}
// Accumulate EC shard information across all disks for this volume server
+89 -2
View File
@@ -194,6 +194,87 @@ templ Admin(data dash.AdminData) {
</div>
</div>
<!-- Storage Tiers -->
if len(data.TierStats) > 0 {
<div class="row mb-4">
<div class="col-12">
<div class="card shadow mb-4">
<div class="card-header py-3">
<h6 class="m-0 font-weight-bold text-primary">
<i class="fas fa-layer-group me-2"></i>Storage Tiers
</h6>
</div>
<div class="card-body">
<div class="table-responsive">
<table class="table table-hover" width="100%" cellspacing="0">
<thead>
<tr>
<th>Tier</th>
<th>Volumes</th>
<th>EC Shards</th>
<th>Data Size</th>
<th>Disk Used</th>
<th>Capacity</th>
<th>Usage</th>
</tr>
</thead>
<tbody>
for _, tier := range data.TierStats {
<tr>
<td>
if tier.IsRemote {
<i class="fas fa-cloud me-1 text-info"></i>{tier.Name}
<span class="badge bg-info text-white ms-1">remote</span>
} else {
<i class="fas fa-hdd me-1 text-muted"></i>{tier.Name}
}
</td>
<td>{fmt.Sprintf("%d", tier.VolumeCount)}</td>
<td>
if tier.IsRemote {
<span class="text-muted">-</span>
} else {
{fmt.Sprintf("%d", tier.EcShardCount)}
}
</td>
<td>{formatBytes(tier.DataSize)}</td>
<td>
if tier.IsRemote || tier.DiskUsed == 0 {
<span class="text-muted">-</span>
} else {
{formatBytes(tier.DiskUsed)}
}
</td>
<td>
if tier.IsRemote {
<span class="text-muted">-</span>
} else {
{formatBytes(tier.DiskCapacity)}
}
</td>
<td>
if tier.IsRemote {
<span class="text-muted">-</span>
} else {
<div class="progress" style="height: 20px;">
<div class="progress-bar" role="progressbar"
style={fmt.Sprintf("width: %d%%", tier.UsagePercent())}>
{fmt.Sprintf("%d%%", tier.UsagePercent())}
</div>
</div>
}
</td>
</tr>
}
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
}
<!-- Mount Clients -->
<div class="row mb-4">
<div class="col-12">
@@ -348,7 +429,7 @@ templ Admin(data dash.AdminData) {
<th>Rack</th>
<th>Volumes</th>
<th>EC Shards</th>
<th>Capacity</th>
<th>Usage / Capacity</th>
</tr>
</thead>
<tbody>
@@ -381,7 +462,13 @@ templ Admin(data dash.AdminData) {
<span class="text-muted">-</span>
}
</td>
<td>{formatBytes(vs.DiskUsage)} / {formatBytes(vs.DiskCapacity)}</td>
<td>
{formatBytes(vs.DiskUsage)} / {formatBytes(vs.DiskCapacity)}
if vs.RemoteSize > 0 {
<br/>
<small class="text-muted"><i class="fas fa-cloud me-1"></i>{formatBytes(vs.RemoteSize)} remote</small>
}
</td>
</tr>
}
if len(data.VolumeServers) == 0 {
File diff suppressed because it is too large Load Diff
@@ -164,12 +164,17 @@ templ ClusterVolumeServers(data dash.ClusterVolumeServersData) {
<td>
<div class="d-flex align-items-center">
<div class="progress me-2" style="width: 60px; height: 16px;">
<div class="progress-bar" role="progressbar"
<div class="progress-bar" role="progressbar"
style={fmt.Sprintf("width: %d%%", calculatePercent(int(host.DiskUsage), int(host.DiskCapacity)))}>
</div>
</div>
<small>{formatBytes(host.DiskUsage)}</small>
</div>
if host.RemoteSize > 0 {
<div class="mt-1">
<small class="text-muted"><i class="fas fa-cloud me-1"></i>{formatBytes(host.RemoteSize)} remote</small>
</div>
}
</td>
<td>
if host.PublicURL != "" {
@@ -186,6 +191,7 @@ templ ClusterVolumeServers(data dash.ClusterVolumeServersData) {
data-max-volumes={fmt.Sprintf("%d", host.MaxVolumes)}
data-disk-usage={fmt.Sprintf("%d", host.DiskUsage)}
data-disk-capacity={fmt.Sprintf("%d", host.DiskCapacity)}
data-remote-size={fmt.Sprintf("%d", host.RemoteSize)}
data-ec-volumes={fmt.Sprintf("%d", host.EcVolumes)}
data-ec-shards={fmt.Sprintf("%d", host.EcShards)}
data-last-heartbeat={host.LastHeartbeat.Format("2006-01-02 15:04:05")}>
@@ -205,6 +211,7 @@ templ ClusterVolumeServers(data dash.ClusterVolumeServersData) {
data-max-volumes={fmt.Sprintf("%d", host.MaxVolumes)}
data-disk-usage={fmt.Sprintf("%d", host.DiskUsage)}
data-disk-capacity={fmt.Sprintf("%d", host.DiskCapacity)}
data-remote-size={fmt.Sprintf("%d", host.RemoteSize)}
data-ec-volumes={fmt.Sprintf("%d", host.EcVolumes)}
data-ec-shards={fmt.Sprintf("%d", host.EcShards)}
data-last-heartbeat={host.LastHeartbeat.Format("2006-01-02 15:04:05")}>
@@ -260,6 +267,7 @@ templ ClusterVolumeServers(data dash.ClusterVolumeServersData) {
maxVolumes: parseInt(button.getAttribute('data-max-volumes')),
diskUsage: parseInt(button.getAttribute('data-disk-usage')),
diskCapacity: parseInt(button.getAttribute('data-disk-capacity')),
remoteSize: parseInt(button.getAttribute('data-remote-size')) || 0,
ecVolumes: parseInt(button.getAttribute('data-ec-volumes')),
ecShards: parseInt(button.getAttribute('data-ec-shards')),
lastHeartbeat: button.getAttribute('data-last-heartbeat')
@@ -314,6 +322,8 @@ templ ClusterVolumeServers(data dash.ClusterVolumeServersData) {
'</div>' +
'</td></tr>' +
'<tr><td><strong>Available Space:</strong></td><td>' + formatBytes(server.diskCapacity - server.diskUsage) + '</td></tr>' +
(server.remoteSize > 0 ?
'<tr><td><strong>Remote Data:</strong></td><td><i class="fas fa-cloud me-1"></i>' + formatBytes(server.remoteSize) + '</td></tr>' : '') +
'</table>' +
'</div>' +
'</div>' +
File diff suppressed because one or more lines are too long
+15 -6
View File
@@ -400,7 +400,12 @@ templ ClusterVolumes(data dash.ClusterVolumesData) {
</td>
if data.ShowDiskTypeColumn {
<td>
<span class="badge bg-primary">{volume.DiskType}</span>
<span class="badge bg-primary">{displayDiskType(volume.DiskType)}</span>
if volume.RemoteStorageName != "" {
<span class="badge bg-info text-white ms-1" title={volume.RemoteStorageKey}>
<i class="fas fa-cloud me-1"></i>{volume.RemoteStorageName}
</span>
}
</td>
}
if data.ShowVersionColumn {
@@ -683,14 +688,18 @@ func countUniqueRacks(volumes []dash.VolumeWithTopology) int {
return len(rackMap)
}
// displayDiskType maps the empty disk type to its display name.
func displayDiskType(diskType string) string {
if diskType == "" {
return "hdd"
}
return diskType
}
func countUniqueDiskTypes(volumes []dash.VolumeWithTopology) int {
diskTypeMap := make(map[string]bool)
for _, volume := range volumes {
diskType := volume.DiskType
if diskType == "" {
diskType = "hdd"
}
diskTypeMap[diskType] = true
diskTypeMap[displayDiskType(volume.DiskType)] = true
}
return len(diskTypeMap)
}
File diff suppressed because one or more lines are too long