admin: show a copyable S3 object URL in the bucket file browser (#10933)

* admin: offer copyable S3 object URLs in the bucket file browser

* admin: hide object urls when the bucket type lookup fails

* admin: ignore an s3.public_endpoint that is not an absolute http url

* mini: build the seeded s3 endpoint with JoinHostPort for ipv6

* admin: reject a query or fragment in s3.public_endpoint

* mini: drop the seeded s3 endpoint when a later run disables s3

* admin: reject userinfo and bare delimiters in s3.public_endpoint, redact the warning

* mini: pass its s3 endpoint as an admin option instead of mutating viper

* admin: keep the rejected s3.public_endpoint value out of the log
This commit is contained in:
Chris Lu
2026-08-24 19:29:01 -07:00
committed by GitHub
parent 2a70532d0d
commit 4a2879abad
9 changed files with 335 additions and 88 deletions
+34 -1
View File
@@ -8,6 +8,7 @@ import (
"io"
"net"
"net/http"
"net/url"
"sort"
"strings"
"sync"
@@ -160,11 +161,15 @@ type AdminServer struct {
s3TablesManager *s3tables.Manager
icebergPort int
lancePort int
// s3PublicEndpoint is the client-facing S3 address from admin.toml
// (s3.public_endpoint); discovered S3 servers are the fallback.
s3PublicEndpoint string
}
// Type definitions moved to types.go
func NewAdminServer(masters string, filerGroup string, templateFS http.FileSystem, dataDir string, icebergPort, lancePort int) *AdminServer {
func NewAdminServer(masters string, filerGroup string, templateFS http.FileSystem, dataDir string, icebergPort, lancePort int, s3PublicEndpoint string) *AdminServer {
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.admin")
// Create master client with multiple master support
@@ -201,6 +206,7 @@ func NewAdminServer(masters string, filerGroup string, templateFS http.FileSyste
s3TablesManager: newS3TablesManager(),
icebergPort: icebergPort,
lancePort: lancePort,
s3PublicEndpoint: normalizeS3PublicEndpoint(s3PublicEndpoint),
pluginLock: lockManager,
adminPresenceLock: presenceLock,
bgCancel: bgCancel,
@@ -1547,6 +1553,32 @@ func (s *AdminServer) GetClusterS3Servers() (*ClusterS3ServersData, error) {
}, nil
}
// GetS3Endpoint returns the configured address clients reach the S3 gateway
// at, or "". S3 servers register only their gRPC address with the master, so
// the client-facing address cannot be discovered and must be configured.
func (s *AdminServer) GetS3Endpoint() string {
return s.s3PublicEndpoint
}
// normalizeS3PublicEndpoint trims a trailing slash and drops, with a warning,
// a value that is not a plain absolute http or https URL, so the file browser
// hides its URL actions instead of copying broken links. Checking the string
// for "?" and "#" rather than the parsed query and fragment also catches
// delimiters with nothing after them, which url.Parse stores as empty.
func normalizeS3PublicEndpoint(endpoint string) string {
endpoint = strings.TrimRight(endpoint, "/")
if endpoint == "" {
return ""
}
u, err := url.Parse(endpoint)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || strings.ContainsAny(endpoint, "?#") {
// the value is not echoed: it may hold credentials in userinfo or a query
glog.Warningf("ignoring s3.public_endpoint: expecting an http:// or https:// URL with a host and no credentials, query, or fragment")
return ""
}
return endpoint
}
// GetAllFilers method moved to client_management.go
// GetVolumeDetails method moved to volume_management.go
@@ -1573,6 +1605,7 @@ func (as *AdminServer) GetConfigInfo(w http.ResponseWriter, r *http.Request) {
configInfo["master_address"] = string(currentMaster)
configInfo["cache_expiration"] = as.cacheExpiration.String()
configInfo["filer_cache_expiration"] = as.filerCacheExpiration.String()
configInfo["s3_public_endpoint"] = as.s3PublicEndpoint
// Add maintenance system info
if as.maintenanceManager != nil {
+60
View File
@@ -2,6 +2,7 @@ package dash
import (
"context"
"net/url"
"path"
"strings"
"time"
@@ -46,6 +47,7 @@ type FileBrowserData struct {
BucketName string `json:"bucket_name"`
IsTableBucketPath bool `json:"is_table_bucket_path"`
TableBucketName string `json:"table_bucket_name"`
S3Endpoint string `json:"s3_endpoint,omitempty"`
// Pagination fields
PageSize int `json:"page_size"`
HasNextPage bool `json:"has_next_page"`
@@ -188,6 +190,7 @@ func (s *AdminServer) GetFileBrowser(dir string, prefix string, lastFileName str
bucketName := ""
isTableBucketPath := false
tableBucketName := ""
isRegularBucket := false
if strings.HasPrefix(dir, "/buckets/") {
isBucketPath = true
pathParts := strings.Split(strings.Trim(dir, "/"), "/")
@@ -205,6 +208,8 @@ func (s *AdminServer) GetFileBrowser(dir string, prefix string, lastFileName str
if s3tables.IsTableBucketEntry(resp.Entry) {
isTableBucketPath = true
tableBucketName = bucketName
} else {
isRegularBucket = true
}
return nil
}); err != nil {
@@ -213,6 +218,11 @@ func (s *AdminServer) GetFileBrowser(dir string, prefix string, lastFileName str
}
}
s3Endpoint := ""
if isRegularBucket {
s3Endpoint = s.GetS3Endpoint()
}
return &FileBrowserData{
CurrentPath: dir,
ParentPath: parentPath,
@@ -224,6 +234,7 @@ func (s *AdminServer) GetFileBrowser(dir string, prefix string, lastFileName str
BucketName: bucketName,
IsTableBucketPath: isTableBucketPath,
TableBucketName: tableBucketName,
S3Endpoint: s3Endpoint,
// Pagination metadata
PageSize: pageSize,
HasNextPage: hasNextPage,
@@ -272,3 +283,52 @@ func (s *AdminServer) generateBreadcrumbs(dir string) []BreadcrumbItem {
return breadcrumbs
}
// S3ObjectURL builds the path-style S3 URL for a filer path under /buckets/,
// percent-encoding each path segment. Returns "" for other paths.
func S3ObjectURL(endpoint, fullPath string) string {
rel, ok := strings.CutPrefix(fullPath, "/buckets/")
if !ok || rel == "" || endpoint == "" {
return ""
}
segments := strings.Split(rel, "/")
for i, segment := range segments {
segments[i] = url.PathEscape(segment)
}
return strings.TrimRight(endpoint, "/") + "/" + strings.Join(segments, "/")
}
// GetS3ObjectURL returns the S3 URL an object under a regular bucket is served
// at, or "" when the path is not a bucket object, the bucket is an S3 Tables
// bucket, or no S3 endpoint is known.
func (s *AdminServer) GetS3ObjectURL(fullPath string) string {
rel, ok := strings.CutPrefix(fullPath, "/buckets/")
if !ok {
return ""
}
bucketName, key, found := strings.Cut(rel, "/")
if !found || key == "" {
return ""
}
endpoint := s.GetS3Endpoint()
if endpoint == "" {
return ""
}
if err := s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
resp, err := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{
Directory: "/buckets",
Name: bucketName,
})
if err != nil {
return err
}
if s3tables.IsTableBucketEntry(resp.Entry) {
endpoint = ""
}
return nil
}); err != nil {
glog.V(1).Infof("object url bucket lookup failed for %s: %v", bucketName, err)
return ""
}
return S3ObjectURL(endpoint, fullPath)
}
+92
View File
@@ -93,6 +93,98 @@ func TestGenerateBreadcrumbs(t *testing.T) {
}
}
// TestS3ObjectURL verifies path-style S3 URL construction with per-segment
// percent-encoding
func TestS3ObjectURL(t *testing.T) {
tests := []struct {
name string
endpoint string
fullPath string
expected string
}{
{
name: "simple object",
endpoint: "https://s3.example.com",
fullPath: "/buckets/public-images/homarr/example.png",
expected: "https://s3.example.com/public-images/homarr/example.png",
},
{
name: "space hash and question mark in key",
endpoint: "https://s3.example.com",
fullPath: "/buckets/b/a b#c?d.txt",
expected: "https://s3.example.com/b/a%20b%23c%3Fd.txt",
},
{
name: "non-ascii key",
endpoint: "https://s3.example.com",
fullPath: "/buckets/b/图片.png",
expected: "https://s3.example.com/b/%E5%9B%BE%E7%89%87.png",
},
{
name: "percent in key",
endpoint: "https://s3.example.com",
fullPath: "/buckets/b/100%.txt",
expected: "https://s3.example.com/b/100%25.txt",
},
{
name: "trailing slash on endpoint",
endpoint: "https://s3.example.com/",
fullPath: "/buckets/b/k",
expected: "https://s3.example.com/b/k",
},
{
name: "not a bucket path",
endpoint: "https://s3.example.com",
fullPath: "/topics/t/k",
expected: "",
},
{
name: "empty endpoint",
endpoint: "",
fullPath: "/buckets/b/k",
expected: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := S3ObjectURL(tt.endpoint, tt.fullPath); got != tt.expected {
t.Errorf("S3ObjectURL(%q, %q) = %q, expected %q", tt.endpoint, tt.fullPath, got, tt.expected)
}
})
}
}
// TestNormalizeS3PublicEndpoint verifies unusable configured endpoints are
// dropped rather than producing broken links
func TestNormalizeS3PublicEndpoint(t *testing.T) {
tests := []struct {
endpoint string
expected string
}{
{"https://s3.example.com", "https://s3.example.com"},
{"http://10.0.0.1:8333/", "http://10.0.0.1:8333"},
{"http://[::1]:8333", "http://[::1]:8333"},
{"https://proxy.example.com/s3", "https://proxy.example.com/s3"},
{"", ""},
{"/", ""},
{"s3.example.com", ""},
{"ftp://s3.example.com", ""},
{"https://", ""},
{"https://s3.example.com?x=1", ""},
{"https://s3.example.com/?", ""},
{"https://s3.example.com#frag", ""},
{"https://s3.example.com/#", ""},
{"http://user:pass@s3.example.com", ""},
}
for _, tt := range tests {
if got := normalizeS3PublicEndpoint(tt.endpoint); got != tt.expected {
t.Errorf("normalizeS3PublicEndpoint(%q) = %q, expected %q", tt.endpoint, got, tt.expected)
}
}
}
// TestPathHandlingWithForwardSlashes verifies that the production code
// correctly handles paths with forward slashes (not OS-specific backslashes)
func TestPathHandlingWithForwardSlashes(t *testing.T) {
@@ -631,6 +631,12 @@ func (h *FileBrowserHandlers) GetFileProperties(w http.ResponseWriter, r *http.R
return
}
if isDir, _ := properties["is_directory"].(bool); !isDir {
if objectURL := h.adminServer.GetS3ObjectURL(filePath); objectURL != "" {
properties["object_url"] = objectURL
}
}
writeJSON(w, http.StatusOK, properties)
}
+15 -1
View File
@@ -194,6 +194,11 @@ templ FileBrowser(data dash.FileBrowserData) {
<button type="button" class="btn btn-outline-info btn-sm" title="View" data-action="view" data-path={ entry.FullPath }>
<i class="fas fa-eye"></i>
</button>
if data.S3Endpoint != "" {
<button type="button" class="btn btn-outline-secondary btn-sm" title="Copy S3 URL" data-action="copy-url" data-url={ dash.S3ObjectURL(data.S3Endpoint, entry.FullPath) }>
<i class="fas fa-link"></i>
</button>
}
}
<button type="button" class="btn btn-outline-secondary btn-sm" title="Properties" data-action="properties" data-path={ entry.FullPath }>
<i class="fas fa-info-circle"></i>
@@ -363,8 +368,13 @@ templ FileBrowser(data dash.FileBrowserData) {
if (!button) return;
const action = button.getAttribute('data-action');
if (action === 'copy-url') {
const url = button.getAttribute('data-url');
if (url) adminCopyToClipboard(url);
return;
}
const path = button.getAttribute('data-path');
if (!path) return;
switch(action) {
@@ -476,6 +486,10 @@ templ FileBrowser(data dash.FileBrowserData) {
if (!data.is_directory) {
html += '<tr><td><strong>Size:</strong></td><td>' + (data.size_formatted || (data.size ? formatBytes(data.size) : 'N/A')) + '</td></tr>' +
'<tr><td><strong>MIME Type:</strong></td><td>' + (data.mime_type || 'N/A') + '</td></tr>';
if (data.object_url) {
html += '<tr><td><strong>Object URL:</strong></td><td><code class="text-break">' + data.object_url + '</code>' +
' <button type="button" class="btn btn-outline-secondary btn-sm" title="Copy S3 URL" data-action="copy-url" data-url="' + data.object_url + '"><i class="fas fa-copy"></i></button></td></tr>';
}
}
html += '</table>' +
File diff suppressed because one or more lines are too long
+10 -1
View File
@@ -64,6 +64,11 @@ type AdminOptions struct {
// the caller. `weed mini` reserves the port this way because the admin
// binds it only after every other service is up.
workerGrpcListener net.Listener
// defaultS3PublicEndpoint, when set, is used for object URLs when
// s3.public_endpoint is not configured. `weed mini` sets it to its own
// S3 address.
defaultS3PublicEndpoint string
}
func init() {
@@ -396,7 +401,11 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool,
r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", admin.StaticHandler()))
// Create admin server (plugin is always enabled)
adminServer := dash.NewAdminServer(*options.master, *options.filerGroup, nil, dataDir, icebergPort, lancePort)
s3PublicEndpoint := util.GetViper().GetString("s3.public_endpoint")
if s3PublicEndpoint == "" {
s3PublicEndpoint = options.defaultS3PublicEndpoint
}
adminServer := dash.NewAdminServer(*options.master, *options.filerGroup, nil, dataDir, icebergPort, lancePort, s3PublicEndpoint)
if err := adminServer.ApplyPluginConfigFromToml(util.GetViper()); err != nil {
return fmt.Errorf("apply admin.toml to plugin config: %w", err)
+8
View File
@@ -1585,6 +1585,14 @@ func startMiniAdminWithWorker(allServicesReady chan struct{}) {
// Set admin options
*miniAdminOptions.master = masterAddr
// Mini knows its own S3 address, so the file browser can offer object
// URLs without any configuration. Assigned unconditionally so a value
// from a prior in-process run cannot outlive its S3 server.
miniAdminOptions.defaultS3PublicEndpoint = ""
if *miniEnableS3 {
miniAdminOptions.defaultS3PublicEndpoint = "http://" + util.JoinHostPort(*miniIp, *miniS3Options.port)
}
// Resolve admin credentials from security.toml [admin] / WEED_ADMIN_* env
// vars, matching the standalone `weed admin` command.
applyMiniAdminCredentialFallback(&miniAdminOptions)
+6
View File
@@ -11,6 +11,12 @@
# Each value can also be set via environment variable, e.g.
# export WEED_MAINTENANCE_VACUUM_GARBAGE_THRESHOLD=0.3
[s3]
# The address clients reach the S3 gateway at, including the scheme.
# The file browser uses it to offer copyable object URLs; without it,
# the copy action is hidden.
# public_endpoint = "https://s3.example.com"
[maintenance]
# toggle the entire maintenance system (task detection and execution)
# enabled = true