mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-08-28 03:46:24 +00:00
admin: show a copyable S3 object URL in the bucket file browser (#10933)
* admin: offer copyable S3 object URLs in the bucket file browser * admin: hide object urls when the bucket type lookup fails * admin: ignore an s3.public_endpoint that is not an absolute http url * mini: build the seeded s3 endpoint with JoinHostPort for ipv6 * admin: reject a query or fragment in s3.public_endpoint * mini: drop the seeded s3 endpoint when a later run disables s3 * admin: reject userinfo and bare delimiters in s3.public_endpoint, redact the warning * mini: pass its s3 endpoint as an admin option instead of mutating viper * admin: keep the rejected s3.public_endpoint value out of the log
This commit is contained in:
@@ -8,6 +8,7 @@ import (
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -160,11 +161,15 @@ type AdminServer struct {
|
||||
s3TablesManager *s3tables.Manager
|
||||
icebergPort int
|
||||
lancePort int
|
||||
|
||||
// s3PublicEndpoint is the client-facing S3 address from admin.toml
|
||||
// (s3.public_endpoint); discovered S3 servers are the fallback.
|
||||
s3PublicEndpoint string
|
||||
}
|
||||
|
||||
// Type definitions moved to types.go
|
||||
|
||||
func NewAdminServer(masters string, filerGroup string, templateFS http.FileSystem, dataDir string, icebergPort, lancePort int) *AdminServer {
|
||||
func NewAdminServer(masters string, filerGroup string, templateFS http.FileSystem, dataDir string, icebergPort, lancePort int, s3PublicEndpoint string) *AdminServer {
|
||||
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.admin")
|
||||
|
||||
// Create master client with multiple master support
|
||||
@@ -201,6 +206,7 @@ func NewAdminServer(masters string, filerGroup string, templateFS http.FileSyste
|
||||
s3TablesManager: newS3TablesManager(),
|
||||
icebergPort: icebergPort,
|
||||
lancePort: lancePort,
|
||||
s3PublicEndpoint: normalizeS3PublicEndpoint(s3PublicEndpoint),
|
||||
pluginLock: lockManager,
|
||||
adminPresenceLock: presenceLock,
|
||||
bgCancel: bgCancel,
|
||||
@@ -1547,6 +1553,32 @@ func (s *AdminServer) GetClusterS3Servers() (*ClusterS3ServersData, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetS3Endpoint returns the configured address clients reach the S3 gateway
|
||||
// at, or "". S3 servers register only their gRPC address with the master, so
|
||||
// the client-facing address cannot be discovered and must be configured.
|
||||
func (s *AdminServer) GetS3Endpoint() string {
|
||||
return s.s3PublicEndpoint
|
||||
}
|
||||
|
||||
// normalizeS3PublicEndpoint trims a trailing slash and drops, with a warning,
|
||||
// a value that is not a plain absolute http or https URL, so the file browser
|
||||
// hides its URL actions instead of copying broken links. Checking the string
|
||||
// for "?" and "#" rather than the parsed query and fragment also catches
|
||||
// delimiters with nothing after them, which url.Parse stores as empty.
|
||||
func normalizeS3PublicEndpoint(endpoint string) string {
|
||||
endpoint = strings.TrimRight(endpoint, "/")
|
||||
if endpoint == "" {
|
||||
return ""
|
||||
}
|
||||
u, err := url.Parse(endpoint)
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || strings.ContainsAny(endpoint, "?#") {
|
||||
// the value is not echoed: it may hold credentials in userinfo or a query
|
||||
glog.Warningf("ignoring s3.public_endpoint: expecting an http:// or https:// URL with a host and no credentials, query, or fragment")
|
||||
return ""
|
||||
}
|
||||
return endpoint
|
||||
}
|
||||
|
||||
// GetAllFilers method moved to client_management.go
|
||||
|
||||
// GetVolumeDetails method moved to volume_management.go
|
||||
@@ -1573,6 +1605,7 @@ func (as *AdminServer) GetConfigInfo(w http.ResponseWriter, r *http.Request) {
|
||||
configInfo["master_address"] = string(currentMaster)
|
||||
configInfo["cache_expiration"] = as.cacheExpiration.String()
|
||||
configInfo["filer_cache_expiration"] = as.filerCacheExpiration.String()
|
||||
configInfo["s3_public_endpoint"] = as.s3PublicEndpoint
|
||||
|
||||
// Add maintenance system info
|
||||
if as.maintenanceManager != nil {
|
||||
|
||||
@@ -2,6 +2,7 @@ package dash
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -46,6 +47,7 @@ type FileBrowserData struct {
|
||||
BucketName string `json:"bucket_name"`
|
||||
IsTableBucketPath bool `json:"is_table_bucket_path"`
|
||||
TableBucketName string `json:"table_bucket_name"`
|
||||
S3Endpoint string `json:"s3_endpoint,omitempty"`
|
||||
// Pagination fields
|
||||
PageSize int `json:"page_size"`
|
||||
HasNextPage bool `json:"has_next_page"`
|
||||
@@ -188,6 +190,7 @@ func (s *AdminServer) GetFileBrowser(dir string, prefix string, lastFileName str
|
||||
bucketName := ""
|
||||
isTableBucketPath := false
|
||||
tableBucketName := ""
|
||||
isRegularBucket := false
|
||||
if strings.HasPrefix(dir, "/buckets/") {
|
||||
isBucketPath = true
|
||||
pathParts := strings.Split(strings.Trim(dir, "/"), "/")
|
||||
@@ -205,6 +208,8 @@ func (s *AdminServer) GetFileBrowser(dir string, prefix string, lastFileName str
|
||||
if s3tables.IsTableBucketEntry(resp.Entry) {
|
||||
isTableBucketPath = true
|
||||
tableBucketName = bucketName
|
||||
} else {
|
||||
isRegularBucket = true
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
@@ -213,6 +218,11 @@ func (s *AdminServer) GetFileBrowser(dir string, prefix string, lastFileName str
|
||||
}
|
||||
}
|
||||
|
||||
s3Endpoint := ""
|
||||
if isRegularBucket {
|
||||
s3Endpoint = s.GetS3Endpoint()
|
||||
}
|
||||
|
||||
return &FileBrowserData{
|
||||
CurrentPath: dir,
|
||||
ParentPath: parentPath,
|
||||
@@ -224,6 +234,7 @@ func (s *AdminServer) GetFileBrowser(dir string, prefix string, lastFileName str
|
||||
BucketName: bucketName,
|
||||
IsTableBucketPath: isTableBucketPath,
|
||||
TableBucketName: tableBucketName,
|
||||
S3Endpoint: s3Endpoint,
|
||||
// Pagination metadata
|
||||
PageSize: pageSize,
|
||||
HasNextPage: hasNextPage,
|
||||
@@ -272,3 +283,52 @@ func (s *AdminServer) generateBreadcrumbs(dir string) []BreadcrumbItem {
|
||||
|
||||
return breadcrumbs
|
||||
}
|
||||
|
||||
// S3ObjectURL builds the path-style S3 URL for a filer path under /buckets/,
|
||||
// percent-encoding each path segment. Returns "" for other paths.
|
||||
func S3ObjectURL(endpoint, fullPath string) string {
|
||||
rel, ok := strings.CutPrefix(fullPath, "/buckets/")
|
||||
if !ok || rel == "" || endpoint == "" {
|
||||
return ""
|
||||
}
|
||||
segments := strings.Split(rel, "/")
|
||||
for i, segment := range segments {
|
||||
segments[i] = url.PathEscape(segment)
|
||||
}
|
||||
return strings.TrimRight(endpoint, "/") + "/" + strings.Join(segments, "/")
|
||||
}
|
||||
|
||||
// GetS3ObjectURL returns the S3 URL an object under a regular bucket is served
|
||||
// at, or "" when the path is not a bucket object, the bucket is an S3 Tables
|
||||
// bucket, or no S3 endpoint is known.
|
||||
func (s *AdminServer) GetS3ObjectURL(fullPath string) string {
|
||||
rel, ok := strings.CutPrefix(fullPath, "/buckets/")
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
bucketName, key, found := strings.Cut(rel, "/")
|
||||
if !found || key == "" {
|
||||
return ""
|
||||
}
|
||||
endpoint := s.GetS3Endpoint()
|
||||
if endpoint == "" {
|
||||
return ""
|
||||
}
|
||||
if err := s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: "/buckets",
|
||||
Name: bucketName,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if s3tables.IsTableBucketEntry(resp.Entry) {
|
||||
endpoint = ""
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
glog.V(1).Infof("object url bucket lookup failed for %s: %v", bucketName, err)
|
||||
return ""
|
||||
}
|
||||
return S3ObjectURL(endpoint, fullPath)
|
||||
}
|
||||
|
||||
@@ -93,6 +93,98 @@ func TestGenerateBreadcrumbs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestS3ObjectURL verifies path-style S3 URL construction with per-segment
|
||||
// percent-encoding
|
||||
func TestS3ObjectURL(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
fullPath string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "simple object",
|
||||
endpoint: "https://s3.example.com",
|
||||
fullPath: "/buckets/public-images/homarr/example.png",
|
||||
expected: "https://s3.example.com/public-images/homarr/example.png",
|
||||
},
|
||||
{
|
||||
name: "space hash and question mark in key",
|
||||
endpoint: "https://s3.example.com",
|
||||
fullPath: "/buckets/b/a b#c?d.txt",
|
||||
expected: "https://s3.example.com/b/a%20b%23c%3Fd.txt",
|
||||
},
|
||||
{
|
||||
name: "non-ascii key",
|
||||
endpoint: "https://s3.example.com",
|
||||
fullPath: "/buckets/b/图片.png",
|
||||
expected: "https://s3.example.com/b/%E5%9B%BE%E7%89%87.png",
|
||||
},
|
||||
{
|
||||
name: "percent in key",
|
||||
endpoint: "https://s3.example.com",
|
||||
fullPath: "/buckets/b/100%.txt",
|
||||
expected: "https://s3.example.com/b/100%25.txt",
|
||||
},
|
||||
{
|
||||
name: "trailing slash on endpoint",
|
||||
endpoint: "https://s3.example.com/",
|
||||
fullPath: "/buckets/b/k",
|
||||
expected: "https://s3.example.com/b/k",
|
||||
},
|
||||
{
|
||||
name: "not a bucket path",
|
||||
endpoint: "https://s3.example.com",
|
||||
fullPath: "/topics/t/k",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "empty endpoint",
|
||||
endpoint: "",
|
||||
fullPath: "/buckets/b/k",
|
||||
expected: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := S3ObjectURL(tt.endpoint, tt.fullPath); got != tt.expected {
|
||||
t.Errorf("S3ObjectURL(%q, %q) = %q, expected %q", tt.endpoint, tt.fullPath, got, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestNormalizeS3PublicEndpoint verifies unusable configured endpoints are
|
||||
// dropped rather than producing broken links
|
||||
func TestNormalizeS3PublicEndpoint(t *testing.T) {
|
||||
tests := []struct {
|
||||
endpoint string
|
||||
expected string
|
||||
}{
|
||||
{"https://s3.example.com", "https://s3.example.com"},
|
||||
{"http://10.0.0.1:8333/", "http://10.0.0.1:8333"},
|
||||
{"http://[::1]:8333", "http://[::1]:8333"},
|
||||
{"https://proxy.example.com/s3", "https://proxy.example.com/s3"},
|
||||
{"", ""},
|
||||
{"/", ""},
|
||||
{"s3.example.com", ""},
|
||||
{"ftp://s3.example.com", ""},
|
||||
{"https://", ""},
|
||||
{"https://s3.example.com?x=1", ""},
|
||||
{"https://s3.example.com/?", ""},
|
||||
{"https://s3.example.com#frag", ""},
|
||||
{"https://s3.example.com/#", ""},
|
||||
{"http://user:pass@s3.example.com", ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
if got := normalizeS3PublicEndpoint(tt.endpoint); got != tt.expected {
|
||||
t.Errorf("normalizeS3PublicEndpoint(%q) = %q, expected %q", tt.endpoint, got, tt.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPathHandlingWithForwardSlashes verifies that the production code
|
||||
// correctly handles paths with forward slashes (not OS-specific backslashes)
|
||||
func TestPathHandlingWithForwardSlashes(t *testing.T) {
|
||||
|
||||
@@ -631,6 +631,12 @@ func (h *FileBrowserHandlers) GetFileProperties(w http.ResponseWriter, r *http.R
|
||||
return
|
||||
}
|
||||
|
||||
if isDir, _ := properties["is_directory"].(bool); !isDir {
|
||||
if objectURL := h.adminServer.GetS3ObjectURL(filePath); objectURL != "" {
|
||||
properties["object_url"] = objectURL
|
||||
}
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, properties)
|
||||
}
|
||||
|
||||
|
||||
@@ -194,6 +194,11 @@ templ FileBrowser(data dash.FileBrowserData) {
|
||||
<button type="button" class="btn btn-outline-info btn-sm" title="View" data-action="view" data-path={ entry.FullPath }>
|
||||
<i class="fas fa-eye"></i>
|
||||
</button>
|
||||
if data.S3Endpoint != "" {
|
||||
<button type="button" class="btn btn-outline-secondary btn-sm" title="Copy S3 URL" data-action="copy-url" data-url={ dash.S3ObjectURL(data.S3Endpoint, entry.FullPath) }>
|
||||
<i class="fas fa-link"></i>
|
||||
</button>
|
||||
}
|
||||
}
|
||||
<button type="button" class="btn btn-outline-secondary btn-sm" title="Properties" data-action="properties" data-path={ entry.FullPath }>
|
||||
<i class="fas fa-info-circle"></i>
|
||||
@@ -363,8 +368,13 @@ templ FileBrowser(data dash.FileBrowserData) {
|
||||
if (!button) return;
|
||||
|
||||
const action = button.getAttribute('data-action');
|
||||
if (action === 'copy-url') {
|
||||
const url = button.getAttribute('data-url');
|
||||
if (url) adminCopyToClipboard(url);
|
||||
return;
|
||||
}
|
||||
const path = button.getAttribute('data-path');
|
||||
|
||||
|
||||
if (!path) return;
|
||||
|
||||
switch(action) {
|
||||
@@ -476,6 +486,10 @@ templ FileBrowser(data dash.FileBrowserData) {
|
||||
if (!data.is_directory) {
|
||||
html += '<tr><td><strong>Size:</strong></td><td>' + (data.size_formatted || (data.size ? formatBytes(data.size) : 'N/A')) + '</td></tr>' +
|
||||
'<tr><td><strong>MIME Type:</strong></td><td>' + (data.mime_type || 'N/A') + '</td></tr>';
|
||||
if (data.object_url) {
|
||||
html += '<tr><td><strong>Object URL:</strong></td><td><code class="text-break">' + data.object_url + '</code>' +
|
||||
' <button type="button" class="btn btn-outline-secondary btn-sm" title="Copy S3 URL" data-action="copy-url" data-url="' + data.object_url + '"><i class="fas fa-copy"></i></button></td></tr>';
|
||||
}
|
||||
}
|
||||
|
||||
html += '</table>' +
|
||||
|
||||
File diff suppressed because one or more lines are too long
+10
-1
@@ -64,6 +64,11 @@ type AdminOptions struct {
|
||||
// the caller. `weed mini` reserves the port this way because the admin
|
||||
// binds it only after every other service is up.
|
||||
workerGrpcListener net.Listener
|
||||
|
||||
// defaultS3PublicEndpoint, when set, is used for object URLs when
|
||||
// s3.public_endpoint is not configured. `weed mini` sets it to its own
|
||||
// S3 address.
|
||||
defaultS3PublicEndpoint string
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -396,7 +401,11 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool,
|
||||
r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", admin.StaticHandler()))
|
||||
|
||||
// Create admin server (plugin is always enabled)
|
||||
adminServer := dash.NewAdminServer(*options.master, *options.filerGroup, nil, dataDir, icebergPort, lancePort)
|
||||
s3PublicEndpoint := util.GetViper().GetString("s3.public_endpoint")
|
||||
if s3PublicEndpoint == "" {
|
||||
s3PublicEndpoint = options.defaultS3PublicEndpoint
|
||||
}
|
||||
adminServer := dash.NewAdminServer(*options.master, *options.filerGroup, nil, dataDir, icebergPort, lancePort, s3PublicEndpoint)
|
||||
|
||||
if err := adminServer.ApplyPluginConfigFromToml(util.GetViper()); err != nil {
|
||||
return fmt.Errorf("apply admin.toml to plugin config: %w", err)
|
||||
|
||||
@@ -1585,6 +1585,14 @@ func startMiniAdminWithWorker(allServicesReady chan struct{}) {
|
||||
// Set admin options
|
||||
*miniAdminOptions.master = masterAddr
|
||||
|
||||
// Mini knows its own S3 address, so the file browser can offer object
|
||||
// URLs without any configuration. Assigned unconditionally so a value
|
||||
// from a prior in-process run cannot outlive its S3 server.
|
||||
miniAdminOptions.defaultS3PublicEndpoint = ""
|
||||
if *miniEnableS3 {
|
||||
miniAdminOptions.defaultS3PublicEndpoint = "http://" + util.JoinHostPort(*miniIp, *miniS3Options.port)
|
||||
}
|
||||
|
||||
// Resolve admin credentials from security.toml [admin] / WEED_ADMIN_* env
|
||||
// vars, matching the standalone `weed admin` command.
|
||||
applyMiniAdminCredentialFallback(&miniAdminOptions)
|
||||
|
||||
@@ -11,6 +11,12 @@
|
||||
# Each value can also be set via environment variable, e.g.
|
||||
# export WEED_MAINTENANCE_VACUUM_GARBAGE_THRESHOLD=0.3
|
||||
|
||||
[s3]
|
||||
# The address clients reach the S3 gateway at, including the scheme.
|
||||
# The file browser uses it to offer copyable object URLs; without it,
|
||||
# the copy action is hidden.
|
||||
# public_endpoint = "https://s3.example.com"
|
||||
|
||||
[maintenance]
|
||||
# toggle the entire maintenance system (task detection and execution)
|
||||
# enabled = true
|
||||
|
||||
Reference in New Issue
Block a user