diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 3654bf2b0..ef9b18f8b 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/seaweedfs/seaweedfs/weed/remote_storage" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket" @@ -18,6 +19,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/cluster" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/remote_pb" "google.golang.org/grpc" @@ -42,33 +44,35 @@ var ( ) type Filer struct { - UniqueFilerId int32 - UniqueFilerEpoch int32 - Store VirtualFilerStore - MasterClient *wdclient.MasterClient - FileIdDeletionQueue *util.UnboundedQueue - GrpcDialOption grpc.DialOption - DirBucketsPath string - Cipher bool - LocalMetaLogBuffer *log_buffer.LogBuffer - metaLogCollection string - metaLogReplication string - DefaultDiskType string - MetaAggregator *MetaAggregator - Signature int32 - FilerConf *FilerConf - placementOverlay PlacementOverlay - RemoteStorage *FilerRemoteStorage - lazyFetchGroup singleflight.Group - lazyListGroup singleflight.Group - Dlm *lock_manager.DistributedLockManager - MaxFilenameLength uint32 - deletionQuit chan struct{} - DeletionRetryQueue *DeletionRetryQueue - EmptyFolderCleaner *empty_folder_cleanup.EmptyFolderCleaner - EmptyFolderCleanupDelay time.Duration - persistedLogCache *persistedLogCache - metaLogInflight metaLogInflight + UniqueFilerId int32 + UniqueFilerEpoch int32 + Store VirtualFilerStore + MasterClient *wdclient.MasterClient + FileIdDeletionQueue *util.UnboundedQueue + GrpcDialOption grpc.DialOption + DirBucketsPath string + Cipher bool + LocalMetaLogBuffer *log_buffer.LogBuffer + metaLogCollection string + metaLogReplication string + DefaultDiskType string + MetaAggregator *MetaAggregator + Signature int32 + FilerConf *FilerConf + placementOverlay PlacementOverlay + RemoteStorage *FilerRemoteStorage + BuildGuardedRemoteClient RemoteStorageClientBuilder + AllowUntrustedRemoteEndpoints bool + lazyFetchGroup singleflight.Group + lazyListGroup singleflight.Group + Dlm *lock_manager.DistributedLockManager + MaxFilenameLength uint32 + deletionQuit chan struct{} + DeletionRetryQueue *DeletionRetryQueue + EmptyFolderCleaner *empty_folder_cleanup.EmptyFolderCleaner + EmptyFolderCleanupDelay time.Duration + persistedLogCache *persistedLogCache + metaLogInflight metaLogInflight } func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, filerGroup string, collection string, replication string, dataCenter string, maxFilenameLength uint32, notifyFn func()) *Filer { @@ -108,6 +112,13 @@ func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerH return f } +func (f *Filer) buildRemoteStorageClient(ctx context.Context, remoteConf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) { + if f.BuildGuardedRemoteClient != nil { + return f.BuildGuardedRemoteClient(ctx, remoteConf, f.AllowUntrustedRemoteEndpoints) + } + return remote_storage.GetRemoteStorage(remoteConf) +} + func (f *Filer) MaybeBootstrapFromOnePeer(self pb.ServerAddress, existingNodes []*master_pb.ClusterNodeUpdate, snapshotTime time.Time) (err error) { if len(existingNodes) == 0 { return diff --git a/weed/filer/filer_lazy_remote.go b/weed/filer/filer_lazy_remote.go index 801a3eac0..1b6983be3 100644 --- a/weed/filer/filer_lazy_remote.go +++ b/weed/filer/filer_lazy_remote.go @@ -43,7 +43,7 @@ func (f *Filer) maybeLazyFetchFromRemote(ctx context.Context, p util.FullPath) ( return nil, nil } - client, _, found := f.RemoteStorage.FindRemoteStorageClient(p) + remoteConf, found := f.RemoteStorage.FindRemoteStorageConf(p) if !found { return nil, nil } @@ -67,6 +67,12 @@ func (f *Filer) maybeLazyFetchFromRemote(ctx context.Context, p util.FullPath) ( key := string(p) val, err, _ := f.lazyFetchGroup.Do(key, func() (interface{}, error) { + buildCtx := context.WithoutCancel(ctx) + client, clientErr := f.buildRemoteStorageClient(buildCtx, remoteConf) + if clientErr != nil { + glog.V(1).InfofCtx(ctx, "maybeLazyFetchFromRemote: reject %s: %v", p, clientErr) + return lazyFetchResult{nil}, nil + } remoteEntry, statErr := client.StatFile(objectLoc) if statErr != nil { if errors.Is(statErr, remote_storage.ErrRemoteObjectNotFound) { @@ -126,10 +132,18 @@ func (f *Filer) maybeDeleteFromRemote(ctx context.Context, entry *Entry) (bool, return false, nil } - client, _, found := f.RemoteStorage.GetRemoteStorageClient(remoteLoc.Name) + if !entry.IsDirectory() && entry.Remote == nil { + return false, nil + } + + remoteConf, found := f.RemoteStorage.GetRemoteStorageConf(remoteLoc.Name) if !found { return false, fmt.Errorf("resolve remote storage client for %s: not found", entry.FullPath) } + client, clientErr := f.buildRemoteStorageClient(ctx, remoteConf) + if clientErr != nil { + return false, fmt.Errorf("resolve remote storage client for %s: %w", entry.FullPath, clientErr) + } if client == nil { return false, fmt.Errorf("resolve remote storage client for %s: initialization failed", entry.FullPath) } @@ -147,10 +161,6 @@ func (f *Filer) maybeDeleteFromRemote(ctx context.Context, entry *Entry) (bool, return true, nil } - if entry.Remote == nil { - return false, nil - } - if err := client.DeleteFile(objectLoc); err != nil { if errors.Is(err, remote_storage.ErrRemoteObjectNotFound) { return true, nil diff --git a/weed/filer/filer_lazy_remote_listing.go b/weed/filer/filer_lazy_remote_listing.go index 4965589d7..e292c208e 100644 --- a/weed/filer/filer_lazy_remote_listing.go +++ b/weed/filer/filer_lazy_remote_listing.go @@ -75,13 +75,18 @@ func (f *Filer) maybeLazyListFromRemote(ctx context.Context, p util.FullPath) { } } - client, _, found := f.RemoteStorage.FindRemoteStorageClient(lookupPath) + remoteConf, found := f.RemoteStorage.FindRemoteStorageConf(lookupPath) if !found { return } key := "list:" + string(p) f.lazyListGroup.Do(key, func() (interface{}, error) { + client, clientErr := f.buildRemoteStorageClient(context.WithoutCancel(ctx), remoteConf) + if clientErr != nil { + glog.V(1).InfofCtx(ctx, "maybeLazyListFromRemote: reject %s: %v", p, clientErr) + return nil, nil + } startTime := time.Now() objectLoc := MapFullPathToRemoteStorageLocation(mountDir, remoteLoc, p) diff --git a/weed/filer/filer_lazy_remote_test.go b/weed/filer/filer_lazy_remote_test.go index 7ac2f4f4d..f9b3442e0 100644 --- a/weed/filer/filer_lazy_remote_test.go +++ b/weed/filer/filer_lazy_remote_test.go @@ -483,6 +483,139 @@ func TestMaybeLazyFetchFromRemote_ContextGuardPreventsRecursion(t *testing.T) { assert.Equal(t, 0, countingStub.statCalls, "guard should prevent StatFile from being called") } +func TestMaybeLazyFetchFromRemote_GuardedClientRejectsEndpoint(t *testing.T) { + const storageType = "stub_lazy_guarded" + countingStub := &countingRemoteClient{ + stubRemoteClient: stubRemoteClient{ + statResult: &filer_pb.RemoteEntry{RemoteMtime: 1, RemoteSize: 1}, + }, + } + defer registerStubMaker(t, storageType, countingStub)() + + conf := &remote_pb.RemoteConf{Name: "guardedstore", Type: storageType} + rs := NewFilerRemoteStorage() + rs.storageNameToConf[conf.Name] = conf + rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{ + Name: "guardedstore", + Bucket: "mybucket", + Path: "/", + }) + + store := newStubFilerStore() + f := newTestFiler(t, store, rs) + f.BuildGuardedRemoteClient = func(ctx context.Context, _ *remote_pb.RemoteConf, _ bool) (remote_storage.RemoteStorageClient, error) { + return nil, fmt.Errorf("reject remote endpoint") + } + + entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/file.txt") + require.NoError(t, err) + assert.Nil(t, entry, "guarded rejection should yield no entry") + assert.Equal(t, 0, countingStub.statCalls, "guarded rejection should not reach the remote") +} + +func TestMaybeLazyListFromRemote_GuardedClientRejectsEndpoint(t *testing.T) { + const storageType = "stub_lazy_list_guarded" + stub := &stubRemoteClient{ + listDirFn: func(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) error { + return visitFn("/", "file.txt", false, &filer_pb.RemoteEntry{RemoteSize: 1}) + }, + } + defer registerStubMaker(t, storageType, stub)() + + conf := &remote_pb.RemoteConf{Name: "listguardedstore", Type: storageType} + rs := NewFilerRemoteStorage() + rs.storageNameToConf[conf.Name] = conf + rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{ + Name: "listguardedstore", + Bucket: "mybucket", + Path: "/", + ListingCacheTtlSeconds: 300, + }) + + store := newStubFilerStore() + f := newTestFiler(t, store, rs) + f.BuildGuardedRemoteClient = func(ctx context.Context, _ *remote_pb.RemoteConf, _ bool) (remote_storage.RemoteStorageClient, error) { + return nil, fmt.Errorf("reject remote endpoint") + } + + f.maybeLazyListFromRemote(context.Background(), util.FullPath("/buckets/mybucket")) + assert.Equal(t, 0, stub.listDirCalls, "guarded rejection should not reach the remote") +} + +func TestDeleteEntryMetaAndData_GuardedClientRejectsRemoteBackedFile(t *testing.T) { + const storageType = "stub_lazy_delete_guarded" + stub := &stubRemoteClient{} + defer registerStubMaker(t, storageType, stub)() + + conf := &remote_pb.RemoteConf{Name: "deleteguardedstore", Type: storageType} + rs := NewFilerRemoteStorage() + rs.storageNameToConf[conf.Name] = conf + rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{ + Name: "deleteguardedstore", + Bucket: "mybucket", + Path: "/", + }) + + store := newStubFilerStore() + filePath := util.FullPath("/buckets/mybucket/file.txt") + store.entries[string(filePath)] = &Entry{ + FullPath: filePath, + Attr: Attr{ + Mtime: time.Unix(1700000000, 0), + Crtime: time.Unix(1700000000, 0), + Mode: 0644, + FileSize: 64, + }, + Remote: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 64}, + } + f := newTestFiler(t, store, rs) + f.BuildGuardedRemoteClient = func(ctx context.Context, _ *remote_pb.RemoteConf, _ bool) (remote_storage.RemoteStorageClient, error) { + return nil, fmt.Errorf("reject remote endpoint") + } + + err := f.DeleteEntryMetaAndData(context.Background(), filePath, false, false, false, false, nil, 0) + require.Error(t, err, "guarded rejection should block the remote-backed delete") + assert.Len(t, stub.deleteCalls, 0, "guarded rejection should not reach the remote") +} + +func TestDeleteEntryMetaAndData_GuardedClientAllowsLocalOnlyFile(t *testing.T) { + const storageType = "stub_lazy_delete_local_only" + stub := &stubRemoteClient{} + defer registerStubMaker(t, storageType, stub)() + + conf := &remote_pb.RemoteConf{Name: "localonlystore", Type: storageType} + rs := NewFilerRemoteStorage() + rs.storageNameToConf[conf.Name] = conf + rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{ + Name: "localonlystore", + Bucket: "mybucket", + Path: "/", + }) + + store := newStubFilerStore() + filePath := util.FullPath("/buckets/mybucket/file.txt") + store.entries[string(filePath)] = &Entry{ + FullPath: filePath, + Attr: Attr{ + Mtime: time.Unix(1700000000, 0), + Crtime: time.Unix(1700000000, 0), + Mode: 0644, + FileSize: 64, + }, + // no Remote: a local-only file under the mount needs no remote delete + } + f := newTestFiler(t, store, rs) + f.BuildGuardedRemoteClient = func(ctx context.Context, _ *remote_pb.RemoteConf, _ bool) (remote_storage.RemoteStorageClient, error) { + return nil, fmt.Errorf("reject remote endpoint") + } + + err := f.DeleteEntryMetaAndData(context.Background(), filePath, false, false, false, false, nil, 0) + require.NoError(t, err, "local-only file should delete despite a rejected mount endpoint") + _, findErr := store.FindEntry(context.Background(), filePath) + require.ErrorIs(t, findErr, filer_pb.ErrNotFound, "local metadata should be deleted") + assert.Len(t, stub.deleteCalls, 0, "local-only file should not reach the remote") +} + func TestFindEntry_LazyFetchOnMiss(t *testing.T) { const storageType = "stub_lazy_findentry" stub := &stubRemoteClient{ diff --git a/weed/filer/remote_storage.go b/weed/filer/remote_storage.go index 121c81518..3a2043f39 100644 --- a/weed/filer/remote_storage.go +++ b/weed/filer/remote_storage.go @@ -22,14 +22,37 @@ import ( const REMOTE_STORAGE_CONF_SUFFIX = ".conf" const REMOTE_STORAGE_MOUNT_FILE = "mount.mapping" +// RemoteStorageConfValidator rejects a RemoteConf whose endpoint resolves to +// an address the filer must not dial (loopback / link-local / private / IMDS). +// The filer server injects the volume server's SSRF deny-list validator so the +// filer package — which cannot import the server package — applies the same +// check the volume server's BuildGuardedRemoteStorageClient does. A nil +// validator leaves the historical unguarded behavior for unit tests that +// never dial. +type RemoteStorageConfValidator func(ctx context.Context, conf *remote_pb.RemoteConf) error + type FilerRemoteStorage struct { // guards rules and storageNameToConf, which are replaced wholesale // whenever /etc/remote changes mu sync.RWMutex rules ptrie.Trie[*remote_pb.RemoteStorageLocation] storageNameToConf map[string]*remote_pb.RemoteConf + // confValidator, when set, is applied to every RemoteConf as it is loaded + // from /etc/remote. A conf that fails is dropped from storageNameToConf so + // the lazy-fetch, lazy-list, and remote-delete paths (which resolve clients + // by name) can never dial its endpoint. This closes the unauthenticated-plant + // SSRF: a conf written to /etc/remote with a loopback S3 endpoint is rejected + // at reload instead of being dialed on the next cache miss. + confValidator RemoteStorageConfValidator } +// RemoteStorageClientBuilder builds a remote-storage client for a conf. The +// filer server sets it to the guarded builder (endpoint deny-list + DNS +// rebinding-safe dialer) so the lazy-remote paths apply the same SSRF checks +// as the volume and streaming read paths. When nil the lazy paths fall back to +// the shared unguarded cache. +type RemoteStorageClientBuilder func(ctx context.Context, remoteConf *remote_pb.RemoteConf, allowUntrusted bool) (remote_storage.RemoteStorageClient, error) + func NewFilerRemoteStorage() (rs *FilerRemoteStorage) { rs = &FilerRemoteStorage{ rules: ptrie.New[*remote_pb.RemoteStorageLocation](), @@ -38,6 +61,15 @@ func NewFilerRemoteStorage() (rs *FilerRemoteStorage) { return rs } +// SetConfValidator installs the SSRF deny-list validator applied to every +// RemoteConf loaded from /etc/remote. It is called once by the filer server +// after construction. +func (rs *FilerRemoteStorage) SetConfValidator(v RemoteStorageConfValidator) { + rs.mu.Lock() + rs.confValidator = v + rs.mu.Unlock() +} + func (rs *FilerRemoteStorage) LoadRemoteStorageConfigurationsAndMapping(filer *Filer) (err error) { // execute this on filer @@ -71,6 +103,16 @@ func (rs *FilerRemoteStorage) LoadRemoteStorageConfigurationsAndMapping(filer *F if err := proto.Unmarshal(entry.Content, conf); err != nil { return fmt.Errorf("unmarshal %s/%s: %v", DirectoryEtcRemote, entry.Name(), err) } + if rs.confValidator != nil { + if vErr := rs.confValidator(context.Background(), conf); vErr != nil { + // Drop the conf rather than fail the whole load: a single bad conf + // must not evict the rest of /etc/remote, and the mount mapping that + // references it resolves to "no client" on the lazy paths instead + // of dialing the blocked endpoint. + glog.Warningf("reject remote storage conf %s/%s: %v", DirectoryEtcRemote, entry.Name(), vErr) + continue + } + } storageNameToConf[conf.Name] = conf } @@ -141,6 +183,21 @@ func (rs *FilerRemoteStorage) GetRemoteStorageClient(storageName string) (client return } +func (rs *FilerRemoteStorage) FindRemoteStorageConf(p util.FullPath) (*remote_pb.RemoteConf, bool) { + _, storageLocation := rs.FindMountDirectory(p) + if storageLocation == nil { + return nil, false + } + return rs.GetRemoteStorageConf(storageLocation.Name) +} + +func (rs *FilerRemoteStorage) GetRemoteStorageConf(storageName string) (*remote_pb.RemoteConf, bool) { + rs.mu.RLock() + defer rs.mu.RUnlock() + conf, found := rs.storageNameToConf[storageName] + return conf, found +} + func UnmarshalRemoteStorageMappings(oldContent []byte) (mappings *remote_pb.RemoteStorageMapping, err error) { mappings = &remote_pb.RemoteStorageMapping{ Mappings: make(map[string]*remote_pb.RemoteStorageLocation), diff --git a/weed/filer/remote_storage_test.go b/weed/filer/remote_storage_test.go index 04f12c742..9ef2dd5e8 100644 --- a/weed/filer/remote_storage_test.go +++ b/weed/filer/remote_storage_test.go @@ -1,11 +1,14 @@ package filer import ( + "context" "testing" "github.com/seaweedfs/seaweedfs/weed/pb/remote_pb" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" ) func TestFilerRemoteStorage_FindRemoteStorageClient(t *testing.T) { @@ -68,3 +71,61 @@ func TestFilerRemoteStorage_FindMountDirectory_LongestPrefixWins(t *testing.T) { } } } + +// TestLoadRemoteStorageConfigurationsAndMapping_RejectsBlockedEndpoint +// reproduces the unauthenticated-plant SSRF: a RemoteConf whose S3 endpoint is +// a loopback address is written under /etc/remote. With the SSRF deny-list +// validator injected (as the filer server does), the conf is dropped at load +// so the lazy-fetch path can never resolve a client for it; a conf whose +// endpoint passes validation is loaded as before. +func TestLoadRemoteStorageConfigurationsAndMapping_RejectsBlockedEndpoint(t *testing.T) { + loopbackConf := &remote_pb.RemoteConf{ + Name: "evil", + Type: "s3", + S3Endpoint: "http://127.0.0.1:8000", + S3Region: "us-east-1", + } + okConf := &remote_pb.RemoteConf{ + Name: "good", + Type: "s3", + S3Endpoint: "https://s3.example.com", + S3Region: "us-east-1", + } + loopbackBytes, _ := proto.Marshal(loopbackConf) + okBytes, _ := proto.Marshal(okConf) + + store := newStubFilerStore() + store.entries[string(DirectoryEtcRemote)+"/evil.conf"] = &Entry{ + FullPath: util.FullPath(string(DirectoryEtcRemote) + "/evil.conf"), + Attr: Attr{Mode: 0644}, + Content: loopbackBytes, + } + store.entries[string(DirectoryEtcRemote)+"/good.conf"] = &Entry{ + FullPath: util.FullPath(string(DirectoryEtcRemote) + "/good.conf"), + Attr: Attr{Mode: 0644}, + Content: okBytes, + } + + rs := NewFilerRemoteStorage() + // Validator mirrors the filer server injection: reject loopback endpoints. + rs.SetConfValidator(func(_ context.Context, conf *remote_pb.RemoteConf) error { + if conf.GetS3Endpoint() == "http://127.0.0.1:8000" { + return assertError("reject remote endpoint: loopback") + } + return nil + }) + f := newTestFiler(t, store, rs) + + require.NoError(t, f.RemoteStorage.LoadRemoteStorageConfigurationsAndMapping(f)) + + _, _, foundEvil := f.RemoteStorage.GetRemoteStorageClient("evil") + assert.False(t, foundEvil, "loopback-endpoint conf must be dropped at load") + _, _, foundGood := f.RemoteStorage.GetRemoteStorageClient("good") + assert.True(t, foundGood, "valid-endpoint conf must still be loaded") +} + +// assertError is a tiny error type so the test validator can return a sentinel +// without importing fmt. +type assertError string + +func (e assertError) Error() string { return string(e) } diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index d1abf8e56..7d998ad8f 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -22,6 +22,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/remote_pb" "github.com/seaweedfs/seaweedfs/weed/util" util_http "github.com/seaweedfs/seaweedfs/weed/util/http" @@ -228,6 +229,11 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) fs.filer = filer.NewFiler(*option.Masters, fs.grpcDialOption, option.Host, option.FilerGroup, option.Collection, option.DefaultReplication, option.DataCenter, maxFilenameLength, nil) fs.filer.Cipher = option.Cipher fs.filer.DefaultDiskType = option.DiskType + fs.filer.BuildGuardedRemoteClient = BuildGuardedRemoteStorageClient + fs.filer.AllowUntrustedRemoteEndpoints = option.AllowUntrustedRemoteEndpoints + fs.filer.RemoteStorage.SetConfValidator(func(ctx context.Context, conf *remote_pb.RemoteConf) error { + return ValidateRemoteConfForLoad(ctx, conf, option.AllowUntrustedRemoteEndpoints) + }) // we do not support IP whitelist right now https://github.com/seaweedfs/seaweedfs/issues/7094 if v.GetString("guard.white_list") != "" { glog.Warningf("filer: guard.white_list is configured but the IP whitelist feature is currently disabled. See https://github.com/seaweedfs/seaweedfs/issues/7094") diff --git a/weed/server/volume_grpc_remote.go b/weed/server/volume_grpc_remote.go index 8b32ccb66..770eb56a9 100644 --- a/weed/server/volume_grpc_remote.go +++ b/weed/server/volume_grpc_remote.go @@ -304,6 +304,9 @@ func guardedRemoteClient(remoteConf *remote_pb.RemoteConf) (endpoint string, mak return "", nil, false } if ep, isS3 := s3remote.S3CompatibleEndpoint(remoteConf); isS3 { + if ep == "" && remoteConf.Type == "s3" { + return "", nil, false + } return ep, func(httpClient *http.Client) (remote_storage.RemoteStorageClient, error) { return s3remote.MakeWithHTTPClient(remoteConf, httpClient) }, true @@ -411,6 +414,64 @@ func BuildGuardedRemoteStorageClient(ctx context.Context, remoteConf *remote_pb. return client, nil } +// ValidateRemoteConfForLoad applies the same SSRF deny-list and gcs credential +// checks BuildGuardedRemoteStorageClient enforces at dial time, but without +// building a client. It is injected into the filer's FilerRemoteStorage so a +// RemoteConf planted under /etc/remote is rejected at load — before the +// lazy-fetch / lazy-list / remote-delete paths can resolve and dial it. A conf +// whose type does not steer a caller-supplied endpoint (and so dials a fixed +// provider host) passes; allowUntrusted skips the check to mirror the volume +// server opt-out. +func ValidateRemoteConfForLoad(ctx context.Context, remoteConf *remote_pb.RemoteConf, allowUntrusted bool) error { + if remoteConf == nil { + return nil + } + if allowUntrusted { + return nil + } + if remoteConf.GetType() == "gcs" { + if credsErr := checkGcsCredentials(remoteConf.GetGcsGoogleApplicationCredentials()); credsErr != nil { + return fmt.Errorf("reject remote credentials: %w", credsErr) + } + } + if endpoint, _, ok := guardedRemoteClient(remoteConf); ok { + if validateErr := validateRemoteEndpointForLoad(endpoint); validateErr != nil { + return fmt.Errorf("reject remote endpoint: %w", validateErr) + } + } + return nil +} + +// validateRemoteEndpointForLoad applies the static parts of the SSRF deny-list +// (scheme, IMDS hostnames, IP-literal blocked addresses) without resolving +// hostnames. DNS resolution is left to BuildGuardedRemoteStorageClient at dial +// time, so a transient DNS failure during /etc/remote reload cannot drop a +// working mount from the live map. +func validateRemoteEndpointForLoad(endpoint string) error { + if strings.TrimSpace(endpoint) == "" { + return fmt.Errorf("remote endpoint is empty") + } + u, parseErr := url.Parse(endpoint) + if parseErr != nil { + return fmt.Errorf("parse remote endpoint %q: %w", endpoint, parseErr) + } + scheme := strings.ToLower(u.Scheme) + if scheme != "http" && scheme != "https" { + return fmt.Errorf("remote endpoint %q must use http or https, got %q", endpoint, u.Scheme) + } + host := u.Hostname() + if host == "" { + return fmt.Errorf("remote endpoint %q has no host", endpoint) + } + if _, ok := blockedIMDSHosts[strings.ToLower(host)]; ok { + return fmt.Errorf("remote endpoint %q targets instance metadata service", endpoint) + } + if ip := net.ParseIP(host); ip != nil { + return checkBlockedIP(endpoint, ip) + } + return nil +} + func (vs *VolumeServer) FetchAndWriteNeedle(ctx context.Context, req *volume_server_pb.FetchAndWriteNeedleRequest) (resp *volume_server_pb.FetchAndWriteNeedleResponse, err error) { if err := vs.checkGrpcAdminAuth(ctx); err != nil { return nil, err diff --git a/weed/server/volume_grpc_remote_test.go b/weed/server/volume_grpc_remote_test.go index 3347482f0..03bcf150d 100644 --- a/weed/server/volume_grpc_remote_test.go +++ b/weed/server/volume_grpc_remote_test.go @@ -705,3 +705,77 @@ func TestBuildGuardedRemoteStorageClient(t *testing.T) { t.Errorf("error must not leak the file path: %v", err) } } + +// TestValidateRemoteConfForLoad confirms the load-time validator (injected into +// the filer's FilerRemoteStorage) rejects a RemoteConf whose endpoint resolves +// to a blocked address, while allowUntrusted skips the check. A conf whose type +// dials a fixed provider host (no caller-supplied endpoint) passes. +func TestValidateRemoteConfForLoad(t *testing.T) { + loopbackS3 := &remote_pb.RemoteConf{ + Name: "poc", + Type: "s3", + S3Endpoint: "http://127.0.0.1:8000", + S3Region: "us-east-1", + } + if err := ValidateRemoteConfForLoad(context.Background(), loopbackS3, false); err == nil { + t.Error("expected a loopback s3 endpoint to be rejected at load") + } else if !strings.Contains(err.Error(), "reject remote endpoint") { + t.Errorf("error = %v, want reject remote endpoint", err) + } + // allowUntrusted mirrors the volume server opt-out. + if err := ValidateRemoteConfForLoad(context.Background(), loopbackS3, true); err != nil { + t.Errorf("allowUntrusted should accept the conf: %v", err) + } + // A non-S3-compatible type with no caller-supplied endpoint dials a fixed + // provider host, so there is nothing caller-influenced to deny. + fixedHost := &remote_pb.RemoteConf{Name: "fixed", Type: "gcs"} + if err := ValidateRemoteConfForLoad(context.Background(), fixedHost, false); err != nil { + t.Errorf("fixed-host provider should pass: %v", err) + } + // nil conf is a no-op. + if err := ValidateRemoteConfForLoad(context.Background(), nil, false); err != nil { + t.Errorf("nil conf should be a no-op: %v", err) + } + // A hostname endpoint is not resolved at load time (DNS is left to the + // build-time guard at dial), so it must pass even if it would resolve to a + // blocked address. This prevents transient DNS failures from disabling + // working mounts during /etc/remote reload. + hostnameS3 := &remote_pb.RemoteConf{ + Name: "host", + Type: "s3", + S3Endpoint: "http://internal.example.com", + S3Region: "us-east-1", + } + if err := ValidateRemoteConfForLoad(context.Background(), hostnameS3, false); err != nil { + t.Errorf("hostname endpoint should pass at load (DNS deferred to dial): %v", err) + } + // A standard AWS S3 config with no custom endpoint (empty S3Endpoint) has + // no caller-supplied endpoint to guard — the AWS SDK derives the regional + // endpoint. Both the load-time validator and the build-time guard must + // accept it so standard AWS S3 mounts keep working. + standardS3 := &remote_pb.RemoteConf{ + Name: "aws", + Type: "s3", + S3Region: "us-east-1", + } + if err := ValidateRemoteConfForLoad(context.Background(), standardS3, false); err != nil { + t.Errorf("standard AWS S3 (empty endpoint) should pass: %v", err) + } + if _, err := BuildGuardedRemoteStorageClient(context.Background(), standardS3, false); err != nil { + t.Errorf("standard AWS S3 (empty endpoint) should build: %v", err) + } + // A non-s3 S3-compatible type with an empty endpoint is a misconfiguration + // (the AWS SDK would derive an AWS endpoint). The guard must reject it + // rather than fall through to the unguarded cache. + aliyunNoEndpoint := &remote_pb.RemoteConf{ + Name: "aliyun", + Type: "aliyun", + AliyunRegion: "cn-hangzhou", + } + if err := ValidateRemoteConfForLoad(context.Background(), aliyunNoEndpoint, false); err == nil { + t.Error("aliyun with empty endpoint should be rejected at load") + } + if _, err := BuildGuardedRemoteStorageClient(context.Background(), aliyunNoEndpoint, false); err == nil { + t.Error("aliyun with empty endpoint should be rejected by the guard") + } +}