Compare commits

..
Author SHA1 Message Date
Chris Lu a4bf9fe47f test(remote_storage/s3): assert WriteFile success in mime tests 2026-05-27 12:15:21 -07:00
Chris Lu b3ea57b5d7 fix(remote_storage/s3): forward entry.Attributes.Mime as ContentType
filer.remote.sync was uploading every object without a Content-Type, so
S3-compatible backends (e.g. Backblaze B2) stored binary/octet-stream
and browsers refused to render HTML, CSS, etc.

Pass entry.Attributes.Mime through to UploadInput.ContentType, leaving
the header unset when no Mime is recorded so the remote keeps its own
default behavior.
2026-05-27 10:34:11 -07:00
127 changed files with 9599 additions and 4124 deletions
+137
View File
@@ -0,0 +1,137 @@
name: "NFS Integration Tests"
on:
push:
branches: [ master, main ]
paths:
- 'weed/server/nfs/**'
- 'weed/command/nfs.go'
- 'weed/filer/filer_inode.go'
- 'weed/filer/filer_inode_index.go'
- 'weed/filer/filerstore_wrapper.go'
- 'weed/server/filer_grpc_server_rename.go'
- 'test/nfs/**'
- '.github/workflows/nfs-tests.yml'
pull_request:
branches: [ master, main ]
paths:
- 'weed/server/nfs/**'
- 'weed/command/nfs.go'
- 'weed/filer/filer_inode.go'
- 'weed/filer/filer_inode_index.go'
- 'weed/filer/filerstore_wrapper.go'
- 'weed/server/filer_grpc_server_rename.go'
- 'test/nfs/**'
- '.github/workflows/nfs-tests.yml'
concurrency:
group: ${{ github.head_ref }}/nfs-tests
cancel-in-progress: true
permissions:
contents: read
env:
TEST_TIMEOUT: '15m'
jobs:
nfs-integration:
name: NFS Integration Testing
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Build SeaweedFS
run: |
cd weed
go build -o weed .
chmod +x weed
./weed version
- name: Run NFS Integration Tests
run: |
cd test/nfs
echo "Running NFS integration tests..."
echo "============================================"
# Install test dependencies
go mod download
# Run the protocol-layer tests. The kernel-mount tests require root
# for mount(2) and are exercised in their own privileged step below;
# skip them here so a "skipped because not root" line doesn't show
# up as noise on every CI run.
go test -v -timeout=${{ env.TEST_TIMEOUT }} -skip '^TestKernelMount' ./...
echo "============================================"
echo "NFS integration tests completed"
- name: Install kernel NFS client
run: |
# nfs-common provides mount.nfs; netbase provides /etc/protocols
# which mount.nfs's protocol-name lookups (`tcp`, `udp`) need.
sudo apt-get update
sudo apt-get install -y nfs-common netbase
- name: Run kernel-mount E2E tests
run: |
cd test/nfs
echo "Running kernel-mount end-to-end tests..."
echo "These mount the running 'weed nfs' subprocess via the actual"
echo "Linux NFS client to catch protocol regressions invisible to"
echo "the go-nfs-client-based tests above."
echo "============================================"
# mount(2) is privileged. Preserve PATH so 'go' (and the weed
# binary that test/nfs/framework.go locates via $PATH) resolve
# correctly under sudo, and pass through the Go module/cache dirs
# so we don't redownload modules under root.
sudo env "PATH=$PATH" \
GOMODCACHE="$(go env GOMODCACHE)" \
GOCACHE="$(go env GOCACHE)" \
go test -v -timeout=${{ env.TEST_TIMEOUT }} -run '^TestKernelMount' ./...
echo "============================================"
echo "Kernel-mount E2E tests completed"
- name: Test Summary
if: always()
run: |
echo "## NFS Integration Test Summary" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
echo "- **Read/Write Round Trip**: Basic file create + read" >> $GITHUB_STEP_SUMMARY
echo "- **Directory Operations**: Mkdir, ReadDirPlus, RmDir" >> $GITHUB_STEP_SUMMARY
echo "- **Nested Directories**: Deep tree creation and leaf I/O" >> $GITHUB_STEP_SUMMARY
echo "- **Rename**: Content preserved across rename" >> $GITHUB_STEP_SUMMARY
echo "- **Overwrite + Truncate**: Setattr(size=0) + shorter write" >> $GITHUB_STEP_SUMMARY
echo "- **Large Files**: 3 MiB binary round trip" >> $GITHUB_STEP_SUMMARY
echo "- **Edge Payloads**: All 256 byte values + empty files" >> $GITHUB_STEP_SUMMARY
echo "- **Symlinks**: Symlink + Lookup" >> $GITHUB_STEP_SUMMARY
echo "- **Missing Path**: Remove on missing entry errors cleanly" >> $GITHUB_STEP_SUMMARY
echo "- **FSINFO**: Non-zero rtpref/wtpref advertised" >> $GITHUB_STEP_SUMMARY
echo "- **Sequential Append**: Two-part concatenation" >> $GITHUB_STEP_SUMMARY
echo "- **ReadDir After Remove**: Meta cache does not serve stale entries" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Kernel-Mount E2E Coverage" >> $GITHUB_STEP_SUMMARY
echo "- **V3 over TCP**: baseline NFSv3 mount + readdir" >> $GITHUB_STEP_SUMMARY
echo "- **V3 with mountproto=udp**: regression test for UDP MOUNT v3 responder" >> $GITHUB_STEP_SUMMARY
echo "- **V4 rejects cleanly**: regression test for the v4 PROG_MISMATCH path (#9262)" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Harness" >> $GITHUB_STEP_SUMMARY
echo "Most tests boot their own master + volume + filer + nfs subprocess" >> $GITHUB_STEP_SUMMARY
echo "stack on loopback and drive it via the NFSv3 RPC protocol using" >> $GITHUB_STEP_SUMMARY
echo "go-nfs-client. The kernel-mount E2E tests reuse the same harness" >> $GITHUB_STEP_SUMMARY
echo "but mount the export through the in-tree Linux NFS client to" >> $GITHUB_STEP_SUMMARY
echo "catch protocol regressions a Go-only client can't see; they run" >> $GITHUB_STEP_SUMMARY
echo "in a separate privileged step (mount(2) requires root)." >> $GITHUB_STEP_SUMMARY
+1 -3
View File
@@ -35,7 +35,6 @@ Your support will be really appreciated by me and other supporters!
[![nodion](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_nodion.png)](https://www.nodion.com)
[![piknik](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/piknik.png)](https://www.piknik.com)
[![keepsec](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/keepsec.png)](https://www.keepsec.ca)
[![zyner](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_zyner.png)](https://zyner.org)
---
@@ -655,8 +654,7 @@ Cluster Total: 3302.88 MiB/s, 550.51 obj/s over 43s.
## Enterprise ##
For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition,
which has advanced features, including data recovery, self-healing storage,
customizable erasure coding, EC vacuum and repair, etc.
which has a self-healing storage format with better data protection.
[Back to TOC](#table-of-contents)
-3
View File
@@ -127,9 +127,6 @@ test_tarantool: tags = tarantool
test_tarantool: build_tarantool_dev_env build
docker compose -f compose/test-tarantool-filer.yml -p seaweedfs up
test_keycloak_s3: build
docker compose -f compose/test-keycloak-s3.yml -p seaweedfs up
clean:
rm ./weed
-46
View File
@@ -1,46 +0,0 @@
services:
keycloak:
image: quay.io/keycloak/keycloak:26.0.7
command: ["start-dev", "--import-realm"]
environment:
KC_BOOTSTRAP_ADMIN_USERNAME: admin
KC_BOOTSTRAP_ADMIN_PASSWORD: admin
KC_HTTP_ENABLED: "true"
KC_HOSTNAME: "http://keycloak:8080"
KC_HOSTNAME_STRICT: "false"
ports:
- "8080:8080"
volumes:
- ../../test/s3/iam/seaweedfs-test-realm.json:/opt/keycloak/data/import/seaweedfs-test-realm.json:ro
healthcheck:
test: ["CMD-SHELL", "exec 3<>/dev/tcp/localhost/8080"]
interval: 10s
timeout: 5s
retries: 5
start_period: 60s
s3:
image: chrislusf/seaweedfs:local
command:
- -v=9
- server
- -ip=s3
- -filer
- -master.volumeSizeLimitMB=16
- -volume
- -volume.max=0
- -volume.preStopSeconds=1
- -s3
- -s3.port=8333
- -s3.config=/etc/seaweedfs/s3_config.json
- -s3.iam.config=/etc/seaweedfs/iam_config.json
environment:
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
volumes:
- ../../test/s3/iam/test_config.json:/etc/seaweedfs/s3_config.json:ro
- ../../test/s3/iam/iam_config_docker.json:/etc/seaweedfs/iam_config.json:ro
ports:
- "8333:8333"
depends_on:
keycloak:
condition: service_healthy
+4 -1
View File
@@ -129,6 +129,7 @@ require (
github.com/cognusion/imaging v1.0.3
github.com/fluent/fluent-logger-golang v1.10.1
github.com/getsentry/sentry-go v0.44.1
github.com/go-git/go-billy/v5 v5.9.0
github.com/go-ldap/ldap/v3 v3.4.13
github.com/golang-jwt/jwt/v5 v5.3.1
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
@@ -151,6 +152,8 @@ require (
github.com/tarantool/go-tarantool/v2 v2.4.2
github.com/testcontainers/testcontainers-go v0.40.0
github.com/tikv/client-go/v2 v2.0.7
github.com/willscott/go-nfs v0.0.4
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886
github.com/xeipuuv/gojsonschema v1.2.0
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2
@@ -208,7 +211,6 @@ require (
github.com/dromara/dongle v1.0.1 // indirect
github.com/gin-gonic/gin v1.11.0 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
github.com/go-git/go-billy/v5 v5.9.0 // indirect
github.com/goccy/go-yaml v1.18.0 // indirect
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
github.com/google/go-cmp v0.7.0 // indirect
@@ -255,6 +257,7 @@ require (
github.com/pquerna/otp v1.5.0 // indirect
github.com/pterm/pterm v0.12.82 // indirect
github.com/quic-go/qpack v0.6.0 // indirect
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
github.com/rclone/Proton-API-Bridge v1.0.3 // indirect
github.com/rclone/go-proton-api v1.0.2 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
+6
View File
@@ -1783,6 +1783,8 @@ github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SA
github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
github.com/rabbitmq/amqp091-go v1.11.0 h1:HxIctVm9Gid/Vtn706necmZ7Wj6pgGI2eqplRbEY8O8=
github.com/rabbitmq/amqp091-go v1.11.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
github.com/rclone/Proton-API-Bridge v1.0.3 h1:Bs7RC4xCFSN0BPIYVda/BNxp0qo3NV0gB2VZqx2KIew=
github.com/rclone/Proton-API-Bridge v1.0.3/go.mod h1:26RAest751Ofk+F/d8xtl4UyWXrZvMQwn39U8rm/WKM=
github.com/rclone/go-proton-api v1.0.2 h1:cJtJUab0MGJ3C6q5kiEJs3pbyhSLnOKMyYOQehA0PBc=
@@ -2026,6 +2028,10 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
github.com/willscott/go-nfs v0.0.4 h1:1vpOPAdECmoT2KmZ8u+ukO/jfvDjMEUNYhA2F1jGJtI=
github.com/willscott/go-nfs v0.0.4/go.mod h1:VhNccO67Oug787VNXcyx9JDI3ZoSpqoKMT/lWMhUIDg=
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886 h1:DtrBtkgTJk2XGt4T7eKdKVkd9A5NCevN2e4inLXtsqA=
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886/go.mod h1:Tq++Lr/FgiS3X48q5FETemXiSLGuYMQT2sPjYNPJSwA=
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "4.30"
appVersion: "4.29"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.30.0
version: 4.29.0
+1 -1
View File
@@ -384,4 +384,4 @@ helm install seaweedfs seaweedfs/seaweedfs \
## Enterprise
For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition,
which has advanced features, including data recovery, self-healing storage, customizable erasure coding, EC vacuum and repair, etc.
which has a self-healing storage format with better data protection.
Binary file not shown.

Before

Width:  |  Height:  |  Size: 746 B

+2 -2
View File
@@ -131,8 +131,8 @@ impl DiskLocation {
volume_id = vid.0,
"volume was not completed: {}, removing files", note
);
remove_volume_files(&volume_name, false);
remove_volume_files(&idx_name, false);
remove_volume_files(&volume_name);
remove_volume_files(&idx_name);
continue;
}
+3 -67
View File
@@ -3157,34 +3157,12 @@ impl Volume {
}
}
// A regular volume and an EC volume for the same id share <base>.vif.
// When EC artefacts coexist on this disk (e.g. shards distributed onto
// a source replica before it is deleted), keep the .vif so removing the
// regular volume does not strip the EC volume's info file.
let keep_vif = self.shares_vif_with_ec_volume();
self.close();
remove_volume_files(&self.data_file_name(), keep_vif);
remove_volume_files(&self.index_file_name(), keep_vif);
remove_volume_files(&self.data_file_name());
remove_volume_files(&self.index_file_name());
Ok(())
}
/// Reports whether an EC volume for this id has a sealed .ecx on the same
/// disk, in which case its .vif is the same file as the regular volume's
/// and must outlive the regular volume's deletion. Mirrors the on-disk
/// portion of Go's Volume.sharesVifWithEcVolume / HasEcxFileOnDisk.
fn shares_vif_with_ec_volume(&self) -> bool {
let has_ecx = |base: &str| -> bool {
fs::metadata(format!("{}.ecx", base))
.map(|m| !m.is_dir() && m.len() > 0)
.unwrap_or(false)
};
if has_ecx(&volume_file_name(&self.dir_idx, &self.collection, self.id)) {
return true;
}
self.dir != self.dir_idx
&& has_ecx(&volume_file_name(&self.dir, &self.collection, self.id))
}
/// Check if an I/O error is EIO (errno 5) and record it for health monitoring.
/// On success (None), clears any previously recorded EIO error.
/// Matches Go's `checkReadWriteError` in volume_write.go.
@@ -3253,13 +3231,10 @@ fn get_append_at_ns(last: u64) -> u64 {
/// Remove all files associated with a volume.
/// .dat/.idx removals log at info level so destructive calls are traceable.
pub(crate) fn remove_volume_files(base: &str, keep_vif: bool) {
pub(crate) fn remove_volume_files(base: &str) {
for ext in &[
".dat", ".idx", ".vif", ".sdx", ".cpd", ".cpx", ".note", ".rdb",
] {
if *ext == ".vif" && keep_vif {
continue;
}
let path = format!("{}{}", base, ext);
let size = fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
let existed = fs::remove_file(&path).is_ok();
@@ -4765,43 +4740,4 @@ mod tests {
".vif removed from data dir"
);
}
/// When an EC volume for the same id has a sealed .ecx on the same disk, the
/// .vif is shared with it and must survive the regular volume's deletion.
#[test]
fn test_destroy_keeps_vif_when_ec_coexists() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let mut v = make_test_volume(dir);
let mut n = Needle {
id: NeedleId(1),
cookie: Cookie(1),
data: b"test".to_vec(),
data_size: 4,
..Needle::default()
};
v.write_needle(&mut n, true).unwrap();
let vif_path = format!("{}/1.vif", dir);
std::fs::write(&vif_path, r#"{"version":3}"#).unwrap();
// A sealed .ecx marks a coexisting EC volume for the same id.
let ecx_path = format!("{}/1.ecx", dir);
std::fs::write(&ecx_path, b"ec-index").unwrap();
v.destroy(false, false).unwrap();
let dat_path = format!("{}/1.dat", dir);
let idx_path = format!("{}/1.idx", dir);
assert!(!std::path::Path::new(&dat_path).exists(), ".dat removed");
assert!(!std::path::Path::new(&idx_path).exists(), ".idx removed");
assert!(
std::path::Path::new(&vif_path).exists(),
".vif kept: shared with the coexisting EC volume"
);
assert!(
std::path::Path::new(&ecx_path).exists(),
".ecx is an EC sidecar, never touched here"
);
}
}
+36
View File
@@ -0,0 +1,36 @@
.PHONY: all build test test-verbose test-short test-debug clean deps tidy
all: build test
# Build the weed binary first
build:
cd ../../weed && go build -o weed .
# Install test dependencies
deps:
go mod download
# Run all tests
test: build deps
go test -timeout 5m ./...
# Run tests with verbose output
test-verbose: build deps
go test -v -timeout 5m ./...
# Skip long-running integration tests
test-short: deps
go test -short -v ./...
# Run tests with debug output from SeaweedFS
test-debug: build deps
go test -v -timeout 5m ./... 2>&1 | tee test.log
# Clean up test artifacts
clean:
rm -f test.log
go clean -testcache
# Update go.sum
tidy:
go mod tidy
+92
View File
@@ -0,0 +1,92 @@
# SeaweedFS NFS Integration Tests
End-to-end tests that boot a real SeaweedFS cluster (`master` + `volume` +
`filer`) plus the experimental `weed nfs` frontend and drive it through the
NFSv3 wire protocol. The tests talk to the server over TCP using
`github.com/willscott/go-nfs-client`, which means they do **not** need a
kernel NFS mount, privileged ports, or any platform-specific tooling.
## Prerequisites
1. Build the `weed` binary:
```bash
cd ../../weed
go build -o weed .
```
2. Go 1.24 or later.
## Running the tests
```bash
# Build weed and run everything
make test
# Verbose output, keeps the subprocess stdout
make test-verbose
# Skip integration tests — useful when iterating on the framework itself
make test-short
# Run a single test
go test -v -run TestNfsBasicReadWrite ./...
```
Every test starts its own cluster on random loopback ports, so runs are
isolated and can execute in parallel.
## Layout
- `framework.go` — launches `weed master`, `weed volume`, `weed filer`, and
`weed nfs` as subprocesses, waits for each to accept TCP, and exposes a
`Mount()` helper that returns an `nfsclient.Target`.
- `basic_test.go` — covers the most common NFS operations:
- Read/write round-trip (`TestNfsBasicReadWrite`)
- Mkdir / ReadDirPlus / RmDir (`TestNfsMkdirAndRmdir`)
- Nested directory + leaf file (`TestNfsNestedDirectories`)
- Rename preserves content (`TestNfsRenamePreservesContent`)
- Overwrite shrinks file size (`TestNfsOverwriteShrinksFile`)
- Large binary file round-trip (`TestNfsLargeFile`)
- Arbitrary binary and empty files (`TestNfsBinaryAndEmptyFiles`)
- Symlink + Readlink (`TestNfsSymlinkRoundTrip`)
- ReadDirPlus ordering sanity (`TestNfsReadDirPlusOrdering`)
- Remove on missing path errors cleanly (`TestNfsRemoveMissingFailsCleanly`)
- FSINFO advertises non-zero limits (`TestNfsFSInfoReturnsSaneLimits`)
- Sequential append writes concatenate (`TestNfsAppendIsSequential`)
- ReadDir after remove (`TestNfsReadDirAfterRemove`)
## Debugging a failing test
Keep the cluster temp dir for inspection:
```go
config := DefaultTestConfig()
config.SkipCleanup = true
```
Enable subprocess stdout/stderr:
```go
config := DefaultTestConfig()
config.EnableDebug = true
```
Or run with `-v`, which flips `EnableDebug` automatically via `testing.Verbose()`.
## Notes
- The NFS server binds to `127.0.0.1` with `-ip.bind=127.0.0.1` and exports
`/nfs_export`. The test framework pre-creates that directory via the
filer's HTTP API before starting the NFS server — the NFS server requires
its export root to exist in the filer's namespace with a real entry, and
the filer's synthetic `/` root does not match the `Name=="/"` check the
NFS server performs during `ensureIndexedEntry`.
- Ports are allocated dynamically. Each test run opens a short-lived
listener on `127.0.0.1:0`, reads back the assigned port, closes the
listener, and hands the port to `weed master/volume/filer/nfs`. There is
a tiny race window between close and reopen that has not been a problem
in practice but is worth remembering if you see a "bind: address already
in use" failure.
- All four `weed` components are started with explicit `-port.grpc=...`
flags. Without them, the default is `-port + 10000`, which overflows
`65535` whenever the HTTP port lands above `55535` — the kernel's
ephemeral port range on macOS routinely does.
+400
View File
@@ -0,0 +1,400 @@
package nfs
import (
"bytes"
"fmt"
"io"
"os"
"path"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
nfsclient "github.com/willscott/go-nfs-client/nfs"
)
// setupFramework is a small helper that boots the cluster for a single test
// and tears everything down on completion. Every test gets a fresh filer +
// volume pair so they cannot step on each other's namespace.
func setupFramework(t *testing.T) *NfsTestFramework {
t.Helper()
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
config := DefaultTestConfig()
config.EnableDebug = testing.Verbose()
fw := NewNfsTestFramework(t, config)
require.NoError(t, fw.Setup(config), "framework setup")
t.Cleanup(fw.Cleanup)
return fw
}
// writeAll writes payload to path on the target in a single Write call. The
// NFS WRITE3 RPC chunks internally, so this exists purely so tests read
// linearly.
func writeAll(t *testing.T, target *nfsclient.Target, remotePath string, payload []byte) {
t.Helper()
file, err := target.OpenFile(remotePath, 0o644)
require.NoError(t, err, "open %s for write", remotePath)
if len(payload) > 0 {
n, err := file.Write(payload)
require.NoError(t, err, "write %s", remotePath)
require.Equal(t, len(payload), n, "short write on %s", remotePath)
}
require.NoError(t, file.Close(), "close %s", remotePath)
}
// readAll opens path on the target and returns the full file contents.
func readAll(t *testing.T, target *nfsclient.Target, remotePath string) []byte {
t.Helper()
file, err := target.Open(remotePath)
require.NoError(t, err, "open %s for read", remotePath)
defer file.Close()
content, err := io.ReadAll(file)
require.NoError(t, err, "read %s", remotePath)
return content
}
// TestNfsBasicReadWrite exercises the most common NFS path: OpenFile + Write
// + Close followed by Open + Read to verify round-trip data integrity.
func TestNfsBasicReadWrite(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
payload := []byte("hello from seaweedfs nfs integration test")
writeAll(t, target, "/hello.txt", payload)
got := readAll(t, target, "/hello.txt")
assert.Equal(t, payload, got, "round-tripped content must match")
info, err := target.Getattr("/hello.txt")
require.NoError(t, err)
assert.Equal(t, int64(len(payload)), int64(info.Filesize))
}
// TestNfsMkdirAndRmdir covers Mkdir, ReadDirPlus, and RmDir. The readdir
// assertion also verifies that the newly-created directory shows up under
// the export root the way a POSIX client would expect.
func TestNfsMkdirAndRmdir(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
_, err = target.Mkdir("/dir1", 0o755)
require.NoError(t, err)
entries, err := target.ReadDirPlus("/")
require.NoError(t, err)
found := false
for _, entry := range entries {
if entry.Name() == "dir1" {
found = true
assert.True(t, entry.IsDir(), "dir1 should be a directory")
}
}
assert.True(t, found, "expected dir1 in readdir listing")
require.NoError(t, target.RmDir("/dir1"))
// After removal, dir1 must be gone from the listing.
entries, err = target.ReadDirPlus("/")
require.NoError(t, err)
for _, entry := range entries {
assert.NotEqual(t, "dir1", entry.Name(), "dir1 should be removed")
}
}
// TestNfsNestedDirectories ensures the server can materialise a deep tree in
// a single Mkdir-per-segment sequence and that reads/writes work at the
// leaves.
func TestNfsNestedDirectories(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
for _, segment := range []string{"/a", "/a/b", "/a/b/c"} {
_, err := target.Mkdir(segment, 0o755)
require.NoError(t, err, "mkdir %s", segment)
}
payload := []byte("deep path content")
writeAll(t, target, "/a/b/c/leaf.txt", payload)
got := readAll(t, target, "/a/b/c/leaf.txt")
assert.Equal(t, payload, got)
require.NoError(t, target.Remove("/a/b/c/leaf.txt"))
require.NoError(t, target.RmDir("/a/b/c"))
require.NoError(t, target.RmDir("/a/b"))
require.NoError(t, target.RmDir("/a"))
}
// TestNfsRenamePreservesContent renames a file and makes sure the content
// at the new path matches what was written at the old one, and that the
// old path disappears. It does not assert on inode identity because pjdfstest
// already covers that and this test intentionally avoids depending on the
// mount-side identity plumbing.
func TestNfsRenamePreservesContent(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
payload := []byte("rename me")
writeAll(t, target, "/src.txt", payload)
require.NoError(t, target.Rename("/src.txt", "/dst.txt"))
_, _, err = target.Lookup("/src.txt")
assert.Error(t, err, "source should be gone after rename")
got := readAll(t, target, "/dst.txt")
assert.Equal(t, payload, got)
require.NoError(t, target.Remove("/dst.txt"))
}
// TestNfsOverwriteShrinksFile rewrites an existing file with shorter content
// and asserts Getattr reports the new (smaller) size. go-nfs-client's
// OpenFile does not pass O_TRUNC, so the test truncates explicitly via
// Setattr(size=0) before the second write — mirroring what `echo >file`
// does on a POSIX client.
func TestNfsOverwriteShrinksFile(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
writeAll(t, target, "/overwrite.txt", []byte("the quick brown fox"))
require.NoError(t, target.Setattr("/overwrite.txt", nfsclient.Sattr3{
Size: nfsclient.SetSize{SetIt: true, Size: 0},
}))
writeAll(t, target, "/overwrite.txt", []byte("short"))
info, err := target.Getattr("/overwrite.txt")
require.NoError(t, err)
assert.Equal(t, int64(len("short")), int64(info.Filesize))
got := readAll(t, target, "/overwrite.txt")
assert.Equal(t, []byte("short"), got)
require.NoError(t, target.Remove("/overwrite.txt"))
}
// TestNfsLargeFile writes a multi-megabyte payload so the write path has to
// cut chunks and flush through the volume server rather than inlining
// content in the filer entry.
func TestNfsLargeFile(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
const size = 3 * 1024 * 1024 // 3 MiB — exceeds the 4 MiB inline cutoff boundary when combined with metadata
payload := make([]byte, size)
for i := range payload {
payload[i] = byte(i % 251) // non-repeating to catch offset bugs
}
writeAll(t, target, "/big.bin", payload)
info, err := target.Getattr("/big.bin")
require.NoError(t, err)
assert.Equal(t, int64(size), int64(info.Filesize))
got := readAll(t, target, "/big.bin")
require.Equal(t, size, len(got))
assert.True(t, bytes.Equal(payload, got), "large file content must round-trip byte-for-byte")
require.NoError(t, target.Remove("/big.bin"))
}
// TestNfsBinaryAndEmptyFiles covers two edge-case payloads the write path
// tends to regress on: arbitrary binary bytes and zero-length files.
func TestNfsBinaryAndEmptyFiles(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
t.Run("AllByteValues", func(t *testing.T) {
payload := make([]byte, 256)
for i := range payload {
payload[i] = byte(i)
}
writeAll(t, target, "/binary.bin", payload)
assert.Equal(t, payload, readAll(t, target, "/binary.bin"))
require.NoError(t, target.Remove("/binary.bin"))
})
t.Run("EmptyFile", func(t *testing.T) {
writeAll(t, target, "/empty.txt", nil)
info, err := target.Getattr("/empty.txt")
require.NoError(t, err)
assert.Equal(t, int64(0), int64(info.Filesize))
require.NoError(t, target.Remove("/empty.txt"))
})
}
// TestNfsSymlinkRoundTrip covers Symlink and Readlink through the nfs server.
// Readlink returns the target path; the server does not auto-traverse it.
func TestNfsSymlinkRoundTrip(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
// Symlink uses a different RPC than open+create, and our server routes it
// through the billy Change interface.
require.NoError(t, target.Symlink("/target.txt", "/link.txt"))
// The underlying target does not need to exist for readlink to succeed.
file, _, err := target.Lookup("/link.txt")
require.NoError(t, err, "lookup symlink")
assert.True(t, file.Mode()&os.ModeSymlink != 0, "expected symlink mode, got %s", file.Mode())
require.NoError(t, target.Remove("/link.txt"))
}
// TestNfsReadDirPlusOrdering creates a handful of files with distinct names
// and ensures ReadDirPlus surfaces every one of them. The server pages
// listings from the filer, so we want to make sure nothing is truncated.
func TestNfsReadDirPlusOrdering(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
_, err = target.Mkdir("/listing", 0o755)
require.NoError(t, err)
names := []string{"alpha.txt", "beta.txt", "gamma.txt", "delta.txt", "epsilon.txt"}
for _, name := range names {
writeAll(t, target, path.Join("/listing", name), []byte(name))
}
entries, err := target.ReadDirPlus("/listing")
require.NoError(t, err)
seen := make(map[string]struct{}, len(entries))
for _, entry := range entries {
if entry.Name() == "." || entry.Name() == ".." {
continue
}
seen[entry.Name()] = struct{}{}
}
for _, name := range names {
_, ok := seen[name]
assert.True(t, ok, "expected %s in directory listing", name)
}
for _, name := range names {
require.NoError(t, target.Remove(path.Join("/listing", name)))
}
require.NoError(t, target.RmDir("/listing"))
}
// TestNfsRemoveMissingFailsCleanly asserts that removing a non-existent path
// surfaces an error instead of silently succeeding. A bug where the server
// returned NFS3_OK on missing entries would hide metadata drift.
func TestNfsRemoveMissingFailsCleanly(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
err = target.Remove("/does_not_exist.txt")
require.Error(t, err, "removing a missing file must error")
// NFS3 surfaces this as NFS3ERR_NOENT; make sure the error text is
// recognisable without locking us into the library's exact wording.
assert.True(t,
strings.Contains(strings.ToLower(err.Error()), "noent") ||
strings.Contains(strings.ToLower(err.Error()), "not exist") ||
strings.Contains(strings.ToLower(err.Error()), "no such"),
"unexpected error shape: %v", err)
}
// TestNfsFSInfoReturnsSaneLimits pokes at FSINFO so we catch regressions
// where the server advertises zero read/write limits (which would make
// clients fall back to the 8 KiB floor and slow every test that follows).
func TestNfsFSInfoReturnsSaneLimits(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
info, err := target.FSInfo()
require.NoError(t, err)
require.NotNil(t, info)
assert.Greater(t, info.RTPref, uint32(0), "rtpref must be positive")
assert.Greater(t, info.WTPref, uint32(0), "wtpref must be positive")
}
// TestNfsAppendIsSequential writes two chunks to the same file in separate
// Open cycles and asserts the concatenation is preserved. The second write
// uses O_APPEND (the default Open path in go-nfs-client does not pass
// flags, so we explicitly reopen after writing the first chunk).
func TestNfsAppendIsSequential(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
const prefix = "part1-"
const suffix = "part2"
writeAll(t, target, "/concat.txt", []byte(prefix))
file, err := target.OpenFile("/concat.txt", 0o644)
require.NoError(t, err)
// Seek to end before writing so we append rather than overwrite. go-nfs
// client's File.Seek uses the same offset tracking as Write so this is
// enough to place the second chunk after the first.
_, err = file.Seek(int64(len(prefix)), io.SeekStart)
require.NoError(t, err)
_, err = file.Write([]byte(suffix))
require.NoError(t, err)
require.NoError(t, file.Close())
got := readAll(t, target, "/concat.txt")
assert.Equal(t, prefix+suffix, string(got))
require.NoError(t, target.Remove("/concat.txt"))
}
// Regression: readdir should not emit stale entries after a remove. This is
// the scenario the PR's meta cache invalidation logic was written to fix.
func TestNfsReadDirAfterRemove(t *testing.T) {
fw := setupFramework(t)
target, cleanup, err := fw.Mount()
require.NoError(t, err)
defer cleanup()
_, err = target.Mkdir("/churn", 0o755)
require.NoError(t, err)
for i := 0; i < 5; i++ {
writeAll(t, target, path.Join("/churn", fmt.Sprintf("f%d.txt", i)), []byte{byte(i)})
}
// Remove the middle one and re-list.
require.NoError(t, target.Remove("/churn/f2.txt"))
entries, err := target.ReadDirPlus("/churn")
require.NoError(t, err)
for _, entry := range entries {
assert.NotEqual(t, "f2.txt", entry.Name(), "removed file should not reappear in listing")
}
for i := 0; i < 5; i++ {
if i == 2 {
continue
}
require.NoError(t, target.Remove(path.Join("/churn", fmt.Sprintf("f%d.txt", i))))
}
require.NoError(t, target.RmDir("/churn"))
}
+423
View File
@@ -0,0 +1,423 @@
package nfs
import (
"bytes"
"fmt"
"io"
"mime/multipart"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"syscall"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/stretchr/testify/require"
nfsclient "github.com/willscott/go-nfs-client/nfs"
"github.com/willscott/go-nfs-client/nfs/rpc"
)
// NfsTestFramework boots a minimal SeaweedFS cluster (master + volume + filer)
// plus the experimental `weed nfs` frontend and hands out NFSv3 RPC clients
// that talk to it. Everything is driven via subprocesses so the tests exercise
// the same binary an operator would deploy, and no kernel mount is required.
type NfsTestFramework struct {
t *testing.T
tempDir string
dataDir string
masterProcess *os.Process
volumeProcess *os.Process
filerProcess *os.Process
nfsProcess *os.Process
masterAddr string
masterGrpc int
volumeAddr string
volumeGrpc int
filerAddr string
filerGrpc int
nfsAddr string
exportRoot string
weedBinary string
isSetup bool
skipCleanup bool
}
// TestConfig controls how the framework boots the cluster.
type TestConfig struct {
NumVolumes int
EnableDebug bool
SkipCleanup bool // keep temp dir on failure for inspection
// ExportRoot is the filer path the NFS server exports. Defaults to "/"
// so tests can use any path, with a single warning logged by the server.
ExportRoot string
}
// DefaultTestConfig returns the defaults used by most tests. A dedicated
// /nfs_export subtree is used as the NFS export root because the NFS server
// requires the export directory to exist in the filer's namespace and carry
// a non-zero inode — passing "/" would succeed only for filer setups that
// have already backfilled the root inode.
func DefaultTestConfig() *TestConfig {
return &TestConfig{
NumVolumes: 3,
EnableDebug: false,
SkipCleanup: false,
ExportRoot: "/nfs_export",
}
}
// NewNfsTestFramework allocates a framework bound to the current test. Call
// Setup next to actually start the cluster.
func NewNfsTestFramework(t *testing.T, config *TestConfig) *NfsTestFramework {
if config == nil {
config = DefaultTestConfig()
}
tempDir, err := os.MkdirTemp("", "seaweedfs_nfs_test_")
require.NoError(t, err)
// testutil.MustAllocatePorts holds every listener open until the full
// batch has been reserved, which avoids the "close-then-hope" race my
// original per-port helper had. We need seven ports: four HTTP (master,
// volume, filer, nfs) and three gRPC (master, volume, filer — nfs has
// no gRPC endpoint).
ports := testutil.MustAllocatePorts(t, 7)
exportRoot := config.ExportRoot
if exportRoot == "" {
exportRoot = "/"
}
return &NfsTestFramework{
t: t,
tempDir: tempDir,
dataDir: filepath.Join(tempDir, "data"),
masterAddr: fmt.Sprintf("127.0.0.1:%d", ports[0]),
masterGrpc: ports[1],
volumeAddr: fmt.Sprintf("127.0.0.1:%d", ports[2]),
volumeGrpc: ports[3],
filerAddr: fmt.Sprintf("127.0.0.1:%d", ports[4]),
filerGrpc: ports[5],
nfsAddr: fmt.Sprintf("127.0.0.1:%d", ports[6]),
exportRoot: exportRoot,
weedBinary: findWeedBinary(),
isSetup: false,
skipCleanup: config.SkipCleanup,
}
}
// Setup starts the SeaweedFS cluster and the NFS frontend, waiting for each
// component to accept connections before moving on.
func (f *NfsTestFramework) Setup(config *TestConfig) error {
if f.isSetup {
return fmt.Errorf("framework already setup")
}
dirs := []string{
f.dataDir,
filepath.Join(f.dataDir, "master"),
filepath.Join(f.dataDir, "volume"),
}
for _, dir := range dirs {
if err := os.MkdirAll(dir, 0755); err != nil {
return fmt.Errorf("failed to create directory %s: %v", dir, err)
}
}
if err := f.startMaster(config); err != nil {
return fmt.Errorf("failed to start master: %v", err)
}
if !testutil.WaitForPort(portFromAddr(f.masterAddr), testutil.SeaweedMiniStartupTimeout) {
return fmt.Errorf("master not ready at %s", f.masterAddr)
}
if err := f.startVolumeServer(config); err != nil {
return fmt.Errorf("failed to start volume server: %v", err)
}
if !testutil.WaitForPort(portFromAddr(f.volumeAddr), testutil.SeaweedMiniStartupTimeout) {
return fmt.Errorf("volume server not ready at %s", f.volumeAddr)
}
if err := f.startFiler(config); err != nil {
return fmt.Errorf("failed to start filer: %v", err)
}
if !testutil.WaitForPort(portFromAddr(f.filerAddr), testutil.SeaweedMiniStartupTimeout) {
return fmt.Errorf("filer not ready at %s", f.filerAddr)
}
// Pre-create the export root in the filer's namespace. The NFS server
// expects its export directory to exist with a real inode; uploading a
// placeholder file creates the parent directory implicitly and then
// removing the file leaves the empty directory in place.
if f.exportRoot != "/" {
if err := f.ensureExportRootExists(); err != nil {
return fmt.Errorf("failed to pre-create export root %s: %v", f.exportRoot, err)
}
}
if err := f.startNfsServer(config); err != nil {
return fmt.Errorf("failed to start NFS server: %v", err)
}
if !testutil.WaitForPort(portFromAddr(f.nfsAddr), testutil.SeaweedMiniStartupTimeout) {
return fmt.Errorf("NFS server not ready at %s", f.nfsAddr)
}
// Let the NFS server finish wiring up its gRPC subscription to the filer
// before the first client call hits MOUNT/LOOKUP.
time.Sleep(500 * time.Millisecond)
f.isSetup = true
return nil
}
// Cleanup stops all processes. Temp state is preserved if SkipCleanup is set.
func (f *NfsTestFramework) Cleanup() {
processes := []*os.Process{f.nfsProcess, f.filerProcess, f.volumeProcess, f.masterProcess}
for _, proc := range processes {
if proc != nil {
_ = proc.Signal(syscall.SIGTERM)
_, _ = proc.Wait()
}
}
if !f.skipCleanup {
_ = os.RemoveAll(f.tempDir)
}
}
// NfsAddr returns the TCP address the NFS server is listening on.
func (f *NfsTestFramework) NfsAddr() string { return f.nfsAddr }
// FilerAddr returns the TCP address of the filer.
func (f *NfsTestFramework) FilerAddr() string { return f.filerAddr }
// ExportRoot returns the path the NFS server exports.
func (f *NfsTestFramework) ExportRoot() string { return f.exportRoot }
// Mount opens an NFSv3 MOUNT+NFS connection against the running NFS server
// and returns a Target that tests can drive like a mini-VFS. Caller is
// responsible for calling the returned cleanup func to Unmount and close the
// TCP connection.
func (f *NfsTestFramework) Mount() (*nfsclient.Target, func(), error) {
var (
client *rpc.Client
err error
)
// The NFS server's TCP listener may already be accepting connections when
// waitForService returns, but the RPC program registration can trail it
// by a few milliseconds. Retry the dial to absorb that small window.
for attempt := 0; attempt < 20; attempt++ {
client, err = rpc.DialTCP("tcp", f.nfsAddr, false)
if err == nil {
break
}
time.Sleep(25 * time.Millisecond)
}
if err != nil {
return nil, nil, fmt.Errorf("dial NFS: %w", err)
}
// Note: do not set Mount.Addr here. When Addr is non-empty, the go-nfs
// client re-dials via portmapper and concatenates `:111` onto the
// address, which produces "too many colons" for a raw `host:port`
// string. Reusing the existing RPC client avoids that path entirely.
mounter := &nfsclient.Mount{Client: client}
target, err := mounter.Mount(f.exportRoot, rpc.AuthNull)
if err != nil {
client.Close()
return nil, nil, fmt.Errorf("mount %s: %w", f.exportRoot, err)
}
cleanup := func() {
_ = mounter.Unmount()
client.Close()
}
return target, cleanup, nil
}
func (f *NfsTestFramework) startMaster(config *TestConfig) error {
_, masterPort := splitHostPort(f.masterAddr)
args := []string{
"master",
"-ip=127.0.0.1",
fmt.Sprintf("-port=%d", masterPort),
fmt.Sprintf("-port.grpc=%d", f.masterGrpc),
"-mdir=" + filepath.Join(f.dataDir, "master"),
"-raftBootstrap",
"-peers=none",
}
return f.startProcess(&f.masterProcess, config, args)
}
func (f *NfsTestFramework) startVolumeServer(config *TestConfig) error {
_, volumePort := splitHostPort(f.volumeAddr)
// pb.ServerAddress encodes a non-default gRPC port as `host:port.grpc`.
// See weed/pb/server_address.go — the dot, not a colon, is the separator
// between the HTTP port and the gRPC port.
masterWithGrpc := fmt.Sprintf("%s.%d", f.masterAddr, f.masterGrpc)
args := []string{
"volume",
"-master=" + masterWithGrpc,
"-ip=127.0.0.1",
fmt.Sprintf("-port=%d", volumePort),
fmt.Sprintf("-port.grpc=%d", f.volumeGrpc),
"-dir=" + filepath.Join(f.dataDir, "volume"),
fmt.Sprintf("-max=%d", config.NumVolumes),
}
return f.startProcess(&f.volumeProcess, config, args)
}
func (f *NfsTestFramework) startFiler(config *TestConfig) error {
_, filerPort := splitHostPort(f.filerAddr)
masterWithGrpc := fmt.Sprintf("%s.%d", f.masterAddr, f.masterGrpc)
args := []string{
"filer",
"-master=" + masterWithGrpc,
"-ip=127.0.0.1",
fmt.Sprintf("-port=%d", filerPort),
fmt.Sprintf("-port.grpc=%d", f.filerGrpc),
}
return f.startProcess(&f.filerProcess, config, args)
}
func (f *NfsTestFramework) startNfsServer(config *TestConfig) error {
_, nfsPort := splitHostPort(f.nfsAddr)
// `host:port.grpc` encoding — see pb/server_address.go.
filerWithGrpc := fmt.Sprintf("%s.%d", f.filerAddr, f.filerGrpc)
args := []string{
"nfs",
"-filer=" + filerWithGrpc,
"-ip.bind=127.0.0.1",
fmt.Sprintf("-port=%d", nfsPort),
"-filer.path=" + f.exportRoot,
}
return f.startProcess(&f.nfsProcess, config, args)
}
func (f *NfsTestFramework) startProcess(target **os.Process, config *TestConfig, args []string) error {
cmd := exec.Command(f.weedBinary, args...)
cmd.Dir = f.tempDir
if config.EnableDebug {
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
}
if err := cmd.Start(); err != nil {
return err
}
*target = cmd.Process
return nil
}
// portFromAddr returns just the port number from a `host:port` string.
// testutil.WaitForPort takes an int port, not a full address.
func portFromAddr(addr string) int {
_, port := splitHostPort(addr)
return port
}
// ensureExportRootExists posts a placeholder file to f.exportRoot via the
// filer's HTTP API, then deletes it. That roundtrip implicitly creates the
// target directory so the NFS server has something to mount. We bypass
// weed/pb here because the HTTP client is simpler and needs no gRPC stubs.
func (f *NfsTestFramework) ensureExportRootExists() error {
exportRoot := strings.TrimRight(f.exportRoot, "/")
if exportRoot == "" {
return nil
}
placeholder := exportRoot + "/.nfs_test_init"
filerURL := "http://" + f.filerAddr + placeholder
var body bytes.Buffer
writer := multipart.NewWriter(&body)
part, err := writer.CreateFormFile("file", ".nfs_test_init")
if err != nil {
return err
}
if _, err := io.WriteString(part, ""); err != nil {
return err
}
if err := writer.Close(); err != nil {
return err
}
httpClient := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequest(http.MethodPost, filerURL, &body)
if err != nil {
return err
}
req.Header.Set("Content-Type", writer.FormDataContentType())
resp, err := httpClient.Do(req)
if err != nil {
return err
}
_, _ = io.Copy(io.Discard, resp.Body)
resp.Body.Close()
if resp.StatusCode/100 != 2 {
return fmt.Errorf("filer POST %s returned status %d", filerURL, resp.StatusCode)
}
// Delete the placeholder; the directory stays behind.
deleteReq, err := http.NewRequest(http.MethodDelete, filerURL, nil)
if err != nil {
return err
}
deleteResp, err := httpClient.Do(deleteReq)
if err != nil {
return err
}
_, _ = io.Copy(io.Discard, deleteResp.Body)
deleteResp.Body.Close()
if deleteResp.StatusCode/100 != 2 && deleteResp.StatusCode != http.StatusNotFound {
return fmt.Errorf("filer DELETE %s returned status %d", filerURL, deleteResp.StatusCode)
}
return nil
}
func splitHostPort(addr string) (string, int) {
host, portStr, err := net.SplitHostPort(addr)
if err != nil {
return "", 0
}
var port int
_, _ = fmt.Sscanf(portStr, "%d", &port)
return host, port
}
// findWeedBinary locates the weed binary, preferring the local build in the
// checkout so tests run against the code under review rather than whatever is
// on $PATH.
func findWeedBinary() string {
if _, thisFile, _, ok := runtime.Caller(0); ok {
thisDir := filepath.Dir(thisFile)
candidates := []string{
filepath.Join(thisDir, "../../weed/weed"),
filepath.Join(thisDir, "../weed/weed"),
}
for _, candidate := range candidates {
if _, err := os.Stat(candidate); err == nil {
abs, _ := filepath.Abs(candidate)
return abs
}
}
}
cwd, _ := os.Getwd()
candidates := []string{
filepath.Join(cwd, "../../weed/weed"),
filepath.Join(cwd, "../weed/weed"),
filepath.Join(cwd, "./weed"),
}
for _, candidate := range candidates {
if _, err := os.Stat(candidate); err == nil {
abs, _ := filepath.Abs(candidate)
return abs
}
}
if path, err := exec.LookPath("weed"); err == nil {
return path
}
return "weed"
}
+21
View File
@@ -0,0 +1,21 @@
module seaweedfs-nfs-tests
go 1.25.0
// test/testutil lives inside the main seaweedfs module; pull it in via a
// local replace so this integration suite can reuse the shared port
// allocator and readiness helpers instead of reinventing them.
replace github.com/seaweedfs/seaweedfs => ../..
require (
github.com/seaweedfs/seaweedfs v0.0.0-00010101000000-000000000000
github.com/stretchr/testify v1.11.1
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886
)
require (
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
+14
View File
@@ -0,0 +1,14 @@
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886 h1:DtrBtkgTJk2XGt4T7eKdKVkd9A5NCevN2e4inLXtsqA=
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886/go.mod h1:Tq++Lr/FgiS3X48q5FETemXiSLGuYMQT2sPjYNPJSwA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+193
View File
@@ -0,0 +1,193 @@
//go:build linux
package nfs
// End-to-end mount tests that drive the real Linux NFS client (mount.nfs +
// in-tree kernel) against a running `weed nfs` subprocess. These exist to
// catch regressions that the existing framework can't see, because the
// framework drives the server with willscott/go-nfs-client — the same RPC
// library the server uses internally — so any bug shared between the two
// (XDR layout, version dispatch, RPC framing) round-trips invisibly.
//
// Two real bugs hit recently were exactly that shape:
// 1. NFSv4 mis-routed to the v3 SETATTR handler (#9262). The client
// library never sends NFSv4, so the test suite never noticed; the
// Linux kernel mount path did notice, with EIO.
// 2. UDP MOUNT v3 missing. Only TCP MOUNT was advertised; the kernel
// defaults mountproto=udp in many setups, so the in-tree client
// surfaced EPROTONOSUPPORT during MOUNT setup.
//
// These tests mount over the actual loopback interface using mount.nfs and
// shell out to /bin/mount and /bin/umount. They require root (mount(2) is
// privileged) and Linux (the in-tree NFS client is what's being exercised);
// they t.Skip cleanly when either prerequisite is missing.
//
// Run locally with:
//
// cd test/nfs
// sudo go test -v -run TestKernelMount ./...
//
// CI runs them via .github/workflows/nfs-tests.yml after installing
// nfs-common (mount.nfs + helpers).
import (
"errors"
"fmt"
"net"
"os"
"os/exec"
"strings"
"testing"
)
// kernelMountSkipIfUnsupported skips the test when the host can't run a
// real NFS mount. The combined check belongs in one place so the three
// kernel-mount tests stay focused on what they're actually verifying.
func kernelMountSkipIfUnsupported(t *testing.T) {
t.Helper()
if os.Geteuid() != 0 {
t.Skip("kernel mount test requires root; mount(2) is privileged")
}
if _, err := exec.LookPath("mount.nfs"); err != nil {
t.Skipf("mount.nfs not installed: %v (CI installs the nfs-common package)", err)
}
}
// kernelMount runs /bin/mount with the given options against the framework's
// running NFS server, returns the mountpoint and an unmount closure. We pass
// explicit port=/mountport= options so the kernel never queries portmap.
// That keeps the harness honest about what it's testing — the NFS / MOUNT
// wire protocol — and avoids colliding with a system rpcbind on shared CI
// runners (port 111 is privileged and frequently in use already).
func kernelMount(t *testing.T, fw *NfsTestFramework, optsTemplate string) (string, func()) {
t.Helper()
host, portStr, err := net.SplitHostPort(fw.NfsAddr())
if err != nil {
t.Fatalf("split nfs addr %q: %v", fw.NfsAddr(), err)
}
mountpoint, err := os.MkdirTemp("", "weed-nfs-kmount-")
if err != nil {
t.Fatalf("mkdtemp: %v", err)
}
opts := strings.ReplaceAll(optsTemplate, "{port}", portStr)
target := fmt.Sprintf("%s:%s", host, fw.ExportRoot())
cmd := exec.Command("mount", "-t", "nfs", "-o", opts, target, mountpoint)
if out, err := cmd.CombinedOutput(); err != nil {
_ = os.RemoveAll(mountpoint)
t.Fatalf("mount %s -o %s failed: %v\nmount output:\n%s", target, opts, err, out)
}
teardown := func() {
// -f to bail out faster if the server's already gone.
_ = exec.Command("umount", "-f", mountpoint).Run()
_ = os.RemoveAll(mountpoint)
}
return mountpoint, teardown
}
func newKernelMountFramework(t *testing.T) *NfsTestFramework {
t.Helper()
cfg := DefaultTestConfig()
fw := NewNfsTestFramework(t, cfg)
if err := fw.Setup(cfg); err != nil {
fw.Cleanup()
t.Fatalf("framework setup: %v", err)
}
t.Cleanup(fw.Cleanup)
return fw
}
// TestKernelMountV3TCP exercises the most common mount form: NFSv3 + MOUNT
// v3, both over TCP. This is what the existing go-nfs-client tests cover at
// the protocol layer, but running it through mount.nfs and the kernel
// confirms that the wire format we emit decodes cleanly under a different
// XDR/RPC parser.
func TestKernelMountV3TCP(t *testing.T) {
kernelMountSkipIfUnsupported(t)
fw := newKernelMountFramework(t)
mountpoint, undo := kernelMount(t, fw,
"nfsvers=3,nolock,port={port},mountport={port},proto=tcp,mountproto=tcp")
defer undo()
if _, err := os.Stat(mountpoint); err != nil {
t.Errorf("stat mountpoint: %v", err)
}
if _, err := os.ReadDir(mountpoint); err != nil {
t.Errorf("readdir mountpoint: %v", err)
}
}
// TestKernelMountV3MountProtoUDP is the regression test for the UDP MOUNT
// v3 responder. mountproto=udp forces the kernel to call MOUNT over UDP
// only; before the responder existed the kernel hit nothing (MOUNT was
// advertised TCP-only) and surfaced EPROTONOSUPPORT during mount setup.
func TestKernelMountV3MountProtoUDP(t *testing.T) {
kernelMountSkipIfUnsupported(t)
fw := newKernelMountFramework(t)
mountpoint, undo := kernelMount(t, fw,
"nfsvers=3,nolock,port={port},mountport={port},proto=tcp,mountproto=udp")
defer undo()
if _, err := os.Stat(mountpoint); err != nil {
t.Errorf("stat mountpoint: %v", err)
}
}
// TestKernelMountV4RejectsCleanly is the regression test for the NFSv4
// PROG_MISMATCH path (#9262). The server only speaks NFSv3, but the
// previous behaviour was to mis-route v4 COMPOUND to the v3 SETATTR
// handler and write garbage; the kernel surfaced EIO instead of a
// version-mismatch error and (depending on distro) didn't fall back to
// v3. The version filter now answers PROG_MISMATCH so the kernel sees
// "v4 not supported" cleanly.
//
// The test asserts:
// 1. mount.nfs exits non-zero (no silent success against a v3 server);
// 2. the failure message mentions protocol/version/io, which is what the
// kernel surfaces when it gets PROG_MISMATCH instead of garbage. A
// pre-fix server returns "mount system call failed" with no further
// context, so a regression collapses the assertion onto that branch.
func TestKernelMountV4RejectsCleanly(t *testing.T) {
kernelMountSkipIfUnsupported(t)
fw := newKernelMountFramework(t)
host, portStr, err := net.SplitHostPort(fw.NfsAddr())
if err != nil {
t.Fatalf("split nfs addr: %v", err)
}
mountpoint, err := os.MkdirTemp("", "weed-nfs-kmount-v4-")
if err != nil {
t.Fatalf("mkdtemp: %v", err)
}
defer os.RemoveAll(mountpoint)
target := fmt.Sprintf("%s:%s", host, fw.ExportRoot())
cmd := exec.Command("mount", "-t", "nfs", "-o",
fmt.Sprintf("vers=4,port=%s", portStr),
target, mountpoint)
out, err := cmd.CombinedOutput()
defer exec.Command("umount", "-f", mountpoint).Run()
if err == nil {
t.Fatalf("v4 mount unexpectedly succeeded against v3-only server\nmount output:\n%s", out)
}
// Don't pin the exact error string — different distros print slightly
// different things — but require some hint that the kernel saw a
// protocol-level failure rather than a generic "mount system call
// failed". Without the version filter, mount.nfs prints the latter
// alone; with it, the former.
lower := strings.ToLower(string(out))
if !strings.Contains(lower, "protocol") &&
!strings.Contains(lower, "version") &&
!strings.Contains(lower, "i/o") {
t.Errorf("v4 mount failure didn't mention protocol/version/io; output:\n%s", out)
}
// Also require a non-zero exit so a future change that makes mount(2)
// silently succeed (e.g. by relaxing the version filter) shows up
// here even if the message phrasing changes.
var ee *exec.ExitError
if !errors.As(err, &ee) {
t.Errorf("expected mount to exit non-zero with ExitError, got %v", err)
}
}
+16 -25
View File
@@ -30,22 +30,21 @@ type VolumeServer struct {
address string
baseDir string
mu sync.Mutex
receivedFiles map[string]uint64
mountRequests []*volume_server_pb.VolumeEcShardsMountRequest
deleteRequests []*volume_server_pb.VolumeDeleteRequest
markReadonlyCalls int
markWritableCalls int
readFileStatusCalls int
vacuumGarbageRatio float64
vacuumCommitReadOnly bool
vacuumCheckCalls int
vacuumCompactCalls int
vacuumCommitCalls int
vacuumCleanupCalls int
volumeCopyCalls int
volumeMountCalls int
tailReceiverCalls int
mu sync.Mutex
receivedFiles map[string]uint64
mountRequests []*volume_server_pb.VolumeEcShardsMountRequest
deleteRequests []*volume_server_pb.VolumeDeleteRequest
markReadonlyCalls int
markWritableCalls int
readFileStatusCalls int
vacuumGarbageRatio float64
vacuumCheckCalls int
vacuumCompactCalls int
vacuumCommitCalls int
vacuumCleanupCalls int
volumeCopyCalls int
volumeMountCalls int
tailReceiverCalls int
}
// NewVolumeServer starts a test volume server using the provided base directory.
@@ -116,13 +115,6 @@ func (v *VolumeServer) SetVacuumGarbageRatio(ratio float64) {
v.vacuumGarbageRatio = ratio
}
// SetVacuumCommitReadOnly sets the IsReadOnly value returned by VacuumVolumeCommit.
func (v *VolumeServer) SetVacuumCommitReadOnly(readOnly bool) {
v.mu.Lock()
defer v.mu.Unlock()
v.vacuumCommitReadOnly = readOnly
}
// VacuumStats returns the vacuum RPC call counts.
func (v *VolumeServer) VacuumStats() (check, compact, commit, cleanup int) {
v.mu.Lock()
@@ -436,9 +428,8 @@ func (v *VolumeServer) VacuumVolumeCompact(req *volume_server_pb.VacuumVolumeCom
func (v *VolumeServer) VacuumVolumeCommit(ctx context.Context, req *volume_server_pb.VacuumVolumeCommitRequest) (*volume_server_pb.VacuumVolumeCommitResponse, error) {
v.mu.Lock()
v.vacuumCommitCalls++
readOnly := v.vacuumCommitReadOnly
v.mu.Unlock()
return &volume_server_pb.VacuumVolumeCommitResponse{IsReadOnly: readOnly}, nil
return &volume_server_pb.VacuumVolumeCommitResponse{}, nil
}
func (v *VolumeServer) VacuumVolumeCleanup(ctx context.Context, req *volume_server_pb.VacuumVolumeCleanupRequest) (*volume_server_pb.VacuumVolumeCleanupResponse, error) {
+18 -63
View File
@@ -15,24 +15,6 @@ import (
"google.golang.org/grpc/credentials/insecure"
)
func vacuumJobSpec(volumeID uint32, server string) *plugin_pb.JobSpec {
return &plugin_pb.JobSpec{
JobId: fmt.Sprintf("vacuum-job-%d", volumeID),
JobType: "vacuum",
Parameters: map[string]*plugin_pb.ConfigValue{
"volume_id": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(volumeID)},
},
"server": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: server},
},
"collection": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "vac-test"},
},
},
}
}
func TestVacuumExecutionIntegration(t *testing.T) {
volumeID := uint32(202)
@@ -49,10 +31,26 @@ func TestVacuumExecutionIntegration(t *testing.T) {
source := pluginworkers.NewVolumeServer(t, "")
source.SetVacuumGarbageRatio(0.6)
job := &plugin_pb.JobSpec{
JobId: fmt.Sprintf("vacuum-job-%d", volumeID),
JobType: "vacuum",
Parameters: map[string]*plugin_pb.ConfigValue{
"volume_id": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(volumeID)},
},
"server": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: source.Address()},
},
"collection": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "vac-test"},
},
},
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
result, err := harness.Plugin().ExecuteJob(ctx, vacuumJobSpec(volumeID, source.Address()), nil, 1)
result, err := harness.Plugin().ExecuteJob(ctx, job, nil, 1)
require.NoError(t, err)
require.NotNil(t, result)
require.True(t, result.Success)
@@ -61,48 +59,5 @@ func TestVacuumExecutionIntegration(t *testing.T) {
require.GreaterOrEqual(t, checkCalls, 2)
require.GreaterOrEqual(t, compactCalls, 1)
require.GreaterOrEqual(t, commitCalls, 1)
// Cleanup is only invoked when Phase 1 (Compact) fails to roll back
// the .cpd/.cpx/.cpldb temp files; on the success path Commit
// consumes them (rename .cpd → .dat, .cpx → .idx, .cpldb → .ldb via
// the leveldb needle map) so no Cleanup call is needed. Matches
// topology.vacuumOneVolumeId which only calls batchVacuumVolumeCleanup
// on the Compact-failure branch.
require.Equal(t, 0, cleanupCalls)
// Phase 3 marks each replica writable so master returns it to the
// writables layout. See upstream seaweedfs#9685.
require.GreaterOrEqual(t, source.MarkWritableCount(), 1)
}
// A replica that commits still read-only (operator-set, EIO-quarantined,
// disk-space-low) must not be force-marked writable: master built-in vacuum
// skips it via SetVolumeAvailable, and it recovers on its own ReadOnly=false
// heartbeat.
func TestVacuumExecutionSkipsMarkWritableWhenReadOnly(t *testing.T) {
volumeID := uint32(203)
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
handler := vacuum.NewVacuumHandler(dialOption, 1)
harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
WorkerOptions: pluginworker.WorkerOptions{
GrpcDialOption: dialOption,
},
Handlers: []pluginworker.JobHandler{handler},
})
harness.WaitForJobType("vacuum")
source := pluginworkers.NewVolumeServer(t, "")
source.SetVacuumGarbageRatio(0.6)
source.SetVacuumCommitReadOnly(true)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
result, err := harness.Plugin().ExecuteJob(ctx, vacuumJobSpec(volumeID, source.Address()), nil, 1)
require.NoError(t, err)
require.NotNil(t, result)
require.True(t, result.Success)
_, _, commitCalls, _ := source.VacuumStats()
require.GreaterOrEqual(t, commitCalls, 1)
require.Equal(t, 0, source.MarkWritableCount())
require.GreaterOrEqual(t, cleanupCalls, 1)
}
-107
View File
@@ -1,107 +0,0 @@
{
"realm": "seaweedfs-test",
"enabled": true,
"roles": {
"realm": [
{ "name": "s3-admin" },
{ "name": "s3-read-only" },
{ "name": "s3-write-only" },
{ "name": "s3-read-write" }
]
},
"clients": [
{
"clientId": "seaweedfs-s3",
"enabled": true,
"secret": "seaweedfs-s3-secret",
"publicClient": false,
"serviceAccountsEnabled": true,
"standardFlowEnabled": true,
"directAccessGrantsEnabled": true,
"fullScopeAllowed": true,
"redirectUris": ["*"],
"webOrigins": ["*"],
"protocol": "openid-connect",
"protocolMappers": [
{
"name": "realm-roles",
"protocol": "openid-connect",
"protocolMapper": "oidc-usermodel-realm-role-mapper",
"config": {
"claim.name": "roles",
"jsonType.label": "String",
"multivalued": "true",
"usermodel.realmRoleMapping.rolePrefix": "",
"access.token.claim": "true",
"id.token.claim": "true",
"userinfo.token.claim": "true"
}
},
{
"name": "audience-mapper",
"protocol": "openid-connect",
"protocolMapper": "oidc-audience-mapper",
"config": {
"included.client.audience": "seaweedfs-s3",
"id.token.claim": "false",
"access.token.claim": "true"
}
}
]
}
],
"users": [
{
"username": "admin-user",
"enabled": true,
"email": "admin-user@test.com",
"firstName": "Test",
"lastName": "User",
"emailVerified": true,
"requiredActions": [],
"realmRoles": ["s3-admin"],
"credentials": [
{ "type": "password", "value": "adminuser123", "temporary": false }
]
},
{
"username": "read-user",
"enabled": true,
"email": "read-user@test.com",
"firstName": "Test",
"lastName": "User",
"emailVerified": true,
"requiredActions": [],
"realmRoles": ["s3-read-only"],
"credentials": [
{ "type": "password", "value": "readuser123", "temporary": false }
]
},
{
"username": "write-user",
"enabled": true,
"email": "write-user@test.com",
"firstName": "Test",
"lastName": "User",
"emailVerified": true,
"requiredActions": [],
"realmRoles": ["s3-read-write"],
"credentials": [
{ "type": "password", "value": "writeuser123", "temporary": false }
]
},
{
"username": "write-only-user",
"enabled": true,
"email": "write-only-user@test.com",
"firstName": "Test",
"lastName": "User",
"emailVerified": true,
"requiredActions": [],
"realmRoles": ["s3-write-only"],
"credentials": [
{ "type": "password", "value": "writeonlyuser123", "temporary": false }
]
}
]
}
+8 -53
View File
@@ -7,7 +7,6 @@ import (
"os/exec"
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
@@ -25,7 +24,6 @@ type MultiVolumeCluster struct {
logsDir string
keepLogs bool
volumeServerCount int
disksPerServer int
masterPort int
masterGrpcPort int
@@ -33,7 +31,6 @@ type MultiVolumeCluster struct {
volumePorts []int
volumeGrpcPorts []int
volumePubPorts []int
volumeDiskDirs [][]string // per server: the data directories (one DiskLocation each)
masterCmd *exec.Cmd
volumeCmds []*exec.Cmd
@@ -41,25 +38,13 @@ type MultiVolumeCluster struct {
cleanupOnce sync.Once
}
// StartMultiVolumeCluster starts a cluster with serverCount volume servers, one
// data directory (DiskLocation) each.
// StartMultiVolumeCluster starts a cluster with a specified number of volume servers
func StartMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCount int) *MultiVolumeCluster {
return StartMultiVolumeClusterWithDisks(t, profile, serverCount, 1)
}
// StartMultiVolumeClusterWithDisks starts serverCount volume servers, each with
// disksPerServer data directories passed to -dir as a comma list so every
// directory becomes its own DiskLocation. Lets tests exercise per-disk EC
// layouts (e.g. shards on one disk, a stub/index on a sibling disk).
func StartMultiVolumeClusterWithDisks(t testing.TB, profile matrix.Profile, serverCount, disksPerServer int) *MultiVolumeCluster {
t.Helper()
if serverCount < 1 {
t.Fatalf("serverCount must be at least 1, got %d", serverCount)
}
if disksPerServer < 1 {
t.Fatalf("disksPerServer must be at least 1, got %d", disksPerServer)
}
weedBinary, err := FindOrBuildWeedBinary()
if err != nil {
@@ -75,21 +60,10 @@ func StartMultiVolumeClusterWithDisks(t testing.TB, profile matrix.Profile, serv
logsDir := filepath.Join(baseDir, "logs")
masterDataDir := filepath.Join(baseDir, "master")
// Create directories for master and all volume servers. With one disk the
// layout stays at baseDir/volumeN; multi-disk servers get baseDir/volumeN/diskD.
// Create directories for master and all volume servers
dirs := []string{configDir, logsDir, masterDataDir}
volumeDiskDirs := make([][]string, serverCount)
for i := 0; i < serverCount; i++ {
serverDir := filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
volumeDiskDirs[i] = make([]string, disksPerServer)
for d := 0; d < disksPerServer; d++ {
if disksPerServer == 1 {
volumeDiskDirs[i][d] = serverDir
} else {
volumeDiskDirs[i][d] = filepath.Join(serverDir, fmt.Sprintf("disk%d", d))
}
dirs = append(dirs, volumeDiskDirs[i][d])
}
dirs = append(dirs, filepath.Join(baseDir, fmt.Sprintf("volume%d", i)))
}
for _, dir := range dirs {
if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
@@ -124,13 +98,11 @@ func StartMultiVolumeClusterWithDisks(t testing.TB, profile matrix.Profile, serv
logsDir: logsDir,
keepLogs: keepLogs,
volumeServerCount: serverCount,
disksPerServer: disksPerServer,
masterPort: masterPort,
masterGrpcPort: masterGrpcPort,
volumePorts: make([]int, serverCount),
volumeGrpcPorts: make([]int, serverCount),
volumePubPorts: make([]int, serverCount),
volumeDiskDirs: volumeDiskDirs,
volumeCmds: make([]*exec.Cmd, serverCount),
}
@@ -161,7 +133,8 @@ func StartMultiVolumeClusterWithDisks(t testing.TB, profile matrix.Profile, serv
// Start all volume servers
for i := 0; i < serverCount; i++ {
if err = c.startVolume(i, c.volumeDiskDirs[i]); err != nil {
volumeDataDir := filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
if err = c.startVolume(i, volumeDataDir); err != nil {
// Log current server's log for debugging startup failures
volumeLog := fmt.Sprintf("volume%d.log", i)
c.Stop()
@@ -234,17 +207,13 @@ func (c *MultiVolumeCluster) startMaster(dataDir string) error {
return c.masterCmd.Start()
}
func (c *MultiVolumeCluster) startVolume(index int, dataDirs []string) error {
func (c *MultiVolumeCluster) startVolume(index int, dataDir string) error {
logName := fmt.Sprintf("volume%d.log", index)
logFile, err := os.Create(filepath.Join(c.logsDir, logName))
if err != nil {
return err
}
maxPerDir := make([]string, len(dataDirs))
for i := range dataDirs {
maxPerDir[i] = "16"
}
args := []string{
"-config_dir=" + c.configDir,
"volume",
@@ -252,8 +221,8 @@ func (c *MultiVolumeCluster) startVolume(index int, dataDirs []string) error {
"-port=" + strconv.Itoa(c.volumePorts[index]),
"-port.grpc=" + strconv.Itoa(c.volumeGrpcPorts[index]),
"-port.public=" + strconv.Itoa(c.volumePubPorts[index]),
"-dir=" + strings.Join(dataDirs, ","),
"-max=" + strings.Join(maxPerDir, ","),
"-dir=" + dataDir,
"-max=16",
"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
"-readMode=" + c.profile.ReadMode,
"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
@@ -332,17 +301,3 @@ func (c *MultiVolumeCluster) VolumePublicURL(index int) string {
func (c *MultiVolumeCluster) BaseDir() string {
return c.baseDir
}
// VolumeDiskDir returns the data directory backing disk diskIndex on volume
// server serverIndex — i.e. the directory for that DiskLocation, for file
// inspection in tests.
func (c *MultiVolumeCluster) VolumeDiskDir(serverIndex, diskIndex int) string {
if serverIndex < 0 || serverIndex >= len(c.volumeDiskDirs) {
return ""
}
disks := c.volumeDiskDirs[serverIndex]
if diskIndex < 0 || diskIndex >= len(disks) {
return ""
}
return disks[diskIndex]
}
+1
View File
@@ -48,6 +48,7 @@ var Commands = []*Command{
cmdVolume,
cmdWebDav,
cmdSftp,
cmdNfs,
cmdWorker,
}
-28
View File
@@ -4,7 +4,6 @@ import (
"container/heap"
"fmt"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
@@ -583,30 +582,3 @@ func BenchmarkConflictCheck(b *testing.B) {
})
}
}
// TestMetadataProcessorEmptyMarkerKeepsWatermarkStale: the MaxUnsyncedEvents
// marker (empty EventNotification, fresh timestamp) is dropped by AddSyncJob and
// does NOT advance processedTsWatermark, so offsetFunc keeps publishing the stale
// offset. This is why the client must not drive sync_offset off the watermark
// for these markers.
func TestMetadataProcessorEmptyMarkerKeepsWatermarkStale(t *testing.T) {
const staleOffset = int64(1_000_000_000)
freshTs := staleOffset + int64(time.Hour) // a "now"-ish source timestamp
p := NewMetadataProcessor(func(*filer_pb.SubscribeMetadataResponse) error { return nil }, 4, staleOffset)
marker := &filer_pb.SubscribeMetadataResponse{
TsNs: freshTs,
EventNotification: &filer_pb.EventNotification{},
}
if !filer_pb.IsEmpty(marker) {
t.Fatal("marker should be IsEmpty")
}
p.AddSyncJob(marker)
if got := p.processedTsWatermark.Load(); got != staleOffset {
t.Fatalf("empty marker advanced watermark to %d; want it to stay stale at %d", got, staleOffset)
}
t.Logf("marker carried fresh ts %d but watermark stayed stale at %d", freshTs, staleOffset)
}
+7 -27
View File
@@ -6,7 +6,6 @@ import (
"io/fs"
"os"
"path"
"sort"
"strconv"
"strings"
@@ -191,34 +190,15 @@ func SaveToIdx(scaner *VolumeFileScanner4Fix, idxName string) (ret error) {
idxFile.Close()
}()
// Emit entries in .dat offset (append) order so the .idx stays the
// append-ordered log the volume server writes at runtime — not sorted by
// key, which is the .sdx / .ecx shape. A key-sorted .idx puts the
// highest-key needle last instead of the .dat-tail needle, which used to
// flip volumes read-only on load. Every tombstone is included (including
// any whose live needle is gone) so the last entry is the real .dat tail.
var values []needle_map.NeedleValue
collect := func(value needle_map.NeedleValue) error {
values = append(values, value)
return nil
}
if err = scaner.nm.AscendingVisit(collect); err != nil {
return scaner.nm.AscendingVisit(func(value needle_map.NeedleValue) error {
_, err := idxFile.Write(value.ToBytes())
if scaner.includeDeleted && err == nil {
if deleted, ok := scaner.nmDeleted.Get(value.Key); ok {
_, err = idxFile.Write(deleted.ToBytes())
}
}
return err
}
if scaner.includeDeleted {
if err = scaner.nmDeleted.AscendingVisit(collect); err != nil {
return err
}
}
sort.Slice(values, func(i, j int) bool {
return values[i].Offset.ToActualOffset() < values[j].Offset.ToActualOffset()
})
for _, value := range values {
if _, err = idxFile.Write(value.ToBytes()); err != nil {
return err
}
}
return nil
}
func doFixOneVolume(basepath string, baseFileName string, collection string, volumeId int64, fixIncludeDeleted bool) {
-75
View File
@@ -1,75 +0,0 @@
package command
import (
"os"
"path/filepath"
"testing"
"github.com/seaweedfs/seaweedfs/weed/storage/idx"
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
)
// TestSaveToIdxWritesOffsetOrder guards that weed fix rebuilds the .idx as an
// append-ordered log (sorted by .dat offset), not sorted by key. A key-sorted
// .idx puts the highest-key needle last instead of the .dat-tail needle, which
// flipped volumes read-only on load (issue #9688). It must also carry every
// tombstone — including one whose live needle is gone — so the last entry is
// the real .dat tail.
func TestSaveToIdxWritesOffsetOrder(t *testing.T) {
nm := needle_map.NewMemDb()
defer nm.Close()
nmDeleted := needle_map.NewMemDb()
defer nmDeleted.Close()
// Live needles with high keys at low offsets (as if written first).
for _, e := range []struct {
key uint64
offset int64
}{{30, 8}, {20, 128}, {10, 256}} {
if err := nm.Set(types.Uint64ToNeedleId(e.key), types.ToOffset(e.offset), types.Size(100)); err != nil {
t.Fatalf("nm.Set: %v", err)
}
}
// A tombstone at the .dat tail whose live needle is gone; the old SaveToIdx
// dropped these because the key was absent from the live map.
if err := nmDeleted.Set(types.Uint64ToNeedleId(5), types.ToOffset(384), types.TombstoneFileSize); err != nil {
t.Fatalf("nmDeleted.Set: %v", err)
}
scanner := &VolumeFileScanner4Fix{nm: nm, nmDeleted: nmDeleted, includeDeleted: true}
idxPath := filepath.Join(t.TempDir(), "v.idx")
if err := SaveToIdx(scanner, idxPath); err != nil {
t.Fatalf("SaveToIdx: %v", err)
}
f, err := os.Open(idxPath)
if err != nil {
t.Fatalf("open idx: %v", err)
}
defer f.Close()
var offsets []int64
var lastKey types.NeedleId
if err := idx.WalkIndexFile(f, 0, func(key types.NeedleId, offset types.Offset, size types.Size) error {
offsets = append(offsets, offset.ToActualOffset())
lastKey = key
return nil
}); err != nil {
t.Fatalf("walk idx: %v", err)
}
if len(offsets) != 4 {
t.Fatalf("expected 4 entries (3 live + 1 tombstone), got %d", len(offsets))
}
for i := 1; i < len(offsets); i++ {
if offsets[i] < offsets[i-1] {
t.Fatalf("entries not in offset order: %v", offsets)
}
}
// The .dat-tail needle (the orphan tombstone at the highest offset) must be last.
if lastKey != types.Uint64ToNeedleId(5) {
t.Errorf("last entry should be the .dat-tail tombstone (key 5), got key %v", lastKey)
}
}
+127
View File
@@ -0,0 +1,127 @@
package command
import (
"fmt"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/security"
weed_server_nfs "github.com/seaweedfs/seaweedfs/weed/server/nfs"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/version"
)
var (
nfsStandaloneOptions NfsOptions
)
type NfsOptions struct {
filer *string
ipBind *string
port *int
filerRootPath *string
readOnly *bool
allowedClients *string
volumeServerAccess *string
portmapBind *string
}
func init() {
cmdNfs.Run = runNfs // break init cycle
nfsStandaloneOptions.filer = cmdNfs.Flag.String("filer", "localhost:8888", "filer server address")
nfsStandaloneOptions.ipBind = cmdNfs.Flag.String("ip.bind", "127.0.0.1", "ip address to bind to. Defaults to loopback; override explicitly to expose the experimental server to the network.")
nfsStandaloneOptions.port = cmdNfs.Flag.Int("port", 2049, "NFS server listen port")
nfsStandaloneOptions.filerRootPath = cmdNfs.Flag.String("filer.path", "", "remote path from filer server to export. Required: no default is provided so operators must opt in to exporting a namespace subtree.")
nfsStandaloneOptions.readOnly = cmdNfs.Flag.Bool("readOnly", false, "export the filer path as read only")
nfsStandaloneOptions.allowedClients = cmdNfs.Flag.String("allowedClients", "", "comma-separated client IPs, hostnames, or CIDRs allowed to connect")
nfsStandaloneOptions.volumeServerAccess = cmdNfs.Flag.String("volumeServerAccess", "direct", "access volume servers by [direct|publicUrl|filerProxy]")
nfsStandaloneOptions.portmapBind = cmdNfs.Flag.String("portmap.bind", "", "when set, bind a built-in portmap v2 responder on <ip>:111 so plain `mount -t nfs` works without client-side portmap bypass. Empty disables it. Binding port 111 requires root or CAP_NET_BIND_SERVICE and must not conflict with a system rpcbind.")
}
var cmdNfs = &Command{
UsageLine: "nfs -port=2049 -filer=<ip:port> -filer.path=<exported subtree>",
Short: "start an experimental NFSv3 server backed by a filer",
Long: `start an experimental NFSv3 server backed by a filer.
This command serves an experimental filer-native NFSv3 frontend with
deterministic filehandles, filer-backed metadata operations, and direct
volume-server data access for chunk reads and buffered writes.
Safer defaults (since export ACLs are still not implemented):
- ip.bind defaults to 127.0.0.1, so the server is not reachable from
other hosts unless you override it explicitly.
- filer.path has no default; you must pick the subtree to export.
Override -ip.bind to a routable address only after you have reviewed
-allowedClients and the readiness of the rest of your deployment.
Mounting from a Linux client
----------------------------
The server does not run portmap/rpcbind by default. That means Linux
mount.nfs, which queries portmap on port 111 first, will fail with
"portmap query failed" against the plain form:
mount -t nfs -o nfsvers=3,nolock <host>:/export /mnt
Either tell the client to bypass portmap:
mount -t nfs -o nfsvers=3,nolock,port=2049,mountport=2049,\
proto=tcp,mountproto=tcp <host>:/export /mnt
or enable the built-in portmap responder on the server:
weed nfs ... -portmap.bind=0.0.0.0
With the responder enabled MOUNT v3 is answered over both TCP and UDP,
so the plain mount form above just works — no mountproto override is
required even on clients whose default mountproto is UDP.
Binding port 111 requires root or CAP_NET_BIND_SERVICE and must not
collide with a system rpcbind.
`,
}
func runNfs(cmd *Command, args []string) bool {
util.LoadSecurityConfiguration()
if *nfsStandaloneOptions.ipBind == "" {
*nfsStandaloneOptions.ipBind = "127.0.0.1"
}
if *nfsStandaloneOptions.filerRootPath == "" {
glog.Errorf("-filer.path is required: pick an explicit subtree to export; exporting \"/\" is not a default")
return false
}
if *nfsStandaloneOptions.filerRootPath == "/" {
glog.Warningf("-filer.path=/ exports the entire filer namespace; ensure -allowedClients or -ip.bind constrains access")
}
listenAddress := fmt.Sprintf("%s:%d", *nfsStandaloneOptions.ipBind, *nfsStandaloneOptions.port)
glog.V(0).Infof("Starting Seaweed NFS Server %s at %s", version.Version(), listenAddress)
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
nfsServer, err := weed_server_nfs.NewServer(&weed_server_nfs.Option{
Filer: pb.ServerAddress(*nfsStandaloneOptions.filer),
BindIp: *nfsStandaloneOptions.ipBind,
Port: *nfsStandaloneOptions.port,
FilerRootPath: *nfsStandaloneOptions.filerRootPath,
ReadOnly: *nfsStandaloneOptions.readOnly,
AllowedClients: util.StringSplit(*nfsStandaloneOptions.allowedClients, ","),
VolumeServerAccess: *nfsStandaloneOptions.volumeServerAccess,
GrpcDialOption: grpcDialOption,
PortmapBind: *nfsStandaloneOptions.portmapBind,
})
if err != nil {
glog.Errorf("NFS Server startup error: %v", err)
return false
}
if err := nfsServer.Start(); err != nil {
glog.Errorf("NFS Server startup error: %v", err)
return false
}
return true
}
+2 -2
View File
@@ -21,7 +21,7 @@ func runVersion(cmd *Command, args []string) bool {
fmt.Printf("version %s %s %s\n", version.Version(), runtime.GOOS, runtime.GOARCH)
println()
println("For enterprise users, please visit https://seaweedfs.com for the SeaweedFS Enterprise Edition,")
println("which has advanced features, including data recovery, self-healing storage, customizable erasure coding, EC vacuum and repair, etc.")
println("For enterprise users, please visit https://seaweedfs.com for SeaweedFS Enterprise Edition,")
println("which has a self-healing storage format with better data protection.")
return true
}
@@ -21,7 +21,7 @@ func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []by
dirStr, dirHash, name := GenDirAndName(key)
_, err = db.ExecContext(ctx, store.GetSqlInsert(DEFAULT_TABLE), dirHash, name, dirStr, value)
res, err := db.ExecContext(ctx, store.GetSqlInsert(DEFAULT_TABLE), dirHash, name, dirStr, value)
if err == nil {
return
}
@@ -34,7 +34,7 @@ func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []by
// now the insert failed possibly due to duplication constraints
glog.V(1).InfofCtx(ctx, "kv insert falls back to update: %s", err)
res, err := db.ExecContext(ctx, store.GetSqlUpdate(DEFAULT_TABLE), value, dirHash, name, dirStr)
res, err = db.ExecContext(ctx, store.GetSqlUpdate(DEFAULT_TABLE), value, dirHash, name, dirStr)
if err != nil {
return fmt.Errorf("kv upsert: %s", err)
}
+3
View File
@@ -13,6 +13,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
"github.com/seaweedfs/seaweedfs/weed/filer/empty_folder_cleanup"
"github.com/seaweedfs/seaweedfs/weed/sequence"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -63,6 +64,7 @@ type Filer struct {
DeletionRetryQueue *DeletionRetryQueue
EmptyFolderCleaner *empty_folder_cleanup.EmptyFolderCleaner
EmptyFolderCleanupDelay time.Duration
inodeSequencer sequence.Sequencer
}
func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, filerGroup string, collection string, replication string, dataCenter string, maxFilenameLength uint32, notifyFn func()) *Filer {
@@ -77,6 +79,7 @@ func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerH
MaxFilenameLength: maxFilenameLength,
deletionQuit: make(chan struct{}),
DeletionRetryQueue: NewDeletionRetryQueue(),
inodeSequencer: newInodeSequencer(filerHost),
}
if f.UniqueFilerId < 0 {
f.UniqueFilerId = -f.UniqueFilerId
+40 -15
View File
@@ -1,26 +1,51 @@
package filer
import (
"time"
"os"
"strconv"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/sequence"
)
// ensureEntryInode derives a stable inode the same way the FUSE mount does, so
// the filer-stored value matches what a mount would otherwise compute and no
// per-object reverse index is required. Hard links hash their shared
// HardLinkId, so every link resolves to one inode; other entries hash the path
// and creation time.
// newInodeSequencer constructs the inode sequencer used to assign object
// identity for filer entries. The Snowflake node id defaults to a masked hash
// of filerHost, which only has 1024 possible values; operators running a
// multi-filer cluster should set SEAWEEDFS_FILER_SNOWFLAKE_ID to an explicit
// per-filer value (1..1023) to avoid birthday-paradox collisions.
//
// Initialization failures are fatal: a process-local fallback allocator would
// re-use inode values across restarts and violate the stable object identity
// guarantee that NFS filehandles and the inode secondary index rely on.
func newInodeSequencer(filerHost pb.ServerAddress) sequence.Sequencer {
snowflakeId := parseSnowflakeIdFromEnv()
seq, err := sequence.NewSnowflakeSequencer(string(filerHost), snowflakeId)
if err != nil {
glog.Fatalf("initialize inode sequencer for filer %s (snowflakeId=%d): %v", filerHost, snowflakeId, err)
}
return seq
}
func parseSnowflakeIdFromEnv() int {
raw := os.Getenv("SEAWEEDFS_FILER_SNOWFLAKE_ID")
if raw == "" {
return 0
}
id, err := strconv.Atoi(raw)
if err != nil || id < 0 || id > 0x3ff {
glog.Fatalf("SEAWEEDFS_FILER_SNOWFLAKE_ID must be an integer in [0,1023], got %q", raw)
}
return id
}
func (f *Filer) ensureEntryInode(entry *Entry) {
if entry == nil || entry.Attr.Inode != 0 {
return
}
if entry.Attr.Crtime.IsZero() {
entry.Attr.Crtime = time.Now()
}
if len(entry.HardLinkId) > 0 {
entry.Attr.Inode = uint64(util.HashStringToLong(string(entry.HardLinkId)))
return
}
entry.Attr.Inode = entry.FullPath.AsInode(entry.Attr.Crtime.Unix())
entry.Attr.Inode = f.nextInode()
}
func (f *Filer) nextInode() uint64 {
return f.inodeSequencer.NextFileId(1)
}
+300
View File
@@ -0,0 +1,300 @@
package filer
import (
"context"
"encoding/binary"
"encoding/json"
"sort"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/util"
)
const inodeIndexKeyPrefix = "filer.inode.path."
const InodeIndexInitialGeneration uint64 = 1
type inodeIndexEntry struct {
path util.FullPath
inode uint64
}
type InodeIndexRecord struct {
Generation uint64 `json:"generation,omitempty"`
Paths []string `json:"paths,omitempty"`
}
func InodeIndexKey(inode uint64) []byte {
key := make([]byte, len(inodeIndexKeyPrefix)+8)
copy(key, inodeIndexKeyPrefix)
binary.BigEndian.PutUint64(key[len(inodeIndexKeyPrefix):], inode)
return key
}
func DecodeInodeIndexRecord(value []byte) (*InodeIndexRecord, error) {
if len(value) == 0 {
return &InodeIndexRecord{}, nil
}
// The first foundation slice stored the current path as raw bytes. Keep that
// format readable so existing records are transparently upgraded on write.
if value[0] != '{' {
record := &InodeIndexRecord{Generation: InodeIndexInitialGeneration}
record.addPath(util.FullPath(value))
return record, nil
}
record := &InodeIndexRecord{}
if err := json.Unmarshal(value, record); err != nil {
return nil, err
}
record.normalize()
return record, nil
}
func (record *InodeIndexRecord) Encode() ([]byte, error) {
record.normalize()
return json.Marshal(record)
}
func (record *InodeIndexRecord) normalize() {
if len(record.Paths) == 0 {
return
}
if record.Generation == 0 {
record.Generation = InodeIndexInitialGeneration
}
sanitized := make([]string, 0, len(record.Paths))
for _, path := range record.Paths {
if path == "" {
continue
}
sanitized = append(sanitized, path)
}
if len(sanitized) == 0 {
record.Paths = nil
return
}
sort.Strings(sanitized)
deduped := sanitized[:1]
for _, path := range sanitized[1:] {
if path == deduped[len(deduped)-1] {
continue
}
deduped = append(deduped, path)
}
record.Paths = deduped
}
func (record *InodeIndexRecord) addPath(path util.FullPath) bool {
if path == "" {
return false
}
record.normalize()
target := string(path)
index := sort.SearchStrings(record.Paths, target)
if index < len(record.Paths) && record.Paths[index] == target {
return false
}
record.Paths = append(record.Paths, "")
copy(record.Paths[index+1:], record.Paths[index:])
record.Paths[index] = target
return true
}
func (record *InodeIndexRecord) removePath(path util.FullPath) bool {
if len(record.Paths) == 0 || path == "" {
return false
}
record.normalize()
target := string(path)
index := sort.SearchStrings(record.Paths, target)
if index >= len(record.Paths) || record.Paths[index] != target {
return false
}
record.Paths = append(record.Paths[:index], record.Paths[index+1:]...)
if len(record.Paths) == 0 {
record.Paths = nil
}
return true
}
func (record *InodeIndexRecord) CanonicalPath() util.FullPath {
record.normalize()
if len(record.Paths) == 0 {
return ""
}
return util.FullPath(record.Paths[0])
}
func (record *InodeIndexRecord) FullPaths() []util.FullPath {
record.normalize()
if len(record.Paths) == 0 {
return nil
}
paths := make([]util.FullPath, 0, len(record.Paths))
for _, path := range record.Paths {
paths = append(paths, util.FullPath(path))
}
return paths
}
func (fsw *FilerStoreWrapper) lookupInodeIndex(ctx context.Context, inode uint64) (*InodeIndexRecord, error) {
if inode == 0 {
return nil, ErrKvNotFound
}
value, err := fsw.KvGet(ctx, InodeIndexKey(inode))
if err != nil {
return nil, err
}
return DecodeInodeIndexRecord(value)
}
func (fsw *FilerStoreWrapper) storeInodeIndex(ctx context.Context, path util.FullPath, inode uint64) error {
if inode == 0 || path == "" {
return nil
}
record, err := fsw.lookupInodeIndex(ctx, inode)
if err != nil {
if err != ErrKvNotFound {
return err
}
record = &InodeIndexRecord{Generation: InodeIndexInitialGeneration}
}
record.addPath(path)
value, err := record.Encode()
if err != nil {
return err
}
return fsw.KvPut(ctx, InodeIndexKey(inode), value)
}
func (fsw *FilerStoreWrapper) lookupInodePath(ctx context.Context, inode uint64) (util.FullPath, error) {
record, err := fsw.lookupInodeIndex(ctx, inode)
if err != nil {
return "", err
}
path := record.CanonicalPath()
if path == "" {
return "", ErrKvNotFound
}
return path, nil
}
func (fsw *FilerStoreWrapper) lookupInodePaths(ctx context.Context, inode uint64) ([]util.FullPath, error) {
record, err := fsw.lookupInodeIndex(ctx, inode)
if err != nil {
return nil, err
}
paths := record.FullPaths()
if len(paths) == 0 {
return nil, ErrKvNotFound
}
return paths, nil
}
func (fsw *FilerStoreWrapper) removePathFromInodeIndex(ctx context.Context, path util.FullPath, inode uint64) error {
if inode == 0 || path == "" {
return nil
}
record, err := fsw.lookupInodeIndex(ctx, inode)
if err != nil {
if err == ErrKvNotFound {
return nil
}
return err
}
if !record.removePath(path) {
return nil
}
if len(record.Paths) == 0 {
return fsw.KvDelete(ctx, InodeIndexKey(inode))
}
value, err := record.Encode()
if err != nil {
return err
}
return fsw.KvPut(ctx, InodeIndexKey(inode), value)
}
func (fsw *FilerStoreWrapper) collectInodeIndexEntries(ctx context.Context, dirPath util.FullPath) ([]inodeIndexEntry, error) {
// Honor caller cancellation during the walk: a DeleteFolderChildren on a
// pathological directory could otherwise loop indefinitely gathering
// entries even after the client has given up, turning into a DoS vector.
// If the walk is aborted, the caller treats the index cleanup as
// best-effort and drops the partial result.
var collected []inodeIndexEntry
if err := fsw.collectInodeIndexEntriesRecursive(ctx, dirPath, &collected); err != nil {
return nil, err
}
return collected, nil
}
func (fsw *FilerStoreWrapper) collectInodeIndexEntriesRecursive(ctx context.Context, dirPath util.FullPath, collected *[]inodeIndexEntry) error {
actualStore := fsw.getActualStore(dirPath + "/")
lastFileName := ""
includeStartFile := false
for {
page := make([]*Entry, 0, PaginationSize)
nextLastFileName, err := actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, includeStartFile, PaginationSize, func(entry *Entry) (bool, error) {
page = append(page, entry)
return true, nil
})
if err != nil {
return err
}
for _, entry := range page {
if entry.Attr.Inode != 0 {
*collected = append(*collected, inodeIndexEntry{path: entry.FullPath, inode: entry.Attr.Inode})
}
if entry.IsDirectory() {
if err := fsw.collectInodeIndexEntriesRecursive(ctx, entry.FullPath, collected); err != nil {
return err
}
}
}
if len(page) < PaginationSize {
return nil
}
lastFileName = nextLastFileName
includeStartFile = false
}
}
// recordInodeIndexWrite updates the inode→path secondary index after the
// primary store mutation has already succeeded. The index is best-effort: a
// failure here must not surface as an operation error, because the caller
// would then observe a failed create/update even though the entry was
// persisted, and a retry cannot heal the index (DeleteEntry exits early once
// the entry is gone). We log and let later writes rebuild the record.
func (fsw *FilerStoreWrapper) recordInodeIndexWrite(ctx context.Context, op string, path util.FullPath, inode uint64) {
if inode == 0 || path == "" {
return
}
if err := fsw.storeInodeIndex(ctx, path, inode); err != nil {
glog.WarningfCtx(ctx, "%s: update inode index for %s (inode %d): %v", op, path, inode, err)
}
}
// recordInodeIndexRemoval mirrors recordInodeIndexWrite for removals.
func (fsw *FilerStoreWrapper) recordInodeIndexRemoval(ctx context.Context, op string, path util.FullPath, inode uint64) {
if inode == 0 || path == "" {
return
}
if err := fsw.removePathFromInodeIndex(ctx, path, inode); err != nil {
glog.WarningfCtx(ctx, "%s: clear inode index for %s (inode %d): %v", op, path, inode, err)
}
}
+206
View File
@@ -0,0 +1,206 @@
package filer
import (
"context"
"os"
"testing"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestFilerStoreWrapperMaintainsInodeIndexLifecycle(t *testing.T) {
wrapper := NewFilerStoreWrapper(newStubFilerStore())
ctx := context.Background()
created := &Entry{
FullPath: util.FullPath("/docs/report.txt"),
Attr: Attr{
Mode: 0o644,
Inode: 42,
},
}
require.NoError(t, wrapper.InsertEntry(ctx, created))
path, err := wrapper.lookupInodePath(ctx, created.Attr.Inode)
require.NoError(t, err)
assert.Equal(t, created.FullPath, path)
paths, err := wrapper.lookupInodePaths(ctx, created.Attr.Inode)
require.NoError(t, err)
assert.Equal(t, []util.FullPath{created.FullPath}, paths)
record, err := wrapper.lookupInodeIndex(ctx, created.Attr.Inode)
require.NoError(t, err)
assert.Equal(t, InodeIndexInitialGeneration, record.Generation)
updated := &Entry{
FullPath: util.FullPath("/docs/report.txt"),
Attr: Attr{
Mode: 0o600,
Inode: 42,
},
}
require.NoError(t, wrapper.UpdateEntry(ctx, updated))
path, err = wrapper.lookupInodePath(ctx, updated.Attr.Inode)
require.NoError(t, err)
assert.Equal(t, updated.FullPath, path)
require.NoError(t, wrapper.DeleteEntry(ctx, created.FullPath))
_, err = wrapper.lookupInodePath(ctx, created.Attr.Inode)
require.ErrorIs(t, err, ErrKvNotFound)
}
func TestFilerStoreWrapperMaintainsMultiplePathsPerInode(t *testing.T) {
wrapper := NewFilerStoreWrapper(newStubFilerStore())
ctx := context.Background()
inode := uint64(88)
hardLinkId := NewHardLinkId()
require.NoError(t, wrapper.InsertEntry(ctx, &Entry{
FullPath: util.FullPath("/links/b.txt"),
Attr: Attr{
Mode: 0o644,
Inode: inode,
},
HardLinkId: hardLinkId,
HardLinkCounter: 2,
}))
require.NoError(t, wrapper.InsertEntry(ctx, &Entry{
FullPath: util.FullPath("/links/a.txt"),
Attr: Attr{
Mode: 0o644,
Inode: inode,
},
HardLinkId: hardLinkId,
HardLinkCounter: 2,
}))
paths, err := wrapper.lookupInodePaths(ctx, inode)
require.NoError(t, err)
assert.Equal(t, []util.FullPath{"/links/a.txt", "/links/b.txt"}, paths)
record, err := wrapper.lookupInodeIndex(ctx, inode)
require.NoError(t, err)
assert.Equal(t, InodeIndexInitialGeneration, record.Generation)
path, err := wrapper.lookupInodePath(ctx, inode)
require.NoError(t, err)
assert.Equal(t, util.FullPath("/links/a.txt"), path)
require.NoError(t, wrapper.DeleteEntry(ctx, util.FullPath("/links/a.txt")))
paths, err = wrapper.lookupInodePaths(ctx, inode)
require.NoError(t, err)
assert.Equal(t, []util.FullPath{"/links/b.txt"}, paths)
path, err = wrapper.lookupInodePath(ctx, inode)
require.NoError(t, err)
assert.Equal(t, util.FullPath("/links/b.txt"), path)
}
func TestFilerStoreWrapperUpgradesLegacySinglePathInodeIndexRecords(t *testing.T) {
wrapper := NewFilerStoreWrapper(newStubFilerStore())
ctx := context.Background()
inode := uint64(91)
require.NoError(t, wrapper.KvPut(ctx, InodeIndexKey(inode), []byte("/legacy/path.txt")))
path, err := wrapper.lookupInodePath(ctx, inode)
require.NoError(t, err)
assert.Equal(t, util.FullPath("/legacy/path.txt"), path)
paths, err := wrapper.lookupInodePaths(ctx, inode)
require.NoError(t, err)
assert.Equal(t, []util.FullPath{"/legacy/path.txt"}, paths)
require.NoError(t, wrapper.storeInodeIndex(ctx, util.FullPath("/legacy/second.txt"), inode))
paths, err = wrapper.lookupInodePaths(ctx, inode)
require.NoError(t, err)
assert.Equal(t, []util.FullPath{"/legacy/path.txt", "/legacy/second.txt"}, paths)
value, err := wrapper.KvGet(ctx, InodeIndexKey(inode))
require.NoError(t, err)
assert.JSONEq(t, `{"generation":1,"paths":["/legacy/path.txt","/legacy/second.txt"]}`, string(value))
}
func TestFilerStoreWrapperKeepsInodeIndexWhenDeleteArrivesAfterRenameInsert(t *testing.T) {
wrapper := NewFilerStoreWrapper(newStubFilerStore())
ctx := context.Background()
inode := uint64(77)
require.NoError(t, wrapper.InsertEntry(ctx, &Entry{
FullPath: util.FullPath("/old/name.txt"),
Attr: Attr{
Mode: 0o644,
Inode: inode,
},
}))
require.NoError(t, wrapper.InsertEntry(ctx, &Entry{
FullPath: util.FullPath("/new/name.txt"),
Attr: Attr{
Mode: 0o644,
Inode: inode,
},
}))
require.NoError(t, wrapper.DeleteEntry(ctx, util.FullPath("/old/name.txt")))
path, err := wrapper.lookupInodePath(ctx, inode)
require.NoError(t, err)
assert.Equal(t, util.FullPath("/new/name.txt"), path)
paths, err := wrapper.lookupInodePaths(ctx, inode)
require.NoError(t, err)
assert.Equal(t, []util.FullPath{"/new/name.txt"}, paths)
}
func TestRecursiveDeleteRemovesDescendantInodeIndexes(t *testing.T) {
f, store := newTestFilerWithStubStore()
ctx := context.Background()
entries := []*Entry{
{
FullPath: util.FullPath("/tree"),
Attr: Attr{
Mode: os.ModeDir | 0o755,
Inode: 100,
},
},
{
FullPath: util.FullPath("/tree/file.txt"),
Attr: Attr{
Mode: 0o644,
Inode: 101,
},
},
{
FullPath: util.FullPath("/tree/subdir"),
Attr: Attr{
Mode: os.ModeDir | 0o755,
Inode: 102,
},
},
{
FullPath: util.FullPath("/tree/subdir/nested.txt"),
Attr: Attr{
Mode: 0o644,
Inode: 103,
},
},
}
for _, entry := range entries {
require.NoError(t, f.Store.InsertEntry(ctx, entry))
}
require.NoError(t, f.DeleteEntryMetaAndData(ctx, util.FullPath("/tree"), true, false, false, false, nil, 0))
for _, inode := range []uint64{100, 101, 102, 103} {
_, err := f.Store.(*FilerStoreWrapper).lookupInodePath(ctx, inode)
require.ErrorIs(t, err, ErrKvNotFound)
}
for _, path := range []string{"/tree", "/tree/file.txt", "/tree/subdir", "/tree/subdir/nested.txt"} {
_, err := store.FindEntry(ctx, util.FullPath(path))
require.Error(t, err)
}
}
-42
View File
@@ -4,7 +4,6 @@ import (
"context"
"os"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/util"
@@ -12,47 +11,6 @@ import (
"github.com/stretchr/testify/require"
)
func TestEnsureEntryInodeMatchesFuseDerivation(t *testing.T) {
f := &Filer{}
crtime := time.Unix(1700000000, 0)
entry := &Entry{
FullPath: util.FullPath("/dir/file.txt"),
Attr: Attr{Crtime: crtime},
}
f.ensureEntryInode(entry)
// The filer stores exactly what the FUSE mount would compute for a
// non-hard-linked entry, and it is deterministic across calls.
assert.Equal(t, entry.FullPath.AsInode(crtime.Unix()), entry.Attr.Inode)
again := &Entry{FullPath: entry.FullPath, Attr: Attr{Crtime: crtime}}
f.ensureEntryInode(again)
assert.Equal(t, entry.Attr.Inode, again.Attr.Inode)
}
func TestEnsureEntryInodeSharesAcrossHardLinks(t *testing.T) {
f := &Filer{}
hardLinkId := NewHardLinkId()
a := &Entry{
FullPath: util.FullPath("/links/a.txt"),
Attr: Attr{Crtime: time.Unix(1700000000, 0)},
HardLinkId: hardLinkId,
}
b := &Entry{
FullPath: util.FullPath("/links/b.txt"),
Attr: Attr{Crtime: time.Unix(1800000000, 0)},
HardLinkId: hardLinkId,
}
f.ensureEntryInode(a)
f.ensureEntryInode(b)
// Every link to the same target resolves to one inode, independent of path
// or creation time.
assert.Equal(t, uint64(util.HashStringToLong(string(hardLinkId))), a.Attr.Inode)
assert.Equal(t, a.Attr.Inode, b.Attr.Inode)
}
func newTestFilerWithStubStore() (*Filer, *stubFilerStore) {
store := newStubFilerStore()
f := NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
+1
View File
@@ -276,6 +276,7 @@ func newTestFiler(t *testing.T, store *stubFilerStore, rs *FilerRemoteStorage) *
FilerConf: NewFilerConf(),
MaxFilenameLength: 255,
MasterClient: mc,
inodeSequencer: newInodeSequencer("test-filer"),
fileIdDeletionQueue: util.NewUnboundedQueue(),
deletionQuit: make(chan struct{}),
LocalMetaLogBuffer: log_buffer.NewLogBuffer("test", time.Minute,
+46 -6
View File
@@ -132,6 +132,7 @@ func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) err
return err
}
ctx = context.WithoutCancel(ctx)
fullPath := entry.FullPath
actualStore := fsw.getActualStore(entry.FullPath)
stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "insert").Inc()
start := time.Now()
@@ -151,7 +152,11 @@ func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) err
return err
}
return actualStore.InsertEntry(ctx, entry)
if err := actualStore.InsertEntry(ctx, entry); err != nil {
return err
}
fsw.recordInodeIndexWrite(ctx, "InsertEntry", fullPath, entry.Attr.Inode)
return nil
}
// InsertEntryKnownAbsent skips the pre-insert FindEntry path when the caller has
@@ -161,6 +166,7 @@ func (fsw *FilerStoreWrapper) InsertEntryKnownAbsent(ctx context.Context, entry
return err
}
ctx = context.WithoutCancel(ctx)
fullPath := entry.FullPath
actualStore := fsw.getActualStore(entry.FullPath)
stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "insert").Inc()
start := time.Now()
@@ -179,7 +185,11 @@ func (fsw *FilerStoreWrapper) InsertEntryKnownAbsent(ctx context.Context, entry
}
}
return actualStore.InsertEntry(ctx, entry)
if err := actualStore.InsertEntry(ctx, entry); err != nil {
return err
}
fsw.recordInodeIndexWrite(ctx, "InsertEntryKnownAbsent", fullPath, entry.Attr.Inode)
return nil
}
func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) error {
@@ -187,6 +197,7 @@ func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) err
return err
}
ctx = context.WithoutCancel(ctx)
fullPath := entry.FullPath
actualStore := fsw.getActualStore(entry.FullPath)
stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "update").Inc()
start := time.Now()
@@ -206,7 +217,11 @@ func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) err
return err
}
return actualStore.UpdateEntry(ctx, entry)
if err := actualStore.UpdateEntry(ctx, entry); err != nil {
return err
}
fsw.recordInodeIndexWrite(ctx, "UpdateEntry", fullPath, entry.Attr.Inode)
return nil
}
func normalizeEntryMimeForStore(entry *Entry) {
@@ -258,6 +273,8 @@ func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp util.FullPath)
if findErr == filer_pb.ErrNotFound || existingEntry == nil {
return nil
}
inode := existingEntry.Attr.Inode
fullPath := existingEntry.FullPath
if len(existingEntry.HardLinkId) != 0 {
// remove hard link
op := ctx.Value("OP")
@@ -272,7 +289,11 @@ func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp util.FullPath)
}
}
return actualStore.DeleteEntry(ctx, fp)
if err := actualStore.DeleteEntry(ctx, fp); err != nil {
return err
}
fsw.recordInodeIndexRemoval(ctx, "DeleteEntry", fullPath, inode)
return nil
}
func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry *Entry) (err error) {
@@ -280,6 +301,8 @@ func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry
return err
}
ctx = context.WithoutCancel(ctx)
fullPath := existingEntry.FullPath
inode := existingEntry.Attr.Inode
actualStore := fsw.getActualStore(existingEntry.FullPath)
stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "delete").Inc()
start := time.Now()
@@ -302,7 +325,11 @@ func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry
}
}
return actualStore.DeleteEntry(ctx, existingEntry.FullPath)
if err := actualStore.DeleteEntry(ctx, existingEntry.FullPath); err != nil {
return err
}
fsw.recordInodeIndexRemoval(ctx, "DeleteOneEntry", fullPath, inode)
return nil
}
func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) {
@@ -317,7 +344,20 @@ func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.
stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "deleteFolderChildren").Observe(time.Since(start).Seconds())
}()
return actualStore.DeleteFolderChildren(ctx, fp)
collected, err := fsw.collectInodeIndexEntries(ctx, fp)
if err != nil {
// Index collection is best-effort: a failure here only prevents inode
// index housekeeping, not the directory removal itself.
glog.WarningfCtx(ctx, "collectInodeIndexEntries %s: %v; deleting folder children without index cleanup", fp, err)
collected = nil
}
if err := actualStore.DeleteFolderChildren(ctx, fp); err != nil {
return err
}
for _, entry := range collected {
fsw.recordInodeIndexRemoval(ctx, "DeleteFolderChildren", entry.path, entry.inode)
}
return nil
}
func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (string, error) {
-7
View File
@@ -13,13 +13,6 @@ type SqlGenMysql struct {
UpsertQueryTemplate string
}
// DefaultUpsertQuery keeps INSERTs idempotent so the inode-index KvPut
// after every entry write does not emit a duplicate-key roundtrip on
// every mutation. The VALUES() form works on MariaDB and MySQL >=5.7;
// the newer "AS new" alias from MySQL 8.0.19 errors on MariaDB, so it
// is left for users to opt into via an explicit upsertQuery.
const DefaultUpsertQuery = "INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE `meta` = VALUES(`meta`)"
var (
_ = abstract_sql.SqlGenerator(&SqlGenMysql{})
)
-28
View File
@@ -1,28 +0,0 @@
package mysql
import (
"strings"
"testing"
)
func TestDefaultUpsertQueryUsesOnDuplicateKey(t *testing.T) {
gen := &SqlGenMysql{UpsertQueryTemplate: DefaultUpsertQuery}
got := gen.GetSqlInsert("filemeta")
if !strings.Contains(got, "ON DUPLICATE KEY UPDATE") {
t.Fatalf("expected ON DUPLICATE KEY UPDATE in default upsert, got: %s", got)
}
if strings.Contains(got, "AS `new`") {
t.Fatalf("default should avoid MySQL 8.0.19 row-alias syntax for MariaDB compat, got: %s", got)
}
if !strings.Contains(got, "`filemeta`") {
t.Fatalf("expected backticked table name, got: %s", got)
}
}
func TestEmptyUpsertTemplateFallsBackToPlainInsert(t *testing.T) {
gen := &SqlGenMysql{}
got := gen.GetSqlInsert("filemeta")
if strings.Contains(got, "ON DUPLICATE KEY UPDATE") {
t.Fatalf("plain INSERT path should not contain ON DUPLICATE KEY UPDATE, got: %s", got)
}
}
-5
View File
@@ -35,9 +35,6 @@ func (store *MysqlStore) GetName() string {
func (store *MysqlStore) Initialize(configuration util.Configuration, prefix string) (err error) {
// Absent key keeps a pooled default; an explicit 0 disables the idle pool.
configuration.SetDefault(prefix+"connection_max_idle", 2)
// Default on so minimal configs avoid the duplicate-key roundtrip the
// inode-index KvPut would otherwise emit on every write.
configuration.SetDefault(prefix+"enableUpsert", true)
return store.initialize(
configuration.GetString(prefix+"dsn"),
configuration.GetString(prefix+"upsertQuery"),
@@ -67,8 +64,6 @@ func (store *MysqlStore) initialize(dsn string, upsertQuery string, enableUpsert
store.SupportBucketTable = false
if !enableUpsert {
upsertQuery = ""
} else if upsertQuery == "" {
upsertQuery = DefaultUpsertQuery
}
store.SqlGenerator = &SqlGenMysql{
CreateTableSqlTemplate: "",
-5
View File
@@ -35,9 +35,6 @@ func (store *MysqlStore2) GetName() string {
func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
// Absent key keeps a pooled default; an explicit 0 disables the idle pool.
configuration.SetDefault(prefix+"connection_max_idle", 2)
// Default on so minimal configs avoid the duplicate-key roundtrip the
// inode-index KvPut would otherwise emit on every write.
configuration.SetDefault(prefix+"enableUpsert", true)
return store.initialize(
configuration.GetString(prefix+"createTable"),
configuration.GetString(prefix+"upsertQuery"),
@@ -60,8 +57,6 @@ func (store *MysqlStore2) initialize(createTable, upsertQuery string, enableUpse
store.SupportBucketTable = true
if !enableUpsert {
upsertQuery = ""
} else if upsertQuery == "" {
upsertQuery = mysql.DefaultUpsertQuery
}
store.SqlGenerator = &mysql.SqlGenMysql{
CreateTableSqlTemplate: createTable,
-5
View File
@@ -13,11 +13,6 @@ type SqlGenPostgres struct {
UpsertQueryTemplate string
}
// DefaultUpsertQuery keeps INSERTs idempotent so a duplicate-key failure
// (23505) cannot poison the surrounding transaction (25P02). Used when the
// user enables upsert but does not provide their own template.
const DefaultUpsertQuery = `INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash, name) DO UPDATE SET directory=EXCLUDED.directory, meta=EXCLUDED.meta`
var (
_ = abstract_sql.SqlGenerator(&SqlGenPostgres{})
)
@@ -1,25 +0,0 @@
package postgres
import (
"strings"
"testing"
)
func TestDefaultUpsertQueryIsConflictSafe(t *testing.T) {
gen := &SqlGenPostgres{UpsertQueryTemplate: DefaultUpsertQuery}
got := gen.GetSqlInsert("filemeta")
if !strings.Contains(got, "ON CONFLICT") {
t.Fatalf("expected ON CONFLICT in default upsert, got: %s", got)
}
if !strings.Contains(got, `"filemeta"`) {
t.Fatalf("expected quoted table name, got: %s", got)
}
}
func TestEmptyUpsertTemplateFallsBackToPlainInsert(t *testing.T) {
gen := &SqlGenPostgres{}
got := gen.GetSqlInsert("filemeta")
if strings.Contains(got, "ON CONFLICT") {
t.Fatalf("plain INSERT path should not contain ON CONFLICT, got: %s", got)
}
}
-5
View File
@@ -30,9 +30,6 @@ func (store *PostgresStore) GetName() string {
func (store *PostgresStore) Initialize(configuration util.Configuration, prefix string) (err error) {
// Absent key keeps a pooled default; an explicit 0 disables the idle pool.
configuration.SetDefault(prefix+"connection_max_idle", 2)
// Default on so minimal configs are not exposed to duplicate-key tx
// poisoning on Postgres; an explicit false still disables it.
configuration.SetDefault(prefix+"enableUpsert", true)
return store.initialize(
configuration.GetString(prefix+"upsertQuery"),
configuration.GetBool(prefix+"enableUpsert"),
@@ -59,8 +56,6 @@ func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, us
store.SupportBucketTable = false
if !enableUpsert {
upsertQuery = ""
} else if upsertQuery == "" {
upsertQuery = DefaultUpsertQuery
}
store.SqlGenerator = &SqlGenPostgres{
CreateTableSqlTemplate: "",
-5
View File
@@ -35,9 +35,6 @@ func (store *PostgresStore2) GetName() string {
func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
// Absent key keeps a pooled default; an explicit 0 disables the idle pool.
configuration.SetDefault(prefix+"connection_max_idle", 2)
// Default on so minimal configs are not exposed to duplicate-key tx
// poisoning on Postgres; an explicit false still disables it.
configuration.SetDefault(prefix+"enableUpsert", true)
return store.initialize(
configuration.GetString(prefix+"createTable"),
configuration.GetString(prefix+"upsertQuery"),
@@ -65,8 +62,6 @@ func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableU
store.SupportBucketTable = true
if !enableUpsert {
upsertQuery = ""
} else if upsertQuery == "" {
upsertQuery = postgres.DefaultUpsertQuery
}
store.SqlGenerator = &postgres.SqlGenPostgres{
CreateTableSqlTemplate: createTable,
+1 -13
View File
@@ -124,23 +124,11 @@ func (iama *IamApiServer) registerRouter(router *mux.Router) {
// apiRouter.Methods("GET").Path("/").HandlerFunc(track(s3a.iam.Auth(s3a.ListBucketsHandler, ACTION_ADMIN), "LIST"))
apiRouter.Methods(http.MethodPost).Path("/").HandlerFunc(iama.iam.Auth(iama.DoActions, ACTION_ADMIN))
// Health probes
apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/healthz").HandlerFunc(iama.healthzHandler)
apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/readyz").HandlerFunc(iama.readyzHandler)
//
// NotFound
apiRouter.NotFoundHandler = http.HandlerFunc(s3err.NotFoundHandler)
}
func (iama *IamApiServer) healthzHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}
func (iama *IamApiServer) readyzHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}
// Shutdown gracefully stops the IAM API server and releases resources.
// It cancels the master client connection goroutine and closes gRPC connections.
// This method is safe to call multiple times.
+2 -14
View File
@@ -40,7 +40,6 @@ type UploadOption struct {
Md5 string
BytesBuffer *bytes.Buffer
SourceUrl string // optional: for logging when reading from a remote source
MaxAttempts int // <=0 uses the default
}
type UploadResult struct {
@@ -248,26 +247,15 @@ func (uploader *Uploader) doUpload(ctx context.Context, reader io.Reader, option
}
func (uploader *Uploader) retriedUploadData(ctx context.Context, data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
maxAttempts := option.MaxAttempts
if maxAttempts <= 0 {
maxAttempts = 3
}
for i := 0; i < maxAttempts; i++ {
for i := 0; i < 3; i++ {
if i > 0 {
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(time.Millisecond * time.Duration(237*(i+1))):
}
time.Sleep(time.Millisecond * time.Duration(237*(i+1)))
}
uploadResult, err = uploader.doUploadData(ctx, data, option)
if err == nil {
uploadResult.RetryCount = i
return
}
if ctx.Err() != nil {
return nil, ctx.Err()
}
glog.WarningfCtx(ctx, "uploading %d to %s: %v", i, option.UploadUrl, err)
}
return
-99
View File
@@ -10,7 +10,6 @@ import (
"strings"
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/security"
)
@@ -199,101 +198,3 @@ func TestUploadRewindsBodyOnConnectionReset(t *testing.T) {
})
}
}
// deadReplicaClient simulates an unreachable replica: every request blocks for
// dialDelay (standing in for the TCP dial timeout) and then fails, unless the
// request context is cancelled first.
type deadReplicaClient struct {
mu sync.Mutex
calls int
dialDelay time.Duration
}
func (c *deadReplicaClient) attempts() int {
c.mu.Lock()
defer c.mu.Unlock()
return c.calls
}
func (c *deadReplicaClient) Do(req *http.Request) (*http.Response, error) {
c.mu.Lock()
c.calls++
c.mu.Unlock()
select {
case <-time.After(c.dialDelay):
return nil, fmt.Errorf("dial tcp %s: i/o timeout", req.URL.Host)
case <-req.Context().Done():
return nil, req.Context().Err()
}
}
// TestUploadToDeadReplicaRetriesThreeTimes reproduces the dead-replica upload
// stall: a synchronous replica write to an unreachable host pays the dial
// timeout three times over, so one dead replica stalls the caller for ~3x the
// dial timeout. In production dialDelay is the 10s dialer timeout, so the
// caller blocks ~30s before the failure surfaces.
func TestUploadToDeadReplicaRetriesThreeTimes(t *testing.T) {
client := &deadReplicaClient{dialDelay: 100 * time.Millisecond}
uploader := newUploader(client)
start := time.Now()
_, err := uploader.UploadData(context.Background(), []byte("hello"), &UploadOption{
UploadUrl: "http://dead-replica:8080/3,01",
Filename: "test.bin",
})
elapsed := time.Since(start)
if err == nil {
t.Fatal("expected an error uploading to a dead replica")
}
if got := client.attempts(); got != 3 {
t.Fatalf("dial attempts = %d, want 3 (each attempt pays the full dial timeout)", got)
}
t.Logf("dead replica stalled the caller for %v across %d attempts", elapsed, client.attempts())
}
// TestUploadToDeadReplicaSingleAttempt verifies the fix: a synchronous replica
// write makes a single attempt (MaxAttempts=1), so a dead replica fails after
// one dial timeout instead of three. The outer client write still retries.
func TestUploadToDeadReplicaSingleAttempt(t *testing.T) {
client := &deadReplicaClient{dialDelay: 100 * time.Millisecond}
uploader := newUploader(client)
_, err := uploader.UploadData(context.Background(), []byte("hello"), &UploadOption{
UploadUrl: "http://dead-replica:8080/3,01",
Filename: "test.bin",
MaxAttempts: 1,
})
if err == nil {
t.Fatal("expected an error uploading to a dead replica")
}
if got := client.attempts(); got != 1 {
t.Fatalf("dial attempts = %d, want 1", got)
}
}
// TestUploadRetryStopsOnContextCancel verifies the retry loop honors context
// cancellation so DistributedOperation can abandon a slow replica once the
// write outcome is already decided by another replica.
func TestUploadRetryStopsOnContextCancel(t *testing.T) {
client := &deadReplicaClient{dialDelay: time.Hour} // would block ~forever per attempt
uploader := newUploader(client)
ctx, cancel := context.WithCancel(context.Background())
cancel() // already decided elsewhere
start := time.Now()
_, err := uploader.UploadData(ctx, []byte("hello"), &UploadOption{
UploadUrl: "http://dead-replica:8080/3,01",
Filename: "test.bin",
})
if err == nil {
t.Fatal("expected a context error")
}
if elapsed := time.Since(start); elapsed > 10*time.Second {
t.Fatalf("upload did not abort on cancellation: took %v", elapsed)
}
if got := client.attempts(); got > 1 {
t.Fatalf("dial attempts = %d, want at most 1 after cancellation", got)
}
}
+5 -10
View File
@@ -145,19 +145,14 @@ func makeSubscribeMetadataFunc(option *MetadataFollowOption, processEventFn Proc
pendingRefs = nil
}
// Freshness signals carry a timestamp but no entry: the idle heartbeat
// (nil EventNotification) and the MaxUnsyncedEvents marker (empty one).
// Route both to OnIdleHeartbeat, else the marker pins sync_offset to the
// stale processed watermark.
if len(resp.Events) == 0 && resp.TsNs > 0 && (resp.EventNotification == nil || filer_pb.IsEmpty(resp)) {
// Idle heartbeat: the source is caught up and has no new events, so
// it sends an empty response carrying the current time. Surface it as
// a freshness signal but leave option.StartTsNs untouched so a restart
// still resumes from the last real event.
if resp.EventNotification == nil && len(resp.Events) == 0 && resp.TsNs > 0 {
if option.OnIdleHeartbeat != nil {
option.OnIdleHeartbeat(resp.TsNs)
}
// Marker advances the resume cursor past the filtered range; the
// heartbeat leaves StartTsNs put so a restart cannot outrun a straggler.
if resp.EventNotification != nil {
option.StartTsNs = resp.TsNs
}
continue
}
-125
View File
@@ -1,125 +0,0 @@
package pb
import (
"context"
"io"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"google.golang.org/grpc"
)
// fakeSubscribeStream plays back a scripted sequence of responses, one per Recv.
type fakeSubscribeStream struct {
grpc.ClientStream
responses []*filer_pb.SubscribeMetadataResponse
delay time.Duration
idx int
}
func (s *fakeSubscribeStream) Recv() (*filer_pb.SubscribeMetadataResponse, error) {
if s.idx >= len(s.responses) {
return nil, io.EOF
}
if s.idx > 0 && s.delay > 0 {
// advance the wall clock so AddOffsetFunc's interval gate opens
time.Sleep(s.delay)
}
r := s.responses[s.idx]
s.idx++
return r, nil
}
// fakeFilerClient only needs SubscribeMetadata; the embedded nil interface
// covers the rest, which makeSubscribeMetadataFunc never calls.
type fakeFilerClient struct {
filer_pb.SeaweedFilerClient
stream *fakeSubscribeStream
}
func (c *fakeFilerClient) SubscribeMetadata(ctx context.Context, in *filer_pb.SubscribeMetadataRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.SubscribeMetadataResponse], error) {
return c.stream, nil
}
// TestFilerSyncOffsetStaysFreshOnFilteredMarker: on a read-only watched path with
// a busy source, the MaxUnsyncedEvents marker (empty EventNotification, fresh
// timestamp) must be treated as a freshness signal, not fed to the offset path
// where it would pin the gauge to the stale watermark. Wiring mirrors filer.sync.
func TestFilerSyncOffsetStaysFreshOnFilteredMarker(t *testing.T) {
const oldEventTs = int64(1_000_000_000) // t0: last real synced event (stale)
nowTs := time.Now().UnixNano() // current source time
markerTs := nowTs + int64(time.Second)
var watermark = oldEventTs // MetadataProcessor.processedTsWatermark
type gaugeWrite struct {
src string
ts int64
}
var timeline []gaugeWrite
var heartbeatCalls, markerToProcessFn int
// AddSyncJob drops empty events and does not advance the watermark; the real
// processor is checked in command.TestMetadataProcessorEmptyMarkerKeepsWatermarkStale.
realProcessFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
if filer_pb.IsEmpty(resp) {
markerToProcessFn++
return nil
}
watermark = resp.TsNs
return nil
}
// offsetFunc publishes the watermark to the gauge (filer_sync.go).
processEventFn := AddOffsetFunc(realProcessFn, 0, func(counter, lastTsNs int64) error {
timeline = append(timeline, gaugeWrite{"offset", watermark})
return nil
})
option := &MetadataFollowOption{
ClientName: "syncFrom_A_To_B",
StartTsNs: oldEventTs,
EventErrorType: DontLogError,
OnIdleHeartbeat: func(tsNs int64) {
heartbeatCalls++
timeline = append(timeline, gaugeWrite{"heartbeat", tsNs})
},
}
stream := &fakeSubscribeStream{
delay: 2 * time.Millisecond,
responses: []*filer_pb.SubscribeMetadataResponse{
// real create on the watched path, long ago -> watermark = t0
{Directory: "/watched", TsNs: oldEventTs, EventNotification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "file"},
}},
// genuine idle heartbeat
{TsNs: nowTs},
// MaxUnsyncedEvents marker: empty EventNotification, fresh timestamp
{TsNs: markerTs, EventNotification: &filer_pb.EventNotification{}},
},
}
if err := makeSubscribeMetadataFunc(option, processEventFn)(&fakeFilerClient{stream: stream}); err != nil {
t.Fatalf("follow: %v", err)
}
t.Logf("gauge timeline: %+v", timeline)
// the marker fires OnIdleHeartbeat instead of reaching processEventFn
if markerToProcessFn != 0 {
t.Errorf("empty marker must not reach processEventFn, got %d calls", markerToProcessFn)
}
if heartbeatCalls != 2 {
t.Errorf("expected OnIdleHeartbeat for both the heartbeat and the marker (2), got %d", heartbeatCalls)
}
if option.StartTsNs != markerTs {
t.Errorf("marker should advance StartTsNs to %d, got %d", markerTs, option.StartTsNs)
}
// gauge stays fresh: last write is the marker's timestamp, not the stale watermark
last := timeline[len(timeline)-1]
if last.src != "heartbeat" || last.ts != markerTs {
t.Fatalf("expected final gauge write fresh at %d, got %+v (spike is back if stale %d)", markerTs, last, oldEventTs)
}
}
@@ -227,9 +227,6 @@ func (gcs *gcsRemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocatio
metadata := toMetadata(entry.Extended)
wc := gcs.client.Bucket(loc.Bucket).Object(key).NewWriter(context.Background())
wc.Metadata = metadata
if entry.Attributes != nil && entry.Attributes.Mime != "" {
wc.ContentType = entry.Attributes.Mime
}
if _, err = io.Copy(wc, reader); err != nil {
return nil, fmt.Errorf("upload to gcs %s/%s%s: %v", loc.Name, loc.Bucket, loc.Path, err)
}
@@ -130,7 +130,8 @@ func TestS3WriteFilePassesMimeAsContentType(t *testing.T) {
Attributes: &filer_pb.FuseAttributes{Mime: "text/html"},
}
_, _ = client.WriteFile(loc, entry, bytes.NewReader([]byte("<html></html>")))
_, err := client.WriteFile(loc, entry, bytes.NewReader([]byte("<html></html>")))
require.NoError(t, err)
require.NotNil(t, rt.uploadReq, "uploader should have issued a PUT")
require.Equal(t, "text/html", rt.uploadContentType(), "Content-Type should match entry.Attributes.Mime")
@@ -147,7 +148,8 @@ func TestS3WriteFileOmitsContentTypeWhenMimeMissing(t *testing.T) {
Attributes: &filer_pb.FuseAttributes{},
}
_, _ = client.WriteFile(loc, entry, bytes.NewReader([]byte("data")))
_, err := client.WriteFile(loc, entry, bytes.NewReader([]byte("data")))
require.NoError(t, err)
require.NotNil(t, rt.uploadReq, "uploader should have issued a PUT")
// When entry.Attributes.Mime is empty we don't force a Content-Type so the
-3
View File
@@ -209,9 +209,6 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry, signatures
if tags != "" {
uploadInput.Tagging = aws.String(tags)
}
if entry.Attributes.Mime != "" {
uploadInput.ContentType = aws.String(entry.Attributes.Mime)
}
if len(entry.Attributes.Md5) > 0 {
uploadInput.ContentMD5 = aws.String(base64.StdEncoding.EncodeToString([]byte(entry.Attributes.Md5)))
}
@@ -1,19 +1,11 @@
package S3Sink
import (
"io"
"net/http"
"net/url"
"strings"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
awss3 "github.com/aws/aws-sdk-go/service/s3"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/stretchr/testify/require"
)
func TestBuildTaggingString_ShouldStripTagPrefix(t *testing.T) {
@@ -52,72 +44,6 @@ func TestBuildTaggingString_ShouldURLEncodeValues(t *testing.T) {
}
}
// capturePutRoundTripper records the s3manager.Uploader's PUT and replies 200
// so the SDK is satisfied without making a real network call.
type capturePutRoundTripper struct {
uploadReq *http.Request
}
func (c *capturePutRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
if req.Method == http.MethodPut {
c.uploadReq = req.Clone(req.Context())
}
if req.Body != nil {
_, _ = io.Copy(io.Discard, req.Body)
_ = req.Body.Close()
}
return &http.Response{
StatusCode: http.StatusOK,
Body: io.NopCloser(strings.NewReader("")),
Header: http.Header{"ETag": []string{"\"etag\""}},
Request: req,
}, nil
}
func newCapturingS3Sink(t *testing.T) (*S3Sink, *capturePutRoundTripper) {
t.Helper()
rt := &capturePutRoundTripper{}
sess, err := session.NewSession(&aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String("https://example.invalid"),
S3ForcePathStyle: aws.Bool(true),
Credentials: credentials.NewStaticCredentials("k", "s", ""),
HTTPClient: &http.Client{Transport: rt},
})
require.NoError(t, err)
s := &S3Sink{
conn: awss3.New(sess),
bucket: "bucket",
uploaderConcurrency: 1,
uploaderPartSizeMb: 5,
}
return s, rt
}
func TestS3SinkCreateEntryPassesMimeAsContentType(t *testing.T) {
sink, rt := newCapturingS3Sink(t)
entry := &filer_pb.Entry{
Attributes: &filer_pb.FuseAttributes{Mime: "text/html"},
Content: []byte("<html></html>"),
}
require.NoError(t, sink.CreateEntry("/dir/test.html", entry, nil))
require.NotNil(t, rt.uploadReq, "uploader should have issued a PUT")
require.Equal(t, "text/html", rt.uploadReq.Header.Get("Content-Type"))
}
func TestS3SinkCreateEntryOmitsContentTypeWhenMimeMissing(t *testing.T) {
sink, rt := newCapturingS3Sink(t)
entry := &filer_pb.Entry{
Attributes: &filer_pb.FuseAttributes{},
Content: []byte("data"),
}
require.NoError(t, sink.CreateEntry("/dir/test.bin", entry, nil))
require.NotNil(t, rt.uploadReq, "uploader should have issued a PUT")
require.Equal(t, "", rt.uploadReq.Header.Get("Content-Type"))
}
func TestBuildTaggingString_EmptyWhenNoTags(t *testing.T) {
extended := map[string][]byte{
"Content-Encoding": []byte("gzip"),
+3 -24
View File
@@ -1388,30 +1388,9 @@ func (iam *IdentityAccessManagement) authenticateRequestInternal(r *http.Request
identity, s3Err = iam.reqSignatureV4Verify(r)
amzAuthType = "SigV4"
case authTypeStreamingUnsigned:
// STREAMING-UNSIGNED-PAYLOAD-TRAILER only describes the body encoding; the
// request may still be SigV4-signed (header/presigned), JWT-bearer, or fully
// anonymous. Modern botocore adds a CRC32 trailer to plain PUTs, so an
// anonymous upload also lands here. Dispatch on whatever credential is present.
switch {
case isRequestSignatureV4(r) || isRequestPresignedSignatureV4(r):
glog.V(4).Infof("unsigned streaming upload, signed request")
identity, s3Err = iam.reqSignatureV4Verify(r)
amzAuthType = "SigV4"
case isRequestJWT(r):
glog.V(4).Infof("unsigned streaming upload, jwt request")
if iam.iamIntegration == nil {
return identity, s3err.ErrNotImplemented, reqAuthType
}
identity, s3Err = iam.authenticateJWTWithIAM(r)
amzAuthType = "Jwt"
default:
glog.V(4).Infof("unsigned streaming upload, anonymous request")
amzAuthType = "Anonymous"
if identity, found = iam.LookupAnonymous(); !found {
r.Header.Set(s3_constants.AmzAuthType, amzAuthType)
return identity, s3err.ErrAccessDenied, reqAuthType
}
}
glog.V(4).Infof("unsigned streaming upload")
identity, s3Err = iam.reqSignatureV4Verify(r)
amzAuthType = "SigV4"
case authTypeJWT:
glog.V(4).Infof("jwt auth type detected, iamIntegration != nil? %t", iam.iamIntegration != nil)
r.Header.Set(s3_constants.AmzAuthType, "Jwt")
@@ -1,60 +0,0 @@
package s3api
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestJWTStreamingUnsignedAuth covers the auth half of the JWT + unsigned-streaming
// gap. A bearer-token client whose SDK appends a CRC32 trailer sends
// x-amz-content-sha256: STREAMING-UNSIGNED-PAYLOAD-TRAILER with no SigV4 signature,
// so the request is classified authTypeStreamingUnsigned. It must authenticate via
// the JWT integration rather than silently falling back to the anonymous identity.
func TestJWTStreamingUnsignedAuth(t *testing.T) {
resetMemoryStore()
defer resetMemoryStore()
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, "memory")
iam.iamIntegration = &MockIAMIntegration{
authenticateJWTFunc: func(ctx context.Context, r *http.Request) (*IAMIdentity, s3err.ErrorCode) {
return &IAMIdentity{Name: "jwt-user", Account: &AccountAdmin}, s3err.ErrNone
},
}
iam.isAuthEnabled = true
r := httptest.NewRequest(http.MethodPut, "http://localhost:8333/somebucket/someobject", nil)
r.Header.Set("x-amz-content-sha256", "STREAMING-UNSIGNED-PAYLOAD-TRAILER")
r.Header.Set("x-amz-trailer", "x-amz-checksum-crc32")
r.Header.Set("Authorization", "Bearer test.jwt.token")
// Stays classified as unsigned-streaming so getRequestDataReader still decodes
// the chunked body...
assert.Equal(t, authTypeStreamingUnsigned, getRequestAuthType(r))
// ...but authentication resolves to the JWT identity, not anonymous.
identity, errCode := iam.AuthenticateRequest(r)
assert.Equal(t, s3err.ErrNone, errCode, "JWT unsigned-streaming PUT must authenticate via the IAM integration")
require.NotNil(t, identity)
assert.Equal(t, "jwt-user", identity.Name)
}
// TestJWTStreamingUnsignedChunkedReader covers the body-decode half of the gap.
// newChunkedReader must not try to verify a bearer token as a SigV4 seed signature
// for an unsigned-streaming upload. It used to compute the seed signature whenever
// any Authorization header was present, so a JWT request failed to decode at all.
func TestJWTStreamingUnsignedChunkedReader(t *testing.T) {
iam := setupIam()
req, err := NewRequestStreamingUnsignedPayloadTrailer(true)
require.NoError(t, err)
req.Header.Set("Authorization", "Bearer test.jwt.token")
runWithRequest(iam, req, t, strings.Repeat("a", 17408))
}
-51
View File
@@ -29,11 +29,6 @@ func signRawHTTPRequest(ctx context.Context, req *http.Request, accessKey, secre
}
func TestReproIssue7912(t *testing.T) {
// This test asserts behavior for a config with no anonymous identity; reset
// the shared in-memory store so a leaked anonymous identity from another test
// does not satisfy the unsigned-streaming auth path.
resetMemoryStore()
// Create a temporary s3.json
configContent := `{
"identities": [
@@ -203,52 +198,6 @@ func TestReproIssue7912(t *testing.T) {
})
}
// TestAnonymousStreamingUnsignedUpload is a regression test for issue #9725.
// Modern botocore/aiobotocore attaches a CRC32 trailer to plain PutObject calls,
// turning the payload into STREAMING-UNSIGNED-PAYLOAD-TRAILER. An anonymous
// (anon=True) upload then carries that header but no Authorization, which used to
// be routed straight to SigV4 verification and rejected as AccessDenied. It must
// instead fall back to the configured anonymous identity, just like a plain
// anonymous PUT does.
func TestAnonymousStreamingUnsignedUpload(t *testing.T) {
// This test loads an anonymous identity into the shared in-memory credential
// store; reset before and after so it neither inherits nor leaks state.
resetMemoryStore()
defer resetMemoryStore()
configContent := `{
"identities": [
{
"name": "anonymous",
"actions": ["Read", "Write", "List"]
}
]
}`
tmpFile, err := os.CreateTemp("", "s3-config-*.json")
require.NoError(t, err)
defer os.Remove(tmpFile.Name())
_, err = tmpFile.Write([]byte(configContent))
require.NoError(t, err)
require.NoError(t, tmpFile.Close())
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{Config: tmpFile.Name()}, nil, "memory")
require.True(t, iam.isEnabled(), "Auth should be enabled")
r := httptest.NewRequest(http.MethodPut, "http://localhost:8333/somebucket/someobject", nil)
r.Header.Set("x-amz-content-sha256", "STREAMING-UNSIGNED-PAYLOAD-TRAILER")
r.Header.Set("x-amz-trailer", "x-amz-checksum-crc32")
// No Authorization header: anonymous upload with a checksum trailer.
// The request must stay classified as unsigned-streaming so getRequestDataReader
// still decodes the chunked body; only the auth resolution falls back to anonymous.
assert.Equal(t, authTypeStreamingUnsigned, getRequestAuthType(r))
identity, errCode := iam.authRequest(r, s3_constants.ACTION_WRITE)
assert.Equal(t, s3err.ErrNone, errCode, "anonymous unsigned-streaming PUT should resolve to the anonymous identity")
require.NotNil(t, identity)
assert.Equal(t, s3_constants.AccountAnonymousId, identity.Name)
}
// TestExternalUrlSignatureVerification tests that S3 signature verification works
// correctly when s3.externalUrl is configured. It uses the real AWS SDK v2 signer
// to prove correctness against actual S3 clients behind a reverse proxy.
-4
View File
@@ -16,7 +16,6 @@ import (
// MockIAMIntegration is a mock implementation of IAM integration for testing
type MockIAMIntegration struct {
authenticateJWTFunc func(ctx context.Context, r *http.Request) (*IAMIdentity, s3err.ErrorCode)
authorizeFunc func(ctx context.Context, identity *IAMIdentity, action Action, bucket, object string, r *http.Request) s3err.ErrorCode
validateTrustPolicyFunc func(ctx context.Context, roleArn, principalArn string) error
authCalled bool
@@ -31,9 +30,6 @@ func (m *MockIAMIntegration) AuthorizeAction(ctx context.Context, identity *IAMI
}
func (m *MockIAMIntegration) AuthenticateJWT(ctx context.Context, r *http.Request) (*IAMIdentity, s3err.ErrorCode) {
if m.authenticateJWTFunc != nil {
return m.authenticateJWTFunc(ctx, r)
}
return nil, s3err.ErrNotImplemented
}
+2 -4
View File
@@ -79,6 +79,7 @@ func (iam *IdentityAccessManagement) newChunkedReader(req *http.Request) (io.Rea
glog.V(3).Infof("creating a new newSignV4ChunkedReader")
contentSha256Header := req.Header.Get("X-Amz-Content-Sha256")
authorizationHeader := req.Header.Get("Authorization")
var credential *Credential
var seedSignature, region, service string
@@ -95,10 +96,7 @@ func (iam *IdentityAccessManagement) newChunkedReader(req *http.Request) (io.Rea
}
case streamingUnsignedPayload:
glog.V(3).Infof("streaming unsigned payload")
// The seed signature is a SigV4 concept; recompute it only for SigV4-signed
// requests. JWT-bearer, presigned, and anonymous unsigned-streaming uploads
// carry no header seed, and verifyV4Signature would fail parsing them.
if isRequestSignatureV4(req) {
if authorizationHeader != "" {
// We do not need to pass the seed signature to the Reader as each chunk is not signed,
// but we do compute it to verify the caller has the correct permissions.
_, _, _, _, _, errCode = iam.calculateSeedSignature(req)
-4
View File
@@ -133,10 +133,6 @@ func (s *Server) finalizeCreateOnCommit(ctx context.Context, input createOnCommi
message: "Failed to apply statistics updates: " + err.Error(),
}
}
// Same spec-compliance fixup we apply on create-table; ensures
// v{N}.metadata.json files written through this create-on-commit path are
// also readable by strict Iceberg clients reading directly from S3.
metadataBytes = ensureMetadataSpecCompliance(metadataBytes)
newMetadata, err = table.ParseMetadataBytes(metadataBytes)
if err != nil {
return nil, &icebergRequestError{
-5
View File
@@ -264,11 +264,6 @@ func (s *Server) handleUpdateTable(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "BadRequestException", "Failed to apply statistics updates: "+err.Error())
return
}
// Same spec-compliance fixup we apply on create-table; ensures
// v{N}.metadata.json files written during commit are also readable by
// strict Iceberg clients reading directly from S3, and that the
// FullMetadata persisted in S3Tables stays consistent.
metadataBytes = ensureMetadataSpecCompliance(metadataBytes)
newMetadata, err = table.ParseMetadataBytes(metadataBytes)
if err != nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to parse committed metadata: "+err.Error())
-4
View File
@@ -161,10 +161,6 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to serialize metadata: "+err.Error())
return
}
// Backfill Iceberg spec-required fields that iceberg-go v0.5.0 omits when
// empty (e.g. current-snapshot-id), so the persisted v*.metadata.json is
// readable by strict clients (Java/Spark/Trino) that go directly to S3.
metadataBytes = ensureMetadataSpecCompliance(metadataBytes)
tableName := req.Name
metadataFileName := "v1.metadata.json" // Initial version is always 1
-132
View File
@@ -1,132 +0,0 @@
package iceberg
import (
"bytes"
"encoding/json"
)
// specRequiredEmptyOrder is iterated in this fixed order so the byte-level
// append in ensureMetadataSpecCompliance produces deterministic output.
var specRequiredEmptyOrder = []string{
"current-snapshot-id",
"snapshots",
"snapshot-log",
"metadata-log",
"refs",
}
// specRequiredEmptyDefaults holds the sentinel values Iceberg requires when
// the corresponding state is empty.
var specRequiredEmptyDefaults = map[string]json.RawMessage{
"current-snapshot-id": json.RawMessage("-1"),
"snapshots": json.RawMessage("[]"),
"snapshot-log": json.RawMessage("[]"),
"metadata-log": json.RawMessage("[]"),
"refs": json.RawMessage("{}"),
}
// isJSONNull reports whether raw encodes the JSON null literal, ignoring
// insignificant whitespace allowed by RFC 8259.
func isJSONNull(raw json.RawMessage) bool {
return bytes.Equal(bytes.TrimSpace(raw), []byte("null"))
}
// ensureMetadataSpecCompliance backfills Iceberg spec-required fields that
// iceberg-go v0.5.0 drops via `omitempty` on optional pointer/slice fields.
//
// For tables with no snapshots (e.g. freshly created tables), iceberg-go
// represents an empty state with nil/empty Go values, then strips those keys
// entirely from the JSON output. The Iceberg REST spec, however, requires
// these keys to be present even when empty — Java/Spark/Trino clients
// fail to parse responses missing them (most visibly with
// "Cannot parse missing long current-snapshot-id"). This helper rehydrates
// the missing keys without overwriting any present real values.
//
// Missing keys are spliced in at the byte level just before the closing
// brace so iceberg-go's struct-declared key order is preserved (a naive
// map remarshal would alphabetize every key in the document). The slower
// remarshal path is only used when an explicit JSON null needs replacing,
// which iceberg-go itself never emits.
//
// Returns the original bytes unchanged when parsing fails or no key needs
// to be added, to avoid corrupting an otherwise-valid payload.
func ensureMetadataSpecCompliance(raw []byte) []byte {
if len(raw) == 0 {
return raw
}
var obj map[string]json.RawMessage
if err := json.Unmarshal(raw, &obj); err != nil {
return raw
}
// A top-level JSON null leaves obj == nil with no error; writes would
// panic on the slow path. It is not a valid TableMetadata document, so
// pass it through unchanged.
if obj == nil {
return raw
}
// A field is "missing" for spec purposes if it's absent OR encoded as
// JSON null. Some writers emit `"current-snapshot-id": null` for empty
// state instead of omitting; strict clients reject both the same way.
var toAppend []string
hasExplicitNull := false
for _, key := range specRequiredEmptyOrder {
v, present := obj[key]
switch {
case !present:
toAppend = append(toAppend, key)
case isJSONNull(v):
hasExplicitNull = true
}
}
if len(toAppend) == 0 && !hasExplicitNull {
return raw
}
if !hasExplicitNull {
return appendMissingObjectKeys(raw, toAppend, len(obj) > 0)
}
// Slow path: replace explicit nulls. Rare in practice, so the
// alphabetical key reordering inherent to map remarshal is acceptable.
for key, v := range obj {
if _, required := specRequiredEmptyDefaults[key]; required && isJSONNull(v) {
obj[key] = specRequiredEmptyDefaults[key]
}
}
for _, key := range toAppend {
obj[key] = specRequiredEmptyDefaults[key]
}
fixed, err := json.Marshal(obj)
if err != nil {
return raw
}
return fixed
}
// appendMissingObjectKeys splices the given keys into raw just before the
// closing brace of the top-level JSON object. raw is assumed to be a valid
// JSON object (the caller has already unmarshalled it). hasMembers tells us
// whether the object had any existing members, so we know whether to emit a
// leading comma for the first appended key.
func appendMissingObjectKeys(raw []byte, keys []string, hasMembers bool) []byte {
closeIdx := bytes.LastIndexByte(raw, '}')
if closeIdx < 0 {
return raw
}
var buf bytes.Buffer
buf.Grow(len(raw) + len(keys)*48)
buf.Write(raw[:closeIdx])
for _, key := range keys {
if hasMembers {
buf.WriteByte(',')
}
buf.WriteByte('"')
buf.WriteString(key)
buf.WriteString(`":`)
buf.Write(specRequiredEmptyDefaults[key])
hasMembers = true
}
buf.Write(raw[closeIdx:])
return buf.Bytes()
}
@@ -1,162 +0,0 @@
package iceberg
import (
"encoding/json"
"testing"
)
func TestEnsureMetadataSpecCompliance_BackfillsMissingFields(t *testing.T) {
// Mirrors the real iceberg-go v0.5.0 output for a freshly created table:
// current-snapshot-id, snapshots, snapshot-log, metadata-log, refs all absent.
input := []byte(`{
"format-version": 2,
"table-uuid": "82e3eec4-3aee-414f-a444-94c03c641d20",
"location": "s3://s3table/default/t1",
"last-sequence-number": 0,
"last-updated-ms": 1779866785466,
"last-column-id": 2,
"current-schema-id": 0,
"default-spec-id": 0,
"last-partition-id": 999,
"default-sort-order-id": 0
}`)
out := ensureMetadataSpecCompliance(input)
var got map[string]json.RawMessage
if err := json.Unmarshal(out, &got); err != nil {
t.Fatalf("output is not valid JSON: %v", err)
}
if v, ok := got["current-snapshot-id"]; !ok || string(v) != "-1" {
t.Errorf("current-snapshot-id missing or wrong: present=%v value=%s", ok, string(v))
}
for _, key := range []string{"snapshots", "snapshot-log", "metadata-log"} {
v, ok := got[key]
if !ok || string(v) != "[]" {
t.Errorf("%s missing or wrong: present=%v value=%s", key, ok, string(v))
}
}
if v, ok := got["refs"]; !ok || string(v) != "{}" {
t.Errorf("refs missing or wrong: present=%v value=%s", ok, string(v))
}
}
func TestEnsureMetadataSpecCompliance_PreservesExistingFields(t *testing.T) {
// Real snapshot id and refs must not be overwritten by sentinels.
input := []byte(`{
"format-version": 2,
"current-snapshot-id": 9876543210,
"snapshots": [{"snapshot-id": 9876543210}],
"snapshot-log": [{"snapshot-id": 9876543210, "timestamp-ms": 1}],
"metadata-log": [{"metadata-file": "v1.metadata.json", "timestamp-ms": 1}],
"refs": {"main": {"snapshot-id": 9876543210, "type": "branch"}}
}`)
out := ensureMetadataSpecCompliance(input)
var got map[string]json.RawMessage
if err := json.Unmarshal(out, &got); err != nil {
t.Fatalf("output is not valid JSON: %v", err)
}
if string(got["current-snapshot-id"]) != "9876543210" {
t.Errorf("real snapshot id was overwritten: %s", string(got["current-snapshot-id"]))
}
if string(got["snapshots"]) == "[]" {
t.Errorf("non-empty snapshots was overwritten with empty array")
}
}
func TestEnsureMetadataSpecCompliance_ReplacesExplicitNullsWithSentinels(t *testing.T) {
// Some writers emit explicit JSON null for unset values instead of omitting
// the key. Strict Iceberg clients reject these the same way as missing keys.
input := []byte(`{
"format-version": 2,
"current-snapshot-id": null,
"snapshots": null,
"snapshot-log": null,
"metadata-log": null,
"refs": null
}`)
out := ensureMetadataSpecCompliance(input)
var got map[string]json.RawMessage
if err := json.Unmarshal(out, &got); err != nil {
t.Fatalf("output is not valid JSON: %v", err)
}
if string(got["current-snapshot-id"]) != "-1" {
t.Errorf("current-snapshot-id should be replaced with -1, got %s", string(got["current-snapshot-id"]))
}
for _, key := range []string{"snapshots", "snapshot-log", "metadata-log"} {
if string(got[key]) != "[]" {
t.Errorf("%s should be replaced with [], got %s", key, string(got[key]))
}
}
if string(got["refs"]) != "{}" {
t.Errorf("refs should be replaced with {}, got %s", string(got["refs"]))
}
}
func TestEnsureMetadataSpecCompliance_InvalidJSONReturnedUnchanged(t *testing.T) {
input := []byte(`{not valid json`)
out := ensureMetadataSpecCompliance(input)
if string(out) != string(input) {
t.Errorf("invalid JSON should be returned unchanged; got %s", string(out))
}
}
func TestEnsureMetadataSpecCompliance_EmptyInputReturnedUnchanged(t *testing.T) {
if out := ensureMetadataSpecCompliance(nil); out != nil {
t.Errorf("nil input should be returned unchanged, got %v", out)
}
if out := ensureMetadataSpecCompliance([]byte{}); len(out) != 0 {
t.Errorf("empty input should be returned unchanged, got %v", out)
}
// A top-level JSON null literal must not panic on the slow path.
if out := ensureMetadataSpecCompliance([]byte("null")); string(out) != "null" {
t.Errorf("top-level null should be returned unchanged, got %s", string(out))
}
}
// Original iceberg-go key ordering must survive the backfill: appended
// sentinels go at the end without disturbing prior fields. A map-based
// remarshal would have sorted everything alphabetically.
func TestEnsureMetadataSpecCompliance_PreservesOriginalKeyOrder(t *testing.T) {
// Compact JSON, keys in struct-declared order from iceberg-go.
input := []byte(`{"format-version":2,"table-uuid":"82e3eec4-3aee-414f-a444-94c03c641d20","location":"s3://x/t","last-sequence-number":0,"last-updated-ms":1,"last-column-id":2,"current-schema-id":0,"default-spec-id":0,"last-partition-id":999,"default-sort-order-id":0}`)
out := ensureMetadataSpecCompliance(input)
// Prior keys keep their order, sentinels appended at the end.
want := `{"format-version":2,"table-uuid":"82e3eec4-3aee-414f-a444-94c03c641d20","location":"s3://x/t","last-sequence-number":0,"last-updated-ms":1,"last-column-id":2,"current-schema-id":0,"default-spec-id":0,"last-partition-id":999,"default-sort-order-id":0,"current-snapshot-id":-1,"snapshots":[],"snapshot-log":[],"metadata-log":[],"refs":{}}`
if string(out) != want {
t.Errorf("unexpected output\n got: %s\nwant: %s", string(out), want)
}
// Sanity: still valid JSON.
var parsed map[string]json.RawMessage
if err := json.Unmarshal(out, &parsed); err != nil {
t.Fatalf("output is not valid JSON: %v", err)
}
}
// An empty object {} must round-trip to a valid JSON object containing
// only the spec sentinels (no leading comma).
func TestEnsureMetadataSpecCompliance_EmptyObjectBackfilled(t *testing.T) {
out := ensureMetadataSpecCompliance([]byte(`{}`))
want := `{"current-snapshot-id":-1,"snapshots":[],"snapshot-log":[],"metadata-log":[],"refs":{}}`
if string(out) != want {
t.Errorf("unexpected output\n got: %s\nwant: %s", string(out), want)
}
}
// When all fields are already present, the original bytes must be returned
// untouched (no whitespace normalization, no key reordering).
func TestEnsureMetadataSpecCompliance_AllPresentReturnsSameBytes(t *testing.T) {
input := []byte("{\n \"current-snapshot-id\": 1,\n \"snapshots\": [],\n \"snapshot-log\": [],\n \"metadata-log\": [],\n \"refs\": {}\n}")
out := ensureMetadataSpecCompliance(input)
if string(out) != string(input) {
t.Errorf("expected original bytes returned unchanged\n got: %q\nwant: %q", string(out), string(input))
}
}
-42
View File
@@ -93,29 +93,6 @@ type loadTableResultAlias struct {
Config iceberg.Properties `json:"config,omitempty"`
}
// MarshalJSON serializes LoadTableResult while backfilling spec-required
// metadata fields that iceberg-go v0.5.0 drops via `omitempty`. Without this,
// strict Iceberg REST clients (Java/Spark/Trino) reject otherwise valid
// responses for empty tables with errors like
// "Cannot parse missing long current-snapshot-id".
func (r LoadTableResult) MarshalJSON() ([]byte, error) {
metaBytes, err := json.Marshal(r.Metadata)
if err != nil {
return nil, err
}
metaBytes = ensureMetadataSpecCompliance(metaBytes)
return json.Marshal(struct {
MetadataLocation string `json:"metadata-location,omitempty"`
Metadata json.RawMessage `json:"metadata"`
Config iceberg.Properties `json:"config"`
}{
MetadataLocation: r.MetadataLocation,
Metadata: metaBytes,
Config: r.Config,
})
}
// UnmarshalJSON implements custom unmarshaling for LoadTableResult
// to properly parse table.Metadata using iceberg-go's parser.
func (r *LoadTableResult) UnmarshalJSON(data []byte) error {
@@ -157,25 +134,6 @@ type commitTableResponseAlias struct {
RawMetadata json.RawMessage `json:"metadata"`
}
// MarshalJSON mirrors LoadTableResult.MarshalJSON: it backfills spec-required
// fields that iceberg-go v0.5.0 omits, so commit responses also parse cleanly
// on strict Iceberg clients.
func (r CommitTableResponse) MarshalJSON() ([]byte, error) {
metaBytes, err := json.Marshal(r.Metadata)
if err != nil {
return nil, err
}
metaBytes = ensureMetadataSpecCompliance(metaBytes)
return json.Marshal(struct {
MetadataLocation string `json:"metadata-location"`
Metadata json.RawMessage `json:"metadata"`
}{
MetadataLocation: r.MetadataLocation,
Metadata: metaBytes,
})
}
// UnmarshalJSON implements custom unmarshaling for CommitTableResponse.
func (r *CommitTableResponse) UnmarshalJSON(data []byte) error {
var alias commitTableResponseAlias
+1 -1
View File
@@ -1059,7 +1059,7 @@ func (s3a *S3ApiServer) PutBucketOwnershipControls(w http.ResponseWriter, r *htt
return
}
if len(v.Rules) != 1 || v.Rules[0] == nil || v.Rules[0].ObjectOwnership == nil {
if len(v.Rules) != 1 {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
return
}
@@ -9,11 +9,9 @@ import (
"testing"
"time"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)
func newMiscTestServer(t *testing.T, bucket string) *S3ApiServer {
@@ -104,36 +102,6 @@ func TestPutBucketRequestPaymentRequesterRejected(t *testing.T) {
}
}
func TestPutBucketOwnershipControlsRejectsRuleWithoutObjectOwnership(t *testing.T) {
ownerID := AccountAdmin.Id
s3a := &S3ApiServer{
bucketRegistry: &BucketRegistry{
metadataCache: map[string]*BucketMetaData{
"b": {
Name: "b",
Owner: &s3.Owner{
ID: &ownerID,
},
},
},
notFound: map[string]struct{}{},
},
}
body := `<OwnershipControls><Rule></Rule></OwnershipControls>`
req := newBucketRequest(http.MethodPut, "b", "ownershipControls=", body)
req.Header.Set(s3_constants.AmzAccountId, AccountAdmin.Id)
rec := httptest.NewRecorder()
s3a.PutBucketOwnershipControls(rec, req)
if rec.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusBadRequest, rec.Body.String())
}
if !strings.Contains(rec.Body.String(), "InvalidRequest") {
t.Fatalf("body missing InvalidRequest: %s", rec.Body.String())
}
}
func TestGetBucketAccelerateConfiguration(t *testing.T) {
s3a := newMiscTestServer(t, "b")
req := newBucketRequest(http.MethodGet, "b", "accelerate=", "")
+1 -1
View File
@@ -2854,7 +2854,7 @@ func (e *EmbeddedIamApi) DoActions(w http.ResponseWriter, r *http.Request) {
// Handle implicit username for HTTP requests
switch r.Form.Get("Action") {
case "ListAccessKeys", "CreateAccessKey", "DeleteAccessKey", "UpdateAccessKey", "ListUserPolicies", "GetUser":
case "ListAccessKeys", "CreateAccessKey", "DeleteAccessKey", "UpdateAccessKey", "ListUserPolicies":
e.handleImplicitUsername(r, values)
case "CreateServiceAccount":
createdBy := s3_constants.GetIdentityNameFromContext(r)
-40
View File
@@ -339,46 +339,6 @@ func TestEmbeddedIamGetUser(t *testing.T) {
assert.Equal(t, "TestUser", *out.GetUserResult.User.UserName)
}
// TestEmbeddedIamGetUserImplicitUsername verifies GetUser without a UserName defaults
// to the user that signed the request rather than returning NoSuchEntity for an empty
// username, matching documented AWS IAM behavior.
func TestEmbeddedIamGetUserImplicitUsername(t *testing.T) {
const accessKey = UserAccessKeyPrefix + "TESTFAKEKEY000001"
api := NewEmbeddedIamApiForTest()
api.mockConfig = &iam_pb.S3ApiConfiguration{
Identities: []*iam_pb.Identity{
{
Name: "TestUser",
Credentials: []*iam_pb.Credential{
{AccessKey: accessKey, SecretKey: "testsecretfake"},
},
},
},
}
// handleImplicitUsername resolves the username via iam.LookupByAccessKey,
// so the iam state must know about the credential.
if err := api.iam.LoadS3ApiConfigurationFromBytes(mustMarshalJSON(api.mockConfig)); err != nil {
t.Fatalf("failed to load iam config: %v", err)
}
// GetUser request with NO UserName, but signed by accessKey.
form := url.Values{}
form.Set("Action", "GetUser")
req, _ := http.NewRequest(http.MethodPost, "/", strings.NewReader(form.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+accessKey+
"/20220420/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesig")
out := iamGetUserResponse{}
rr, err := executeEmbeddedIamRequest(api, req, &out)
require.NoError(t, err)
require.Equal(t, http.StatusOK, rr.Code, "body=%s", rr.Body.String())
require.NotNil(t, out.GetUserResult.User.UserName)
assert.Equal(t, "TestUser", *out.GetUserResult.User.UserName)
}
// TestEmbeddedIamCreatePolicy tests creating a policy via the embedded IAM API
func TestEmbeddedIamCreatePolicy(t *testing.T) {
api := NewEmbeddedIamApiForTest()
+7 -113
View File
@@ -32,41 +32,6 @@ const (
DirectiveReplace = "REPLACE"
)
// AWS default when REPLACE is requested without a Content-Type.
const defaultCopyContentType = "binary/octet-stream"
// System-metadata headers that REPLACE must rewrite on the destination.
// Content-Type lives on Attributes.Mime, tagging/storage-class have their
// own handling, so they're not in this list.
var copyReplaceSystemHeaders = []string{
"Cache-Control",
"Content-Encoding",
"Content-Disposition",
"Content-Language",
"Expires",
}
func resolveDestinationMime(reqHeader http.Header, sourceMime string, replaceMeta bool) string {
if replaceMeta {
if ct := reqHeader.Get("Content-Type"); ct != "" {
return ct
}
return defaultCopyContentType
}
return sourceMime
}
// Empty means default (COPY). Anything else outside {COPY, REPLACE} must be
// rejected, not silently downgraded.
func isValidDirective(value string) bool {
return value == "" || value == DirectiveCopy || value == DirectiveReplace
}
// hasPrefixFold reports whether s starts with prefix, ignoring case.
func hasPrefixFold(s, prefix string) bool {
return len(s) >= len(prefix) && strings.EqualFold(s[:len(prefix)], prefix)
}
func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
t := time.Now().UTC().Truncate(time.Millisecond)
@@ -115,15 +80,6 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
return
}
if !isValidDirective(r.Header.Get(s3_constants.AmzUserMetaDirective)) {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMetadataDirective)
return
}
if !isValidDirective(r.Header.Get(s3_constants.AmzObjectTaggingDirective)) {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTagDirective)
return
}
replaceMeta, replaceTagging := replaceDirective(r.Header)
// If source object is empty or bucket is empty, reply back invalid copy source.
@@ -205,15 +161,8 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
// A non-versioned in-place metadata replace routes to the owner as a
// serialized PATCH (off the distributed lock); versioned/suspended (which
// create a new version) and the no-owner bootstrap keep the lock.
//
// REPLACE can also change Content-Type, which lives on Attributes.Mime,
// not Extended. The routed PATCH only carries Extended keys, so when the
// Mime actually changes keep the lock and take the clone path below: it is
// still metadata-only (reuses the source chunks) but can set the Mime.
owner := s3a.objectWriteOwner(dstBucket, dstObject)
sourceMime := entry.GetAttributes().GetMime()
mimeChanged := resolveDestinationMime(r.Header, sourceMime, replaceMeta) != sourceMime
routeInPlace := owner != "" && dstVersioningState == "" && !mimeChanged
routeInPlace := owner != "" && dstVersioningState == ""
selfCopyBody := func() s3err.ErrorCode {
currentEntry, currentErr := s3a.resolveCopySourceEntry(srcBucket, srcObject, srcVersionId, srcVersioningState)
if currentErr != nil || currentEntry.IsDirectory {
@@ -239,7 +188,6 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
if updatedEntry.Attributes == nil {
updatedEntry.Attributes = &filer_pb.FuseAttributes{}
}
updatedEntry.Attributes.Mime = resolveDestinationMime(r.Header, currentEntry.GetAttributes().GetMime(), replaceMeta)
updatedEntry.Attributes.Mtime = t.Unix()
var finErr error
dstVersionId, etag, finErr = s3a.finalizeCopyDestination(dstBucket, dstObject, dstVersioningState, updatedEntry)
@@ -294,7 +242,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
FileSize: entry.Attributes.FileSize,
Mtime: t.Unix(),
Crtime: entry.Attributes.Crtime,
Mime: resolveDestinationMime(r.Header, entry.Attributes.Mime, replaceMeta),
Mime: entry.Attributes.Mime,
},
Extended: make(map[string][]byte),
}
@@ -340,10 +288,10 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
return
}
// mergeCopyMetadata drops stale managed keys before applying the new set,
// so REPLACE doesn't leak source values through the merge. Mirrors the
// self-copy path's routedMetadataReplace.
dstEntry.Extended = mergeCopyMetadata(dstEntry.Extended, processedMetadata)
// Apply processed metadata to destination entry
for k, v := range processedMetadata {
dstEntry.Extended[k] = v
}
// For zero-size files or files without chunks, handle inline content
// This includes encrypted inline files that need decryption/re-encryption
@@ -617,16 +565,7 @@ func isManagedCopyMetadataKey(key string) bool {
s3_constants.AmzTagCount:
return true
}
for _, h := range copyReplaceSystemHeaders {
if strings.EqualFold(key, h) {
return true
}
}
// Match X-Amz-Meta-* / X-Amz-Tagging case-insensitively so legacy
// non-canonical keys (written by non-S3 paths or older versions) are
// still recognized as managed.
return hasPrefixFold(key, s3_constants.AmzUserMetaPrefix) ||
hasPrefixFold(key, s3_constants.AmzObjectTagging)
return strings.HasPrefix(key, s3_constants.AmzUserMetaPrefix) || strings.HasPrefix(key, s3_constants.AmzObjectTagging)
}
func (s3a *S3ApiServer) resolveSuspendedCopySourceEntry(bucket, normalizedObject, operation string) (*filer_pb.Entry, error) {
@@ -1085,34 +1024,7 @@ func processMetadataBytes(reqHeader http.Header, existing map[string][]byte, rep
}
}
}
for _, h := range copyReplaceSystemHeaders {
if v := reqHeader.Get(h); v != "" {
metadata[h] = []byte(v)
}
}
} else {
// Two-pass copy keeps the result deterministic when both the
// canonical and a legacy-cased variant of the same header live on
// the source: canonical always wins, legacy only fills holes.
for _, h := range copyReplaceSystemHeaders {
if v, ok := existing[h]; ok {
metadata[h] = v
}
}
for k, v := range existing {
for _, h := range copyReplaceSystemHeaders {
if k == h {
continue
}
if !strings.EqualFold(k, h) {
continue
}
if _, present := metadata[h]; present {
continue
}
metadata[h] = v
}
}
// Copy existing metadata as-is
// Note: Metadata should already be normalized during storage (X-Amz-Meta-*),
// but we handle legacy non-canonical formats for backward compatibility
@@ -1154,29 +1066,11 @@ func processMetadataBytes(reqHeader http.Header, existing map[string][]byte, rep
}
}
} else {
// Two passes: canonical exact-prefix wins; legacy variants only
// fill in keys that no canonical entry already provided. Keeps
// the result deterministic when both variants coexist on the
// source.
prefixLen := len(s3_constants.AmzObjectTagging)
for k, v := range existing {
if strings.HasPrefix(k, s3_constants.AmzObjectTagging) {
metadata[k] = v
}
}
for k, v := range existing {
if strings.HasPrefix(k, s3_constants.AmzObjectTagging) {
continue
}
if !hasPrefixFold(k, s3_constants.AmzObjectTagging) {
continue
}
canonicalKey := s3_constants.AmzObjectTagging + k[prefixLen:]
if _, present := metadata[canonicalKey]; present {
continue
}
metadata[canonicalKey] = v
}
delete(metadata, s3_constants.AmzTagCount)
}
@@ -1,281 +0,0 @@
package s3api
import (
"net/http"
"testing"
)
func TestResolveDestinationMime(t *testing.T) {
cases := []struct {
name string
reqCT string
srcMime string
replaceMeta bool
want string
}{
{"COPY keeps source mime", "text/plain", "image/png", false, "image/png"},
{"COPY without request CT", "", "image/png", false, "image/png"},
{"COPY with empty source mime", "text/plain", "", false, ""},
{"REPLACE with request CT wins", "text/plain", "image/png", true, "text/plain"},
{"REPLACE without request CT uses default", "", "image/png", true, defaultCopyContentType},
{"REPLACE with request CT and empty source", "application/json", "", true, "application/json"},
{"REPLACE without request CT and empty source", "", "", true, defaultCopyContentType},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
h := http.Header{}
if c.reqCT != "" {
h.Set("Content-Type", c.reqCT)
}
got := resolveDestinationMime(h, c.srcMime, c.replaceMeta)
if got != c.want {
t.Errorf("got %q want %q", got, c.want)
}
})
}
}
func TestIsValidDirective(t *testing.T) {
cases := []struct {
value string
want bool
}{
{"", true},
{DirectiveCopy, true},
{DirectiveReplace, true},
{"copy", false},
{"replace", false},
{"FOO", false},
{"REPLACE ", false},
}
for _, c := range cases {
t.Run(c.value, func(t *testing.T) {
if got := isValidDirective(c.value); got != c.want {
t.Errorf("isValidDirective(%q) = %v, want %v", c.value, got, c.want)
}
})
}
}
func TestProcessMetadataBytes_ReplaceSystemHeaders(t *testing.T) {
existing := map[string][]byte{
"Cache-Control": []byte("max-age=60"),
"Content-Disposition": []byte(`attachment; filename="old.bin"`),
}
req := http.Header{}
req.Set("Cache-Control", "no-cache")
req.Set("Content-Encoding", "gzip")
out, err := processMetadataBytes(req, existing, true /* replaceMeta */, false /* replaceTagging */)
if err != nil {
t.Fatalf("processMetadataBytes returned error: %v", err)
}
if got := string(out["Cache-Control"]); got != "no-cache" {
t.Errorf("Cache-Control = %q, want %q", got, "no-cache")
}
if got := string(out["Content-Encoding"]); got != "gzip" {
t.Errorf("Content-Encoding = %q, want %q", got, "gzip")
}
if _, present := out["Content-Disposition"]; present {
t.Errorf("Content-Disposition should be dropped under REPLACE when not in request, got %q", string(out["Content-Disposition"]))
}
}
func TestIsManagedCopyMetadataKey_CoversSystemHeaders(t *testing.T) {
for _, h := range copyReplaceSystemHeaders {
if !isManagedCopyMetadataKey(h) {
t.Errorf("isManagedCopyMetadataKey(%q) = false, want true so mergeCopyMetadata drops stale source values", h)
}
}
}
func TestIsManagedCopyMetadataKey_CaseInsensitive(t *testing.T) {
cases := []string{
"cache-control",
"CACHE-CONTROL",
"content-encoding",
"x-amz-meta-owner",
"X-AMZ-META-OWNER",
"x-amz-tagging-env",
}
for _, k := range cases {
t.Run(k, func(t *testing.T) {
if !isManagedCopyMetadataKey(k) {
t.Errorf("isManagedCopyMetadataKey(%q) = false, want true; legacy non-canonical keys must still be recognized so mergeCopyMetadata clears them", k)
}
})
}
}
func TestProcessMetadataBytes_CopyAcceptsLowercaseSourceKeys(t *testing.T) {
existing := map[string][]byte{
"cache-control": []byte("max-age=60"),
"content-disposition": []byte(`attachment; filename="legacy.bin"`),
"CONTENT-ENCODING": []byte("gzip"),
"Content-language": []byte("en"),
}
req := http.Header{}
out, err := processMetadataBytes(req, existing, false, false)
if err != nil {
t.Fatalf("processMetadataBytes error: %v", err)
}
if got := string(out["Cache-Control"]); got != "max-age=60" {
t.Errorf("Cache-Control = %q, want %q (lowercase source must be promoted to canonical)", got, "max-age=60")
}
if got := string(out["Content-Disposition"]); got != `attachment; filename="legacy.bin"` {
t.Errorf("Content-Disposition = %q, want legacy source value promoted to canonical", got)
}
if got := string(out["Content-Encoding"]); got != "gzip" {
t.Errorf("Content-Encoding = %q, want %q (uppercase source must be promoted to canonical)", got, "gzip")
}
if got := string(out["Content-Language"]); got != "en" {
t.Errorf("Content-Language = %q, want %q (mixed-case source must be promoted to canonical)", got, "en")
}
}
func TestProcessMetadataBytes_CopyCanonicalSystemHeaderWinsOverLegacy(t *testing.T) {
// When both canonical and legacy-cased variants live on the source,
// COPY must deterministically prefer the canonical value. Run several
// times to exercise different Go map iteration orders.
for i := 0; i < 32; i++ {
existing := map[string][]byte{
"Cache-Control": []byte("canonical-wins"),
"cache-control": []byte("legacy-loses"),
}
out, err := processMetadataBytes(http.Header{}, existing, false, false)
if err != nil {
t.Fatalf("processMetadataBytes error: %v", err)
}
if got := string(out["Cache-Control"]); got != "canonical-wins" {
t.Fatalf("iter %d: Cache-Control = %q, want canonical-wins (canonical must beat legacy on COPY)", i, got)
}
}
}
func TestProcessMetadataBytes_CopyCanonicalTagWinsOverLegacy(t *testing.T) {
for i := 0; i < 32; i++ {
existing := map[string][]byte{
"X-Amz-Tagging-env": []byte("canonical-wins"),
"x-amz-tagging-env": []byte("legacy-loses"),
}
out, err := processMetadataBytes(http.Header{}, existing, false, false)
if err != nil {
t.Fatalf("processMetadataBytes error: %v", err)
}
if got := string(out["X-Amz-Tagging-env"]); got != "canonical-wins" {
t.Fatalf("iter %d: X-Amz-Tagging-env = %q, want canonical-wins (canonical tag must beat legacy on COPY)", i, got)
}
}
}
func TestProcessMetadataBytes_CopyAcceptsLowercaseTagKeys(t *testing.T) {
existing := map[string][]byte{
"x-amz-tagging-env": []byte("prod"),
"X-AMZ-TAGGING-team": []byte("infra"),
}
req := http.Header{}
out, err := processMetadataBytes(req, existing, false /* replaceMeta */, false /* replaceTagging */)
if err != nil {
t.Fatalf("processMetadataBytes error: %v", err)
}
if got := string(out["X-Amz-Tagging-env"]); got != "prod" {
t.Errorf("X-Amz-Tagging-env = %q, want %q (legacy lowercase tag must be promoted to canonical)", got, "prod")
}
if got := string(out["X-Amz-Tagging-team"]); got != "infra" {
t.Errorf("X-Amz-Tagging-team = %q, want %q (uppercase tag must be promoted to canonical)", got, "infra")
}
}
func TestMergeCopyMetadata_ReplaceDropsStaleSystemHeader(t *testing.T) {
// End-to-end caller flow: source-populated Extended + REPLACE request
// must drop stale managed values, keep non-managed ones.
existing := map[string][]byte{
"Content-Disposition": []byte(`attachment; filename="old.bin"`),
"Cache-Control": []byte("max-age=60"),
"X-Amz-Meta-Owner": []byte("old"),
"X-Amz-Meta-OnlyOnSource": []byte("should-drop"),
"X-Custom-Non-Managed": []byte("keep-me"),
"X-Amz-Server-Side-Encryption": []byte("aws:kms"),
}
req := http.Header{}
req.Set("Cache-Control", "no-cache")
req.Set("X-Amz-Meta-Owner", "new")
processed, err := processMetadataBytes(req, existing, true, false)
if err != nil {
t.Fatalf("processMetadataBytes error: %v", err)
}
merged := mergeCopyMetadata(existing, processed)
if _, leak := merged["Content-Disposition"]; leak {
t.Errorf("Content-Disposition leaked through REPLACE merge: %q", string(merged["Content-Disposition"]))
}
if got := string(merged["Cache-Control"]); got != "no-cache" {
t.Errorf("Cache-Control = %q, want %q", got, "no-cache")
}
if got := string(merged["X-Amz-Meta-Owner"]); got != "new" {
t.Errorf("X-Amz-Meta-Owner = %q, want %q", got, "new")
}
if _, leak := merged["X-Amz-Meta-OnlyOnSource"]; leak {
t.Errorf("stale X-Amz-Meta-OnlyOnSource must be dropped under REPLACE, still present: %q", string(merged["X-Amz-Meta-OnlyOnSource"]))
}
if got := string(merged["X-Custom-Non-Managed"]); got != "keep-me" {
t.Errorf("non-managed key %q must survive REPLACE merge, got %q", "X-Custom-Non-Managed", got)
}
}
func TestMergeCopyMetadata_ReplaceTaggingDropsStaleTags(t *testing.T) {
// REPLACE tagging directive must drop old object tags that the new
// request doesn't redeclare.
existing := map[string][]byte{
"X-Amz-Tagging-old": []byte("v1"),
"X-Amz-Tagging-env": []byte("dev"),
"X-Amz-Meta-Author": []byte("alice"),
}
req := http.Header{}
req.Set("X-Amz-Tagging", "env=prod")
processed, err := processMetadataBytes(req, existing, false /* replaceMeta */, true /* replaceTagging */)
if err != nil {
t.Fatalf("processMetadataBytes error: %v", err)
}
merged := mergeCopyMetadata(existing, processed)
if _, leak := merged["X-Amz-Tagging-old"]; leak {
t.Errorf("stale tag X-Amz-Tagging-old must be dropped, still present: %q", string(merged["X-Amz-Tagging-old"]))
}
if got := string(merged["X-Amz-Tagging-env"]); got != "prod" {
t.Errorf("X-Amz-Tagging-env = %q, want %q", got, "prod")
}
if got := string(merged["X-Amz-Meta-Author"]); got != "alice" {
t.Errorf("COPY-mode user metadata must survive tag-only REPLACE: got %q", got)
}
}
func TestProcessMetadataBytes_CopyInheritsSystemHeaders(t *testing.T) {
existing := map[string][]byte{
"Cache-Control": []byte("max-age=60"),
"Content-Disposition": []byte(`attachment; filename="src.bin"`),
"Content-Encoding": []byte("gzip"),
}
req := http.Header{}
req.Set("Cache-Control", "no-cache")
out, err := processMetadataBytes(req, existing, false /* replaceMeta */, false /* replaceTagging */)
if err != nil {
t.Fatalf("processMetadataBytes returned error: %v", err)
}
if got := string(out["Cache-Control"]); got != "max-age=60" {
t.Errorf("Cache-Control = %q, want %q (source value, request ignored under COPY)", got, "max-age=60")
}
if got := string(out["Content-Disposition"]); got != `attachment; filename="src.bin"` {
t.Errorf("Content-Disposition = %q, want source value", got)
}
if got := string(out["Content-Encoding"]); got != "gzip" {
t.Errorf("Content-Encoding = %q, want %q", got, "gzip")
}
}
+1 -2
View File
@@ -698,10 +698,9 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
// plus REST-style endpoints for AWS CLI
s3a.registerS3TablesRoutes(apiRouter)
// Health probes
// Readiness Probe
apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/status").HandlerFunc(s3a.StatusHandler)
apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/healthz").HandlerFunc(s3a.StatusHandler)
apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/readyz").HandlerFunc(s3a.StatusHandler)
// Object path pattern with (?s) flag to match newlines in object keys
const objectPath = "/{object:(?s).+}"
-15
View File
@@ -145,9 +145,6 @@ const (
ErrNoSuchBucketEncryptionConfiguration
ErrInvalidStorageClass
ErrInvalidMetadataDirective
ErrInvalidTagDirective
ErrInvalidAttributeName
// Object key length errors
@@ -617,18 +614,6 @@ var errorCodeResponse = map[ErrorCode]APIError{
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidMetadataDirective: {
Code: "InvalidArgument",
Description: "Unknown metadata directive.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidTagDirective: {
Code: "InvalidArgument",
Description: "Unknown tag directive.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidAttributeName: {
Code: "InvalidArgument",
Description: "Invalid attribute name specified",
+1 -15
View File
@@ -3,7 +3,6 @@ package dailyrun
import (
"context"
"errors"
"math/rand"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
@@ -53,7 +52,7 @@ func dispatchWithRetry(ctx context.Context, client LifecycleClient, m router.Mat
select {
case <-ctx.Done():
return s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED, ctx.Err()
case <-time.After(jitter(backoff)):
case <-time.After(backoff):
}
backoff *= 2
if backoff > transportRetryMax {
@@ -63,19 +62,6 @@ func dispatchWithRetry(ctx context.Context, client LifecycleClient, m router.Mat
return s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED, lastErr
}
// jitter returns a duration in the range [d/2, d) using equal jitter.
// Prevents thundering herds when many daily-run workers retry simultaneously.
func jitter(d time.Duration) time.Duration {
if d <= 0 {
return 0
}
half := d / 2
if half <= 0 {
return d
}
return half + time.Duration(rand.Int63n(int64(half)))
}
// buildDeleteRequest constructs the LifecycleDelete RPC payload for a
// router Match. Mirrors dispatcher.dispatchOne's request shape — both
// targets the same server-side handler and the proto encoding must
@@ -1,44 +0,0 @@
package dailyrun
import (
"testing"
"time"
)
func TestJitterBounds(t *testing.T) {
cases := []time.Duration{
200 * time.Millisecond,
1 * time.Second,
5 * time.Second,
}
for _, d := range cases {
for i := 0; i < 100; i++ {
j := jitter(d)
if j < d/2 {
t.Errorf("jitter(%v) = %v, below lower bound %v", d, j, d/2)
}
if j >= d {
t.Errorf("jitter(%v) = %v, at or above upper bound %v", d, j, d)
}
}
}
}
func TestJitterZeroAndNegative(t *testing.T) {
if j := jitter(0); j != 0 {
t.Errorf("jitter(0) = %v, want 0", j)
}
if j := jitter(-1 * time.Second); j != 0 {
t.Errorf("jitter(-1s) = %v, want 0", j)
}
}
func TestJitterTinyDuration(t *testing.T) {
// When d < 2, half == 0 and rand.Int63n(0) panics.
// We should return d unmodified in that case.
j := jitter(1 * time.Nanosecond)
if j != 1*time.Nanosecond {
t.Errorf("jitter(1ns) = %v, want 1ns", j)
}
}
+1 -17
View File
@@ -26,19 +26,7 @@ var (
"data": true,
}
// Patterns for valid metadata files.
//
// Note: Iceberg engines (Flink/Spark/Trino, plus different Iceberg versions)
// use a variety of manifest/snapshot naming schemes that the strict patterns
// below don't cover - e.g. Flink emits manifests like
// "{flink-job-id}-{checkpoint}-{operator-id}-{n}.avro". The Iceberg spec
// itself doesn't mandate a specific filename; engines just need a stable
// unique name. So in addition to the strict patterns we keep below for
// documentation, the catch-all entries at the bottom accept any
// *.avro and *.metadata.json filename composed of safe characters
// ([A-Za-z0-9._-]). The catch-all deliberately does NOT cover arbitrary
// *.json — only the *.metadata.json suffix — to avoid letting random
// JSON drop into the metadata/ directory.
// Patterns for valid metadata files
metadataFilePatterns = []*regexp.Regexp{
regexp.MustCompile(`^v\d+\.metadata\.json$`), // Table metadata: v1.metadata.json, v2.metadata.json
regexp.MustCompile(`^snap-\d+-\d+-` + uuidPattern + `\.avro$`), // Snapshot manifests: snap-123-1-uuid.avro
@@ -47,10 +35,6 @@ var (
regexp.MustCompile(`^version-hint\.text$`), // Version hint file
regexp.MustCompile(`^` + uuidPattern + `\.metadata\.json$`), // UUID-named metadata
regexp.MustCompile(`^[^/]+\.stats$`), // Trino/Iceberg stats files
// Catch-all for Iceberg writer-generated manifest / metadata files
// whose naming we can't anticipate across engines and versions.
regexp.MustCompile(`^[A-Za-z0-9._-]+\.avro$`),
regexp.MustCompile(`^[A-Za-z0-9._-]+\.metadata\.json$`),
}
// Patterns for valid data files
-102
View File
@@ -1,102 +0,0 @@
package s3tables
import "testing"
// TestIcebergLayoutValidator_AcceptsRealWorldManifestNames pins down the
// filename patterns we must accept across Iceberg engines. The original strict
// regex only covered Iceberg-internal naming (`{uuid}-m{n}.avro`,
// `snap-{n}-{n}-{uuid}.avro`) and rejected real manifests written by Flink and
// other writers, causing 403s during INSERT commits. The catch-all entries
// added alongside this test must keep these names valid.
func TestIcebergLayoutValidator_AcceptsRealWorldManifestNames(t *testing.T) {
cases := []struct {
name string
path string
}{
{
"flink-style manifest (job-id + checkpoint + operator-id + counter)",
"metadata/02678a59b3d6b460ba392851d77155fc-1-cbc357ccb763df2852fee8c4fc7d55f2-00001.avro",
},
{
"spark-style manifest with two uuids and -m suffix",
"metadata/00000000-0000-0000-0000-000000000000-m0.avro",
},
{
"snapshot manifest list with iceberg-internal naming",
"metadata/snap-7234891234567890123-1-82e3eec4-3aee-414f-a444-94c03c641d20.avro",
},
{
"versioned table metadata",
"metadata/v1.metadata.json",
},
{
"uuid-named metadata json (newer iceberg)",
"metadata/82e3eec4-3aee-414f-a444-94c03c641d20.metadata.json",
},
{
"flink-style metadata json with dashes and digits",
"metadata/02678a59-1-cbc357cc.metadata.json",
},
{
"version hint",
"metadata/version-hint.text",
},
{
"trino/iceberg stats file",
"metadata/table.stats",
},
}
v := NewIcebergLayoutValidator()
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if err := v.ValidateFilePath(tc.path); err != nil {
t.Errorf("expected %q to be accepted, got error: %v", tc.path, err)
}
})
}
}
// TestIcebergLayoutValidator_RejectsClearlyBadMetadataNames guards the
// catch-all from being too permissive — paths that look like attempts to
// escape the metadata layout or have forbidden file types must still fail.
func TestIcebergLayoutValidator_RejectsClearlyBadMetadataNames(t *testing.T) {
cases := []struct {
name string
path string
}{
{"random extension", "metadata/random-file.txt"},
{"executable masquerading as avro path", "metadata/evil.sh"},
{"subdirectory under metadata is not allowed", "metadata/sub/file.avro"},
{"top-level dir other than metadata or data", "garbage/file.avro"},
{"catch-all anchors must reject trailing extension", "metadata/file.avro.txt"},
}
v := NewIcebergLayoutValidator()
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if err := v.ValidateFilePath(tc.path); err == nil {
t.Errorf("expected %q to be rejected, but it passed validation", tc.path)
}
})
}
}
// TestIcebergLayoutValidator_AcceptsRealWorldDataFiles is a sanity check that
// the patterns most engines actually emit for data files still pass.
func TestIcebergLayoutValidator_AcceptsRealWorldDataFiles(t *testing.T) {
cases := []string{
"data/00000-0-ede83b82-08e1-40cd-af8a-6d83680a5194-00001.parquet",
"data/part-00000.parquet",
"data/some-file.orc",
"data/year=2026/month=05/00000-0-uuid.parquet",
}
v := NewIcebergLayoutValidator()
for _, p := range cases {
t.Run(p, func(t *testing.T) {
if err := v.ValidateFilePath(p); err != nil {
t.Errorf("expected %q to be accepted, got error: %v", p, err)
}
})
}
}
@@ -496,47 +496,3 @@ func TestMaybeSendIdleHeartbeat(t *testing.T) {
}
})
}
// TestFilteredEventsEmitMaxUnsyncedMarker pins the source-side shape the client
// keys off: after MaxUnsyncedEvents filtered events, eachEventNotificationFn
// emits a marker with a fresh timestamp and a non-nil but empty EventNotification.
// Consumed by TestFilerSyncOffsetStaysFreshOnFilteredMarker.
func TestFilteredEventsEmitMaxUnsyncedMarker(t *testing.T) {
fs := &FilerServer{
option: &FilerOption{Host: pb.ServerAddress("127.0.0.1:8888")},
filer: &filer.Filer{Signature: 123},
}
req := &filer_pb.SubscribeMetadataRequest{ClientName: "syncFrom_A_To_B", PathPrefix: "/watched/"}
stream := &collectingStream{}
eachEventFn := fs.eachEventNotificationFn(req, stream, "client")
base := time.Now().UnixNano()
var lastTsNs int64
// Feed MaxUnsyncedEvents+1 events on a NON-watched path so every one is filtered.
total := int(MaxUnsyncedEvents) + 1
for i := 0; i < total; i++ {
lastTsNs = base + int64(i)
err := eachEventFn("/other/dir", &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: fmt.Sprintf("file%d", i)},
}, lastTsNs)
if err != nil {
t.Fatalf("eachEventFn: %v", err)
}
}
if len(stream.messages) != 1 {
t.Fatalf("expected exactly 1 MaxUnsyncedEvents marker, got %d", len(stream.messages))
}
marker := stream.messages[0]
if !filer_pb.IsEmpty(marker) {
t.Errorf("marker should have empty EventNotification (IsEmpty), got %+v", marker.EventNotification)
}
if marker.EventNotification == nil {
t.Error("marker EventNotification should be non-nil but empty (the shape the client keys off)")
}
if marker.TsNs != lastTsNs {
t.Errorf("marker TsNs = %d, want fresh source ts %d", marker.TsNs, lastTsNs)
}
t.Logf("source emits marker{EventNotification:&{}, TsNs:%d} after %d filtered events", marker.TsNs, total)
}
+1 -3
View File
@@ -25,6 +25,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/filer/posixlock"
_ "github.com/seaweedfs/seaweedfs/weed/filer/arangodb"
_ "github.com/seaweedfs/seaweedfs/weed/filer/cassandra"
_ "github.com/seaweedfs/seaweedfs/weed/filer/cassandra2"
@@ -38,7 +39,6 @@ import (
_ "github.com/seaweedfs/seaweedfs/weed/filer/mongodb"
_ "github.com/seaweedfs/seaweedfs/weed/filer/mysql"
_ "github.com/seaweedfs/seaweedfs/weed/filer/mysql2"
"github.com/seaweedfs/seaweedfs/weed/filer/posixlock"
_ "github.com/seaweedfs/seaweedfs/weed/filer/postgres"
_ "github.com/seaweedfs/seaweedfs/weed/filer/postgres2"
_ "github.com/seaweedfs/seaweedfs/weed/filer/redis"
@@ -245,7 +245,6 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
handleStaticResources(defaultMux)
if !option.DisableHttp {
defaultMux.HandleFunc("/healthz", requestIDMiddleware(fs.filerHealthzHandler))
defaultMux.HandleFunc("/readyz", requestIDMiddleware(fs.filerHealthzHandler))
// TUS resumable upload protocol handler
if option.TusBasePath != "" {
// Normalize TusPath to always have a leading slash and no trailing slash
@@ -269,7 +268,6 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
if defaultMux != readonlyMux {
handleStaticResources(readonlyMux)
readonlyMux.HandleFunc("/healthz", requestIDMiddleware(fs.filerHealthzHandler))
readonlyMux.HandleFunc("/readyz", requestIDMiddleware(fs.filerHealthzHandler))
readonlyMux.HandleFunc("/", fs.filerGuard.WhiteList(requestIDMiddleware(fs.readonlyFilerHandler)))
}
-27
View File
@@ -169,8 +169,6 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
ms.guard = security.NewGuard(append(ms.option.WhiteList, whiteList...), signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec)
handleStaticResources2(r)
r.HandleFunc("/healthz", requestIDMiddleware(ms.healthzHandler)).Methods(http.MethodGet, http.MethodHead)
r.HandleFunc("/readyz", requestIDMiddleware(ms.readyzHandler)).Methods(http.MethodGet, http.MethodHead)
r.HandleFunc("/", ms.proxyToLeader(requestIDMiddleware(ms.uiStatusHandler)))
r.HandleFunc("/ui/index.html", requestIDMiddleware(ms.uiStatusHandler))
if !ms.option.DisableHttp {
@@ -210,31 +208,6 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
return ms
}
func (ms *MasterServer) healthzHandler(w http.ResponseWriter, r *http.Request) {
// Liveness: process is alive. Keep this fast and simple.
w.WriteHeader(http.StatusOK)
}
func (ms *MasterServer) readyzHandler(w http.ResponseWriter, r *http.Request) {
// Readiness: check we can serve traffic.
leader, err := ms.Topo.Leader()
if err != nil {
w.WriteHeader(http.StatusServiceUnavailable)
return
}
if ms.option.Master.Equals(leader) {
isLocked, err := ms.Topo.IsChildLocked()
if err != nil {
glog.Errorf("readyzHandler: %+v", err)
}
if isLocked {
w.WriteHeader(http.StatusLocked)
return
}
}
w.WriteHeader(http.StatusOK)
}
func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) {
var raftServerName string
+140
View File
@@ -0,0 +1,140 @@
package nfs
import (
"fmt"
"net"
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
)
type clientAuthorizer struct {
exact map[string]struct{}
cidrs map[string]*net.IPNet
enabled bool
}
func newClientAuthorizer(allowed []string) (*clientAuthorizer, error) {
authorizer := &clientAuthorizer{
exact: make(map[string]struct{}),
cidrs: make(map[string]*net.IPNet),
}
for _, raw := range allowed {
entry := strings.TrimSpace(raw)
if entry == "" {
continue
}
if strings.Contains(entry, "/") {
_, network, err := net.ParseCIDR(entry)
if err != nil {
return nil, fmt.Errorf("parse allowed NFS client %q: %w", entry, err)
}
authorizer.cidrs[entry] = network
authorizer.enabled = true
continue
}
if ip := normalizeClientIP(entry); ip != nil {
authorizer.exact[ip.String()] = struct{}{}
authorizer.enabled = true
continue
}
ips, err := net.LookupIP(entry)
if err != nil {
return nil, fmt.Errorf("resolve allowed NFS client %q: %w", entry, err)
}
if len(ips) == 0 {
return nil, fmt.Errorf("resolve allowed NFS client %q: no addresses", entry)
}
authorizer.exact[entry] = struct{}{}
for _, ip := range ips {
if ip == nil {
continue
}
authorizer.exact[ip.String()] = struct{}{}
}
authorizer.enabled = true
}
return authorizer, nil
}
func (a *clientAuthorizer) isAllowedConn(conn net.Conn) bool {
if conn == nil {
return true
}
return a.isAllowedAddr(conn.RemoteAddr())
}
func (a *clientAuthorizer) isAllowedAddr(addr net.Addr) bool {
if a == nil || !a.enabled {
return true
}
if addr == nil {
return false
}
host := remoteHost(addr.String())
if host == "" {
return false
}
if _, found := a.exact[host]; found {
return true
}
ip := normalizeClientIP(host)
if ip == nil {
return false
}
if _, found := a.exact[ip.String()]; found {
return true
}
for _, network := range a.cidrs {
if network.Contains(ip) {
return true
}
}
return false
}
func remoteHost(remote string) string {
host, _, err := net.SplitHostPort(strings.TrimSpace(remote))
if err == nil {
return host
}
host = strings.TrimSpace(remote)
if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") {
host = host[1 : len(host)-1]
}
return host
}
func normalizeClientIP(host string) net.IP {
host = strings.TrimSpace(host)
if zoneIndex := strings.LastIndex(host, "%"); zoneIndex >= 0 {
host = host[:zoneIndex]
}
return net.ParseIP(host)
}
type allowlistListener struct {
net.Listener
authorizer *clientAuthorizer
}
func (l *allowlistListener) Accept() (net.Conn, error) {
for {
conn, err := l.Listener.Accept()
if err != nil {
return nil, err
}
if l.authorizer == nil || l.authorizer.isAllowedConn(conn) {
return conn, nil
}
glog.V(0).Infof("reject unauthorized nfs client %s", conn.RemoteAddr())
_ = conn.Close()
}
}
+29
View File
@@ -0,0 +1,29 @@
package nfs
import (
"net"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestClientAuthorizerResolvesHostnameEntries(t *testing.T) {
ips, err := net.LookupIP("localhost")
require.NoError(t, err)
require.NotEmpty(t, ips)
authorizer, err := newClientAuthorizer([]string{"localhost"})
require.NoError(t, err)
matched := false
for _, ip := range ips {
if authorizer.isAllowedAddr(&net.TCPAddr{IP: ip, Port: 2049}) {
matched = true
break
}
}
assert.True(t, matched)
assert.False(t, authorizer.isAllowedAddr(&net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 2049}))
}
+251
View File
@@ -0,0 +1,251 @@
package nfs
import (
"context"
"encoding/binary"
"errors"
"fmt"
"hash/crc32"
"strings"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
)
const (
fileHandleVersion = 1
fileHandleLength = 28
)
var (
ErrInvalidHandle = errors.New("invalid nfs filehandle")
ErrHandleExportMismatch = errors.New("nfs filehandle export mismatch")
ErrStaleHandle = errors.New("stale nfs filehandle")
)
type FileHandleKind uint8
const (
FileHandleKindUnknown FileHandleKind = 0
FileHandleKindFile FileHandleKind = 1
FileHandleKindDirectory FileHandleKind = 2
)
type FileHandle struct {
Kind FileHandleKind
ExportID uint32
Inode uint64
Generation uint64
}
type filerResolverClient interface {
KvGet(ctx context.Context, in *filer_pb.KvGetRequest, opts ...grpc.CallOption) (*filer_pb.KvGetResponse, error)
LookupDirectoryEntry(ctx context.Context, in *filer_pb.LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error)
}
type Resolver struct {
exportRoot util.FullPath
exportID uint32
client filerResolverClient
}
type ResolvedHandle struct {
Handle FileHandle
Path util.FullPath
Entry *filer_pb.Entry
}
func NewFileHandle(exportID uint32, kind FileHandleKind, inode, generation uint64) FileHandle {
if generation == 0 {
generation = filer.InodeIndexInitialGeneration
}
return FileHandle{
Kind: kind,
ExportID: exportID,
Inode: inode,
Generation: generation,
}
}
func (h FileHandle) Encode() []byte {
buf := make([]byte, fileHandleLength)
buf[0] = fileHandleVersion
buf[1] = byte(h.Kind)
binary.BigEndian.PutUint32(buf[4:8], h.ExportID)
binary.BigEndian.PutUint64(buf[8:16], h.Inode)
binary.BigEndian.PutUint64(buf[16:24], h.Generation)
binary.BigEndian.PutUint32(buf[24:28], crc32.ChecksumIEEE(buf[:24]))
return buf
}
func DecodeFileHandle(raw []byte) (FileHandle, error) {
if len(raw) != fileHandleLength {
return FileHandle{}, fmt.Errorf("%w: unexpected length %d", ErrInvalidHandle, len(raw))
}
if raw[0] != fileHandleVersion {
return FileHandle{}, fmt.Errorf("%w: unsupported version %d", ErrInvalidHandle, raw[0])
}
wantChecksum := binary.BigEndian.Uint32(raw[24:28])
gotChecksum := crc32.ChecksumIEEE(raw[:24])
if wantChecksum != gotChecksum {
return FileHandle{}, fmt.Errorf("%w: checksum mismatch", ErrInvalidHandle)
}
handle := FileHandle{
Kind: FileHandleKind(raw[1]),
ExportID: binary.BigEndian.Uint32(raw[4:8]),
Inode: binary.BigEndian.Uint64(raw[8:16]),
Generation: binary.BigEndian.Uint64(raw[16:24]),
}
if handle.Generation == 0 {
return FileHandle{}, fmt.Errorf("%w: empty generation", ErrInvalidHandle)
}
return handle, nil
}
func NewResolver(exportRoot util.FullPath, client filerResolverClient) *Resolver {
root := normalizeExportRoot(exportRoot)
return &Resolver{
exportRoot: root,
exportID: exportIDForRoot(root),
client: client,
}
}
func (r *Resolver) ExportID() uint32 {
if r == nil {
return 0
}
return r.exportID
}
func (r *Resolver) ResolveHandle(ctx context.Context, raw []byte) (*ResolvedHandle, error) {
if r == nil || r.client == nil {
return nil, errors.New("nfs resolver is not configured")
}
handle, err := DecodeFileHandle(raw)
if err != nil {
return nil, err
}
if handle.ExportID != r.exportID {
return nil, ErrHandleExportMismatch
}
if handle.Inode == 0 {
return r.resolveSyntheticRoot(ctx, handle)
}
kvResp, err := r.client.KvGet(ctx, &filer_pb.KvGetRequest{Key: filer.InodeIndexKey(handle.Inode)})
if err != nil {
return nil, err
}
if kvResp.GetError() != "" {
return nil, errors.New(kvResp.GetError())
}
if len(kvResp.GetValue()) == 0 {
return nil, ErrStaleHandle
}
record, err := filer.DecodeInodeIndexRecord(kvResp.GetValue())
if err != nil {
return nil, err
}
if record.Generation != handle.Generation {
return nil, ErrStaleHandle
}
for _, path := range record.FullPaths() {
if !pathVisibleFromExport(path, r.exportRoot) {
continue
}
dir, name := path.DirAndName()
lookupResp, lookupErr := r.client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
})
if isLookupNotFound(lookupErr) || lookupResp == nil || lookupResp.Entry == nil {
continue
}
if lookupErr != nil {
return nil, lookupErr
}
if attrs := lookupResp.Entry.Attributes; attrs != nil && attrs.Inode != 0 && attrs.Inode != handle.Inode {
continue
}
if handle.Kind == FileHandleKindDirectory && !lookupResp.Entry.IsDirectory {
continue
}
if handle.Kind == FileHandleKindFile && lookupResp.Entry.IsDirectory {
continue
}
return &ResolvedHandle{
Handle: handle,
Path: path,
Entry: lookupResp.Entry,
}, nil
}
return nil, ErrStaleHandle
}
func (r *Resolver) resolveSyntheticRoot(ctx context.Context, handle FileHandle) (*ResolvedHandle, error) {
if handle.Kind != FileHandleKindDirectory || handle.Generation != filer.InodeIndexInitialGeneration {
return nil, ErrStaleHandle
}
dir, name := r.exportRoot.DirAndName()
lookupResp, err := r.client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
})
if isLookupNotFound(err) {
return &ResolvedHandle{
Handle: handle,
Path: r.exportRoot,
Entry: syntheticRootEntry(),
}, nil
}
if err != nil {
return nil, err
}
if lookupResp == nil || lookupResp.Entry == nil {
return &ResolvedHandle{
Handle: handle,
Path: r.exportRoot,
Entry: syntheticRootEntry(),
}, nil
}
return &ResolvedHandle{
Handle: handle,
Path: r.exportRoot,
Entry: lookupResp.Entry,
}, nil
}
func normalizeExportRoot(root util.FullPath) util.FullPath {
if normalized := util.NormalizePath(string(root)); normalized != "" {
return normalized
}
return "/"
}
func exportIDForRoot(root util.FullPath) uint32 {
return crc32.ChecksumIEEE([]byte(normalizeExportRoot(root)))
}
func pathVisibleFromExport(path, exportRoot util.FullPath) bool {
return path == exportRoot || path.IsUnder(exportRoot)
}
func isLookupNotFound(err error) bool {
if err == nil {
return false
}
return err == filer_pb.ErrNotFound || strings.Contains(err.Error(), filer_pb.ErrNotFound.Error())
}
+182
View File
@@ -0,0 +1,182 @@
package nfs
import (
"context"
"testing"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
)
type fakeResolverClient struct {
kv map[string][]byte
entries map[util.FullPath]*filer_pb.Entry
}
func (f *fakeResolverClient) KvGet(_ context.Context, in *filer_pb.KvGetRequest, _ ...grpc.CallOption) (*filer_pb.KvGetResponse, error) {
if value, found := f.kv[string(in.Key)]; found {
return &filer_pb.KvGetResponse{Value: value}, nil
}
return &filer_pb.KvGetResponse{}, nil
}
func (f *fakeResolverClient) LookupDirectoryEntry(_ context.Context, in *filer_pb.LookupDirectoryEntryRequest, _ ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
fullPath := util.NewFullPath(in.Directory, in.Name)
if entry, found := f.entries[fullPath]; found {
return &filer_pb.LookupDirectoryEntryResponse{Entry: entry}, nil
}
return nil, filer_pb.ErrNotFound
}
func TestFileHandleEncodeDecodeRoundTrip(t *testing.T) {
handle := NewFileHandle(1234, FileHandleKindDirectory, 5678, 9)
raw := handle.Encode()
decoded, err := DecodeFileHandle(raw)
require.NoError(t, err)
assert.Equal(t, handle, decoded)
raw[len(raw)-1] ^= 0xff
_, err = DecodeFileHandle(raw)
require.ErrorIs(t, err, ErrInvalidHandle)
}
func TestResolverUsesPathVisibleFromExportRoot(t *testing.T) {
client := &fakeResolverClient{
kv: make(map[string][]byte),
entries: make(map[util.FullPath]*filer_pb.Entry),
}
resolver := NewResolver("/exports", client)
record := &filer.InodeIndexRecord{
Generation: 7,
Paths: []string{"/a/other.txt", "/exports/demo/link.txt"},
}
value, err := record.Encode()
require.NoError(t, err)
client.kv[string(filer.InodeIndexKey(101))] = value
client.entries["/exports/demo/link.txt"] = &filer_pb.Entry{
Name: "link.txt",
Attributes: &filer_pb.FuseAttributes{
Inode: 101,
},
}
handle := NewFileHandle(resolver.ExportID(), FileHandleKindFile, 101, 7)
resolved, err := resolver.ResolveHandle(context.Background(), handle.Encode())
require.NoError(t, err)
assert.Equal(t, util.FullPath("/exports/demo/link.txt"), resolved.Path)
require.NotNil(t, resolved.Entry)
assert.Equal(t, uint64(101), resolved.Entry.Attributes.Inode)
}
func TestResolverRejectsGenerationMismatch(t *testing.T) {
client := &fakeResolverClient{
kv: make(map[string][]byte),
entries: make(map[util.FullPath]*filer_pb.Entry),
}
resolver := NewResolver("/", client)
record := &filer.InodeIndexRecord{
Generation: 3,
Paths: []string{"/data/file.txt"},
}
value, err := record.Encode()
require.NoError(t, err)
client.kv[string(filer.InodeIndexKey(44))] = value
client.entries["/data/file.txt"] = &filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
Inode: 44,
},
}
handle := NewFileHandle(resolver.ExportID(), FileHandleKindFile, 44, 4)
_, err = resolver.ResolveHandle(context.Background(), handle.Encode())
require.ErrorIs(t, err, ErrStaleHandle)
}
func TestResolverKeepsHandleValidAcrossRename(t *testing.T) {
client := &fakeResolverClient{
kv: make(map[string][]byte),
entries: make(map[util.FullPath]*filer_pb.Entry),
}
resolver := NewResolver("/exports", client)
record := &filer.InodeIndexRecord{
Generation: 5,
Paths: []string{"/exports/new-name.txt"},
}
value, err := record.Encode()
require.NoError(t, err)
client.kv[string(filer.InodeIndexKey(88))] = value
client.entries["/exports/new-name.txt"] = &filer_pb.Entry{
Name: "new-name.txt",
Attributes: &filer_pb.FuseAttributes{
Inode: 88,
},
}
handle := NewFileHandle(resolver.ExportID(), FileHandleKindFile, 88, 5)
resolved, err := resolver.ResolveHandle(context.Background(), handle.Encode())
require.NoError(t, err)
assert.Equal(t, util.FullPath("/exports/new-name.txt"), resolved.Path)
require.NotNil(t, resolved.Entry)
assert.Equal(t, uint64(88), resolved.Entry.Attributes.Inode)
}
func TestResolverRejectsHandleAfterDeleteRecreateWithNewInode(t *testing.T) {
client := &fakeResolverClient{
kv: make(map[string][]byte),
entries: make(map[util.FullPath]*filer_pb.Entry),
}
resolver := NewResolver("/exports", client)
client.entries["/exports/file.txt"] = &filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
Inode: 999,
},
}
record := &filer.InodeIndexRecord{
Generation: 4,
Paths: []string{"/exports/file.txt"},
}
value, err := record.Encode()
require.NoError(t, err)
client.kv[string(filer.InodeIndexKey(77))] = value
handle := NewFileHandle(resolver.ExportID(), FileHandleKindFile, 77, 4)
_, err = resolver.ResolveHandle(context.Background(), handle.Encode())
require.ErrorIs(t, err, ErrStaleHandle)
}
func TestResolverSupportsSyntheticRootHandle(t *testing.T) {
client := &fakeResolverClient{
kv: make(map[string][]byte),
entries: make(map[util.FullPath]*filer_pb.Entry),
}
resolver := NewResolver("/", client)
handle := NewFileHandle(resolver.ExportID(), FileHandleKindDirectory, 0, filer.InodeIndexInitialGeneration)
resolved, err := resolver.ResolveHandle(context.Background(), handle.Encode())
require.NoError(t, err)
assert.Equal(t, util.FullPath("/"), resolved.Path)
require.NotNil(t, resolved.Entry)
assert.True(t, resolved.Entry.IsDirectory)
}
func TestNewServerNormalizesExportRootAndExportID(t *testing.T) {
server, err := NewServer(&Option{
FilerRootPath: "/export/path/",
Port: 2049,
})
require.NoError(t, err)
assert.Equal(t, util.FullPath("/export/path"), server.exportRoot)
assert.Equal(t, exportIDForRoot("/export/path"), server.exportID)
}
File diff suppressed because it is too large Load Diff
+182
View File
@@ -0,0 +1,182 @@
package nfs
import (
"context"
"net"
"os"
"strings"
billy "github.com/go-git/go-billy/v5"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
gonfs "github.com/willscott/go-nfs"
)
type Handler struct {
server *Server
rootFS *seaweedFileSystem
}
var _ gonfs.Handler = (*Handler)(nil)
func (h *Handler) Mount(ctx context.Context, conn net.Conn, req gonfs.MountRequest) (gonfs.MountStatus, billy.Filesystem, []gonfs.AuthFlavor) {
if h.server.clientAuthorizer != nil && !h.server.clientAuthorizer.isAllowedConn(conn) {
return gonfs.MountStatusErrAcces, nil, []gonfs.AuthFlavor{gonfs.AuthFlavorNull}
}
fs, status := h.resolveMountFilesystem(ctx, string(req.Dirpath))
if status != gonfs.MountStatusOk {
return status, nil, []gonfs.AuthFlavor{gonfs.AuthFlavorNull}
}
return gonfs.MountStatusOk, fs, []gonfs.AuthFlavor{gonfs.AuthFlavorNull, gonfs.AuthFlavorUnix}
}
// resolveMountFilesystem resolves the MOUNT3 dirpath to a filesystem:
// exact match serves the export root; a path strictly under the export
// is mounted at that subdirectory (NoEnt/NotDir if missing or not a
// directory); anything else falls back to the export root with an INFO
// log. The UDP MOUNT path mirrors this in mount_udp.go.
func (h *Handler) resolveMountFilesystem(ctx context.Context, requestedPath string) (*seaweedFileSystem, gonfs.MountStatus) {
requested := normalizeExportRoot(util.FullPath(requestedPath))
// Exact match and outside-export both fall back to the export root.
// Only the second case logs; the first is the boring common path.
if requested == h.server.exportRoot || !requested.IsUnder(h.server.exportRoot) {
if requested != h.server.exportRoot {
glog.V(0).Infof("nfs mount: client requested %q (outside export %q); serving configured export", requestedPath, h.server.exportRoot)
}
return h.rootFS, h.lstatExportStatus(ctx)
}
entry, err := h.lookupSubexportEntry(ctx, requested)
switch {
case err != nil && isLookupNotFound(err):
return nil, gonfs.MountStatusErrNoEnt
case err != nil:
glog.Errorf("nfs mount: lookup %q under export %q failed: %v", requested, h.server.exportRoot, err)
return nil, gonfs.MountStatusErrServerFault
case entry == nil:
return nil, gonfs.MountStatusErrNoEnt
case !entry.IsDirectory:
return nil, gonfs.MountStatusErrNotDir
}
glog.V(1).Infof("nfs mount: client requested %q under export %q; mounting at subdirectory", requestedPath, h.server.exportRoot)
return newSeaweedFileSystem(h.server, requested, h.server.sharedReaderCache), gonfs.MountStatusOk
}
func (h *Handler) lstatExportStatus(ctx context.Context) gonfs.MountStatus {
if _, err := h.rootFS.fileInfoForVirtualPath(ctx, "/"); err != nil {
if os.IsNotExist(err) {
return gonfs.MountStatusErrNoEnt
}
return gonfs.MountStatusErrServerFault
}
return gonfs.MountStatusOk
}
func (h *Handler) lookupSubexportEntry(ctx context.Context, p util.FullPath) (*filer_pb.Entry, error) {
var entry *filer_pb.Entry
err := h.server.withInternalClient(false, func(client nfsFilerClient) error {
dir, name := p.DirAndName()
resp, lerr := client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
})
if lerr != nil {
return lerr
}
if resp != nil {
entry = resp.Entry
}
return nil
})
return entry, err
}
func (h *Handler) Change(filesystem billy.Filesystem) billy.Change {
if h.server != nil && h.server.option != nil && h.server.option.ReadOnly {
return nil
}
if changer, ok := filesystem.(billy.Change); ok {
return changer
}
return nil
}
func (h *Handler) FSStat(ctx context.Context, _ billy.Filesystem, stat *gonfs.FSStat) error {
return h.server.withInternalClient(false, func(client nfsFilerClient) error {
resp, err := client.Statistics(ctx, &filer_pb.StatisticsRequest{})
if err != nil {
return err
}
if resp == nil {
return nil
}
stat.TotalSize = resp.TotalSize
if resp.TotalSize >= resp.UsedSize {
stat.FreeSize = resp.TotalSize - resp.UsedSize
stat.AvailableSize = resp.TotalSize - resp.UsedSize
}
stat.TotalFiles = resp.FileCount
return nil
})
}
func (h *Handler) ToHandle(filesystem billy.Filesystem, path []string) []byte {
fs, ok := filesystem.(*seaweedFileSystem)
if !ok {
fs = h.rootFS
}
info, err := fs.fileInfoForVirtualPath(context.Background(), fs.Join(path...))
if err != nil {
return nil
}
inode := info.entry.GetAttributes().GetInode()
if inode == 0 && info.actualPath == h.server.exportRoot && info.entry.IsDirectory {
return NewFileHandle(h.server.exportID, FileHandleKindDirectory, 0, filer.InodeIndexInitialGeneration).Encode()
}
return NewFileHandle(h.server.exportID, fileHandleKindForEntry(info.entry), inode, info.generation).Encode()
}
func (h *Handler) FromHandle(raw []byte) (billy.Filesystem, []string, error) {
var resolved *ResolvedHandle
err := h.server.withInternalClient(false, func(client nfsFilerClient) error {
var resolveErr error
resolved, resolveErr = NewResolver(h.server.exportRoot, client).ResolveHandle(context.Background(), raw)
return resolveErr
})
if err != nil {
return nil, nil, err
}
if resolved.Path == h.server.exportRoot {
return h.rootFS, nil, nil
}
if !pathVisibleFromExport(resolved.Path, h.server.exportRoot) {
return nil, nil, ErrHandleExportMismatch
}
relativePath := string(resolved.Path)
if h.server.exportRoot != "/" {
relativePath = strings.TrimPrefix(relativePath, string(h.server.exportRoot))
}
return h.rootFS, util.NormalizePath(relativePath).Split(), nil
}
func (h *Handler) InvalidateHandle(billy.Filesystem, []byte) error {
return nil
}
func (h *Handler) HandleLimit() int {
return h.server.handleLimit
}
func fileHandleKindForEntry(entry *filer_pb.Entry) FileHandleKind {
if entry != nil && entry.IsDirectory {
return FileHandleKindDirectory
}
return FileHandleKindFile
}
+880
View File
@@ -0,0 +1,880 @@
package nfs
import (
"bytes"
"context"
"crypto/md5"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"math/rand"
"mime/multipart"
"net"
"net/http"
"net/http/httptest"
"path"
"strconv"
"strings"
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
gonfs "github.com/willscott/go-nfs"
nfsclient "github.com/willscott/go-nfs-client/nfs"
"github.com/willscott/go-nfs-client/nfs/rpc"
"github.com/willscott/go-nfs-client/nfs/xdr"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
type fakeVolumeBlob struct {
data []byte
contentEncoding string
}
type fakeVolumeServer struct {
mu sync.Mutex
blobs map[string]fakeVolumeBlob
server *httptest.Server
}
type fakeVolumeControlPlane struct {
filer_pb.UnimplementedSeaweedFilerServer
mu sync.Mutex
host string
nextID int
assigns []*filer_pb.AssignVolumeRequest
lookups []*filer_pb.LookupVolumeRequest
}
var initIntegrationHTTPClient sync.Once
const nfsProc3Link = 15
func newFakeVolumeServer(t *testing.T) *fakeVolumeServer {
t.Helper()
fake := &fakeVolumeServer{
blobs: make(map[string]fakeVolumeBlob),
}
fake.server = httptest.NewServer(http.HandlerFunc(fake.serveHTTP))
t.Cleanup(fake.server.Close)
return fake
}
func (f *fakeVolumeServer) host() string {
return strings.TrimPrefix(f.server.URL, "http://")
}
func (f *fakeVolumeServer) serveHTTP(w http.ResponseWriter, r *http.Request) {
fileID := strings.TrimPrefix(r.URL.Path, "/")
if fileID == "" {
http.NotFound(w, r)
return
}
switch r.Method {
case http.MethodPost:
part, err := firstMultipartFile(r)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
defer part.Close()
data, err := io.ReadAll(part)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
contentEncoding := part.Header.Get("Content-Encoding")
sum := md5.Sum(data)
f.mu.Lock()
f.blobs[fileID] = fakeVolumeBlob{
data: bytes.Clone(data),
contentEncoding: contentEncoding,
}
f.mu.Unlock()
w.Header().Set("Content-MD5", base64.StdEncoding.EncodeToString(sum[:]))
w.Header().Set("ETag", `"`+base64.StdEncoding.EncodeToString(sum[:])+`"`)
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"name": path.Base(fileID),
"size": len(data),
})
case http.MethodGet:
f.mu.Lock()
blob, found := f.blobs[fileID]
f.mu.Unlock()
if !found {
http.NotFound(w, r)
return
}
if blob.contentEncoding != "" {
w.Header().Set("Content-Encoding", blob.contentEncoding)
}
http.ServeContent(w, r, fileID, time.Unix(0, 0), bytes.NewReader(blob.data))
default:
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
}
}
func firstMultipartFile(r *http.Request) (*multipart.Part, error) {
reader, err := r.MultipartReader()
if err != nil {
return nil, err
}
for {
part, err := reader.NextPart()
if err == io.EOF {
return nil, io.ErrUnexpectedEOF
}
if err != nil {
return nil, err
}
if part.FormName() == "file" {
return part, nil
}
part.Close()
}
}
func (f *fakeVolumeControlPlane) AssignVolume(_ context.Context, req *filer_pb.AssignVolumeRequest) (*filer_pb.AssignVolumeResponse, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.assigns = append(f.assigns, req)
f.nextID++
fileID := fmt.Sprintf("7,%08x", f.nextID)
return &filer_pb.AssignVolumeResponse{
FileId: fileID,
Count: 1,
Location: &filer_pb.Location{
Url: f.host,
},
}, nil
}
func (f *fakeVolumeControlPlane) LookupVolume(_ context.Context, req *filer_pb.LookupVolumeRequest) (*filer_pb.LookupVolumeResponse, error) {
f.mu.Lock()
f.lookups = append(f.lookups, req)
f.mu.Unlock()
locations := make(map[string]*filer_pb.Locations, len(req.GetVolumeIds()))
for _, volumeID := range req.GetVolumeIds() {
locations[volumeID] = &filer_pb.Locations{
Locations: []*filer_pb.Location{
{Url: f.host},
},
}
}
return &filer_pb.LookupVolumeResponse{LocationsMap: locations}, nil
}
func startFakeVolumeControlPlane(t *testing.T, controlPlane *fakeVolumeControlPlane) string {
t.Helper()
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
grpcServer := grpc.NewServer()
filer_pb.RegisterSeaweedFilerServer(grpcServer, controlPlane)
done := make(chan error, 1)
go func() {
done <- grpcServer.Serve(listener)
}()
t.Cleanup(func() {
grpcServer.Stop()
_ = listener.Close()
select {
case err := <-done:
if err != nil && !isClosedNetworkErr(err) {
t.Errorf("fake control plane exited with error: %v", err)
}
case <-time.After(time.Second):
t.Errorf("timed out waiting for fake control plane shutdown")
}
})
return listener.Addr().String()
}
func mountTestTarget(t *testing.T, server *Server) (*nfsclient.Target, func()) {
t.Helper()
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
handler, err := server.newHandler()
require.NoError(t, err)
done := make(chan error, 1)
go func() {
done <- gonfs.Serve(listener, handler)
}()
var client *rpc.Client
for attempt := 0; attempt < 10; attempt++ {
client, err = rpc.DialTCP(listener.Addr().Network(), listener.Addr().String(), false)
if err == nil {
break
}
if attempt == 9 {
require.NoError(t, err)
}
time.Sleep(10 * time.Millisecond)
}
require.NoError(t, err)
mounter := &nfsclient.Mount{Client: client}
target, err := mounter.Mount(string(server.exportRoot), rpc.AuthNull)
require.NoError(t, err)
cleanup := func() {
_ = mounter.Unmount()
client.Close()
_ = listener.Close()
select {
case err := <-done:
if err != nil && !isClosedNetworkErr(err) {
t.Errorf("nfs server exited with error: %v", err)
}
case <-time.After(time.Second):
t.Errorf("timed out waiting for nfs server shutdown")
}
}
return target, cleanup
}
func isClosedNetworkErr(err error) bool {
if err == nil {
return false
}
if strings.Contains(err.Error(), "use of closed network connection") {
return true
}
return strings.Contains(err.Error(), "listener closed")
}
func nfsLink(target *nfsclient.Target, sourceHandle []byte, linkPath string) error {
parentDir, linkName := path.Split(path.Clean(linkPath))
if linkName == "" {
return fmt.Errorf("invalid hard link path %q", linkPath)
}
if parentDir == "" {
parentDir = "/"
}
_, parentHandle, err := target.Lookup(parentDir)
if err != nil {
return err
}
// Field layout matches the go-nfs server's onLink handler
// (vendor: github.com/willscott/go-nfs/nfs_onlink.go), which reads
// DirOpArg + SetFileAttributes + opaque target handle. That wire
// order differs from RFC 1813 §3.3.15 LINK3args {nfs_fh3 file;
// diropargs3 link;} — the go-nfs library is not strictly compliant
// here, and we mirror its layout so the integration test exercises
// the same parser the server uses. Do not reorder fields to match
// the RFC: the test would then fail against a correctly-functioning
// server.
type LinkArgs struct {
rpc.Header
Link nfsclient.Diropargs3
Sattr nfsclient.Sattr3
Target []byte
}
res, err := target.Call(&LinkArgs{
Header: rpc.Header{
Rpcvers: 2,
Prog: nfsclient.Nfs3Prog,
Vers: nfsclient.Nfs3Vers,
Proc: nfsProc3Link,
Cred: rpc.AuthNull,
Verf: rpc.AuthNull,
},
Link: nfsclient.Diropargs3{
FH: parentHandle,
Filename: linkName,
},
Target: sourceHandle,
})
if err != nil {
return err
}
status, err := xdr.ReadUint32(res)
if err != nil {
return err
}
return nfsclient.NFS3Error(status)
}
func TestSeaweedNFSAcceptsAnyMountPathOverRPC(t *testing.T) {
const exportRoot = "/buckets/data"
client := &fakeNFSFilerClient{
entries: map[util.FullPath]*filer_pb.Entry{
"/buckets": testEntry("buckets", true, 100, uint32(0755), nil),
"/buckets/data": testEntry("data", true, 101, uint32(0755), nil),
},
kv: map[string][]byte{
string(filer.InodeIndexKey(100)): testIndexRecord(t, 100, 1, "/buckets"),
string(filer.InodeIndexKey(101)): testIndexRecord(t, 101, 1, "/buckets/data"),
},
}
server := newTestServer(t, exportRoot, client)
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
handler, err := server.newHandler()
require.NoError(t, err)
serveDone := make(chan error, 1)
go func() {
serveDone <- gonfs.Serve(listener, handler)
}()
t.Cleanup(func() {
_ = listener.Close()
select {
case err := <-serveDone:
if err != nil && !isClosedNetworkErr(err) {
t.Errorf("nfs server exited with error: %v", err)
}
case <-time.After(time.Second):
t.Errorf("timed out waiting for nfs server shutdown")
}
})
dirpaths := []string{
"/",
"/buckets",
"/buckets/other",
"/wrong/path",
exportRoot,
exportRoot + "/",
}
for _, dirpath := range dirpaths {
t.Run(dirpath, func(t *testing.T) {
var rpcClient *rpc.Client
var dialErr error
for attempt := 0; attempt < 10; attempt++ {
rpcClient, dialErr = rpc.DialTCP(listener.Addr().Network(), listener.Addr().String(), false)
if dialErr == nil {
break
}
time.Sleep(10 * time.Millisecond)
}
require.NoError(t, dialErr)
defer rpcClient.Close()
mounter := &nfsclient.Mount{Client: rpcClient}
target, err := mounter.Mount(dirpath, rpc.AuthNull)
require.NoErrorf(t, err, "Mount(%q)", dirpath)
defer target.Close()
entries, err := target.ReadDirPlus("/")
require.NoError(t, err)
assert.Empty(t, entries, "Mount(%q) should land at the empty export root", dirpath)
})
}
}
func TestSeaweedNFSSubexportMountOverRPC(t *testing.T) {
const exportRoot = "/buckets"
client := &fakeNFSFilerClient{
entries: map[util.FullPath]*filer_pb.Entry{
"/buckets": testEntry("buckets", true, 100, uint32(0755), nil),
"/buckets/data": testEntry("data", true, 101, uint32(0755), nil),
"/buckets/data/inner": testEntry("inner", false, 104, uint32(0644), []byte("payload")),
"/buckets/other": testEntry("other", true, 105, uint32(0755), nil),
"/buckets/file.txt": testEntry("file.txt", false, 103, uint32(0644), []byte("hi")),
},
kv: map[string][]byte{
string(filer.InodeIndexKey(100)): testIndexRecord(t, 100, 1, "/buckets"),
string(filer.InodeIndexKey(101)): testIndexRecord(t, 101, 1, "/buckets/data"),
string(filer.InodeIndexKey(103)): testIndexRecord(t, 103, 1, "/buckets/file.txt"),
string(filer.InodeIndexKey(104)): testIndexRecord(t, 104, 1, "/buckets/data/inner"),
string(filer.InodeIndexKey(105)): testIndexRecord(t, 105, 1, "/buckets/other"),
},
}
server := newTestServer(t, exportRoot, client)
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
handler, err := server.newHandler()
require.NoError(t, err)
serveDone := make(chan error, 1)
go func() {
serveDone <- gonfs.Serve(listener, handler)
}()
t.Cleanup(func() {
_ = listener.Close()
select {
case err := <-serveDone:
if err != nil && !isClosedNetworkErr(err) {
t.Errorf("nfs server exited with error: %v", err)
}
case <-time.After(time.Second):
t.Errorf("timed out waiting for nfs server shutdown")
}
})
dial := func(t *testing.T) *rpc.Client {
t.Helper()
var rpcClient *rpc.Client
var dialErr error
for attempt := 0; attempt < 10; attempt++ {
rpcClient, dialErr = rpc.DialTCP(listener.Addr().Network(), listener.Addr().String(), false)
if dialErr == nil {
break
}
time.Sleep(10 * time.Millisecond)
}
require.NoError(t, dialErr)
t.Cleanup(func() { rpcClient.Close() })
return rpcClient
}
t.Run("mounts_under_export_at_subdirectory", func(t *testing.T) {
mounter := &nfsclient.Mount{Client: dial(t)}
target, err := mounter.Mount("/buckets/data", rpc.AuthNull)
require.NoError(t, err)
defer target.Close()
entries, err := target.ReadDirPlus("/")
require.NoError(t, err)
require.Len(t, entries, 1)
assert.Equal(t, "inner", entries[0].Name())
readFile, err := target.Open("/inner")
require.NoError(t, err)
defer readFile.Close()
data, err := io.ReadAll(readFile)
require.NoError(t, err)
assert.Equal(t, []byte("payload"), data)
})
t.Run("missing_entry_under_export_rejects", func(t *testing.T) {
mounter := &nfsclient.Mount{Client: dial(t)}
_, err := mounter.Mount("/buckets/missing", rpc.AuthNull)
require.Error(t, err)
})
t.Run("regular_file_under_export_rejects", func(t *testing.T) {
mounter := &nfsclient.Mount{Client: dial(t)}
_, err := mounter.Mount("/buckets/file.txt", rpc.AuthNull)
require.Error(t, err)
})
}
func TestSeaweedNFSServesInlineRoundTripOverRPC(t *testing.T) {
client := &fakeNFSFilerClient{
kv: map[string][]byte{
string(filer.InodeIndexKey(101)): testIndexRecord(t, 101, 1, "/exports"),
},
entries: map[util.FullPath]*filer_pb.Entry{
"/exports": testEntry("exports", true, 101, uint32(0755), nil),
},
}
server := newTestServer(t, "/exports", client)
target, cleanup := mountTestTarget(t, server)
defer cleanup()
defer target.Close()
_, err := target.Mkdir("/docs", 0o755)
require.NoError(t, err)
file, err := target.OpenFile("/docs/note.txt", 0o644)
require.NoError(t, err)
payload := []byte("hello over rpc")
_, err = file.Write(payload)
require.NoError(t, err)
require.NoError(t, file.Close())
readFile, err := target.Open("/docs/note.txt")
require.NoError(t, err)
defer readFile.Close()
data, err := io.ReadAll(readFile)
require.NoError(t, err)
assert.Equal(t, payload, data)
entry := client.entries["/exports/docs/note.txt"]
require.NotNil(t, entry)
assert.Equal(t, payload, entry.Content)
assert.Empty(t, entry.Chunks)
_, beforeRenameHandle, err := target.Lookup("/docs/note.txt")
require.NoError(t, err)
entries, err := target.ReadDirPlus("/docs")
require.NoError(t, err)
require.Len(t, entries, 1)
assert.Equal(t, "note.txt", entries[0].Name())
require.NoError(t, target.Rename("/docs/note.txt", "/docs/final.txt"))
_, err = target.GetAttr(beforeRenameHandle)
require.NoError(t, err)
_, _, err = target.Lookup("/docs/final.txt")
require.NoError(t, err)
_, _, err = target.Lookup("/docs/note.txt")
require.Error(t, err)
require.NoError(t, target.Remove("/docs/final.txt"))
_, _, err = target.Lookup("/docs/final.txt")
require.Error(t, err)
}
func TestSeaweedNFSReadOnlyRejectsMutations(t *testing.T) {
client := &fakeNFSFilerClient{
kv: map[string][]byte{
string(filer.InodeIndexKey(101)): testIndexRecord(t, 101, 1, "/exports"),
string(filer.InodeIndexKey(202)): testIndexRecord(t, 202, 3, "/exports/existing.txt"),
},
entries: map[util.FullPath]*filer_pb.Entry{
"/exports": testEntry("exports", true, 101, uint32(0755), nil),
"/exports/existing.txt": testEntry("existing.txt", false, 202, uint32(0644), []byte("seed")),
},
}
server := newTestServer(t, "/exports", client)
server.option.ReadOnly = true
target, cleanup := mountTestTarget(t, server)
defer cleanup()
defer target.Close()
_, err := target.OpenFile("/created.txt", 0o644)
require.Error(t, err)
nfsErr, ok := err.(*nfsclient.Error)
require.True(t, ok)
assert.Equal(t, uint32(nfsclient.NFS3ErrROFS), nfsErr.ErrorNum)
file, err := target.Open("/existing.txt")
require.NoError(t, err)
_, err = file.Write([]byte("mutate"))
require.Error(t, err)
nfsErr, ok = err.(*nfsclient.Error)
require.True(t, ok)
assert.Equal(t, uint32(nfsclient.NFS3ErrROFS), nfsErr.ErrorNum)
_ = file.Close()
readFile, err := target.Open("/existing.txt")
require.NoError(t, err)
defer readFile.Close()
data, err := io.ReadAll(readFile)
require.NoError(t, err)
assert.Equal(t, []byte("seed"), data)
}
func TestSeaweedNFSServesSymlinkRoundTripOverRPC(t *testing.T) {
client := &fakeNFSFilerClient{
kv: map[string][]byte{
string(filer.InodeIndexKey(101)): testIndexRecord(t, 101, 1, "/exports"),
},
entries: map[util.FullPath]*filer_pb.Entry{
"/exports": testEntry("exports", true, 101, uint32(0755), nil),
},
}
server := newTestServer(t, "/exports", client)
target, cleanup := mountTestTarget(t, server)
defer cleanup()
defer target.Close()
file, err := target.OpenFile("/target.txt", 0o644)
require.NoError(t, err)
_, err = file.Write([]byte("payload"))
require.NoError(t, err)
require.NoError(t, file.Close())
require.NoError(t, target.Symlink("target.txt", "/target.link"))
info, _, err := target.Lookup("/target.link")
require.NoError(t, err)
attr, ok := info.(*nfsclient.Fattr)
require.True(t, ok)
assert.Equal(t, uint32(nfsclient.NF3Lnk), attr.Type)
linkFile, err := target.Open("/target.link")
require.NoError(t, err)
defer linkFile.Close()
linkTarget, err := linkFile.Readlink()
require.NoError(t, err)
assert.Equal(t, "target.txt", linkTarget)
entry := client.entries["/exports/target.link"]
require.NotNil(t, entry)
assert.Equal(t, "target.txt", entry.GetAttributes().GetSymlinkTarget())
}
func TestSeaweedNFSServesHardLinkRoundTripOverRPC(t *testing.T) {
client := &fakeNFSFilerClient{
kv: map[string][]byte{
string(filer.InodeIndexKey(101)): testIndexRecord(t, 101, 1, "/exports"),
},
entries: map[util.FullPath]*filer_pb.Entry{
"/exports": testEntry("exports", true, 101, uint32(0755), nil),
},
}
server := newTestServer(t, "/exports", client)
target, cleanup := mountTestTarget(t, server)
defer cleanup()
defer target.Close()
file, err := target.OpenFile("/source.txt", 0o644)
require.NoError(t, err)
payload := []byte("shared content")
_, err = file.Write(payload)
require.NoError(t, err)
require.NoError(t, file.Close())
_, sourceHandle, err := target.Lookup("/source.txt")
require.NoError(t, err)
require.NoError(t, nfsLink(target, sourceHandle, "/linked.txt"))
sourceInfo, sourceHandle, err := target.Lookup("/source.txt")
require.NoError(t, err)
linkedInfo, linkedHandle, err := target.Lookup("/linked.txt")
require.NoError(t, err)
sourceAttr, ok := sourceInfo.(*nfsclient.Fattr)
require.True(t, ok)
linkAttr, ok := linkedInfo.(*nfsclient.Fattr)
require.True(t, ok)
assert.Equal(t, sourceHandle, linkedHandle)
assert.Equal(t, sourceAttr.Fileid, linkAttr.Fileid)
assert.Equal(t, uint32(2), sourceAttr.Nlink)
assert.Equal(t, uint32(2), linkAttr.Nlink)
linkedFile, err := target.Open("/linked.txt")
require.NoError(t, err)
defer linkedFile.Close()
data, err := io.ReadAll(linkedFile)
require.NoError(t, err)
assert.Equal(t, payload, data)
sourceEntry := client.entries["/exports/source.txt"]
linkedEntry := client.entries["/exports/linked.txt"]
require.NotNil(t, sourceEntry)
require.NotNil(t, linkedEntry)
assert.Equal(t, sourceEntry.GetHardLinkId(), linkedEntry.GetHardLinkId())
assert.Equal(t, int32(2), sourceEntry.GetHardLinkCounter())
assert.Equal(t, int32(2), linkedEntry.GetHardLinkCounter())
require.NoError(t, target.Remove("/source.txt"))
remainingAttr, err := target.GetAttr(sourceHandle)
require.NoError(t, err)
assert.Equal(t, uint32(1), remainingAttr.Nlink)
_, _, err = target.Lookup("/source.txt")
require.Error(t, err)
linkedFile, err = target.Open("/linked.txt")
require.NoError(t, err)
data, err = io.ReadAll(linkedFile)
require.NoError(t, err)
require.NoError(t, linkedFile.Close())
assert.Equal(t, payload, data)
require.NoError(t, target.Remove("/linked.txt"))
_, err = target.GetAttr(linkedHandle)
require.Error(t, err)
nfsErr, ok := err.(*nfsclient.Error)
require.True(t, ok)
assert.Equal(t, uint32(nfsclient.NFS3ErrStale), nfsErr.ErrorNum)
}
func TestSeaweedNFSServesLargeChunkRoundTripOverRPC(t *testing.T) {
initIntegrationHTTPClient.Do(util_http.InitGlobalHttpClient)
client := &fakeNFSFilerClient{
kv: map[string][]byte{
string(filer.InodeIndexKey(101)): testIndexRecord(t, 101, 1, "/exports"),
},
entries: map[util.FullPath]*filer_pb.Entry{
"/exports": testEntry("exports", true, 101, uint32(0755), nil),
},
}
volumeServer := newFakeVolumeServer(t)
controlPlane := &fakeVolumeControlPlane{host: volumeServer.host()}
controlPlaneAddr := startFakeVolumeControlPlane(t, controlPlane)
_, grpcPortString, err := net.SplitHostPort(controlPlaneAddr)
require.NoError(t, err)
grpcPort, err := strconv.Atoi(grpcPortString)
require.NoError(t, err)
server := newTestServer(t, "/exports", client)
server.option.Filer = pb.NewServerAddressWithGrpcPort(controlPlaneAddr, grpcPort)
server.option.GrpcDialOption = grpc.WithTransportCredentials(insecure.NewCredentials())
if server.filerClient != nil {
server.filerClient.Close()
}
server.filerClient = wdclient.NewFilerClient([]pb.ServerAddress{server.option.Filer}, server.option.GrpcDialOption, "")
server.withFilerClient = func(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error {
conn, err := grpc.NewClient(controlPlaneAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
return err
}
defer conn.Close()
return fn(filer_pb.NewSeaweedFilerClient(conn))
}
target, cleanup := mountTestTarget(t, server)
defer cleanup()
defer target.Close()
payload := make([]byte, maxInlineWriteSize+4096)
_, err = rand.New(rand.NewSource(1)).Read(payload)
require.NoError(t, err)
file, err := target.OpenFile("/big.bin", 0o644)
require.NoError(t, err)
_, err = file.Write(payload)
require.NoError(t, err)
require.NoError(t, file.Close())
entry := client.entries["/exports/big.bin"]
require.NotNil(t, entry)
require.Len(t, entry.GetChunks(), 1)
assert.Nil(t, entry.Content)
assert.Equal(t, uint64(len(payload)), entry.GetAttributes().GetFileSize())
readFile, err := target.Open("/big.bin")
require.NoError(t, err)
defer readFile.Close()
data, err := io.ReadAll(readFile)
require.NoError(t, err)
assert.Equal(t, payload, data)
controlPlane.mu.Lock()
defer controlPlane.mu.Unlock()
require.Len(t, controlPlane.assigns, 1)
assert.Equal(t, "/exports/big.bin", controlPlane.assigns[0].GetPath())
assert.NotEmpty(t, controlPlane.lookups)
}
func TestSeaweedNFSRejectsStaleHandleAfterDeleteRecreate(t *testing.T) {
client := &fakeNFSFilerClient{
kv: map[string][]byte{
string(filer.InodeIndexKey(101)): testIndexRecord(t, 101, 1, "/exports"),
},
entries: map[util.FullPath]*filer_pb.Entry{
"/exports": testEntry("exports", true, 101, uint32(0755), nil),
},
}
server := newTestServer(t, "/exports", client)
target, cleanup := mountTestTarget(t, server)
defer cleanup()
defer target.Close()
file, err := target.OpenFile("/stale.txt", 0o644)
require.NoError(t, err)
_, err = file.Write([]byte("old"))
require.NoError(t, err)
require.NoError(t, file.Close())
_, oldHandle, err := target.Lookup("/stale.txt")
require.NoError(t, err)
require.NoError(t, target.Remove("/stale.txt"))
file, err = target.OpenFile("/stale.txt", 0o644)
require.NoError(t, err)
_, err = file.Write([]byte("new"))
require.NoError(t, err)
require.NoError(t, file.Close())
_, err = target.GetAttr(oldHandle)
require.Error(t, err)
nfsErr, ok := err.(*nfsclient.Error)
require.True(t, ok)
assert.Equal(t, uint32(nfsclient.NFS3ErrStale), nfsErr.ErrorNum)
_, newHandle, err := target.Lookup("/stale.txt")
require.NoError(t, err)
_, err = target.GetAttr(newHandle)
require.NoError(t, err)
}
func TestSeaweedNFSFileHandleSurvivesServerRestart(t *testing.T) {
client := &fakeNFSFilerClient{
kv: map[string][]byte{
string(filer.InodeIndexKey(101)): testIndexRecord(t, 101, 1, "/exports"),
},
entries: map[util.FullPath]*filer_pb.Entry{
"/exports": testEntry("exports", true, 101, uint32(0755), nil),
},
}
server := newTestServer(t, "/exports", client)
target, cleanup := mountTestTarget(t, server)
file, err := target.OpenFile("/restart.txt", 0o644)
require.NoError(t, err)
payload := []byte("survives restart")
_, err = file.Write(payload)
require.NoError(t, err)
require.NoError(t, file.Close())
_, handle, err := target.Lookup("/restart.txt")
require.NoError(t, err)
target.Close()
cleanup()
restartedServer := newTestServer(t, "/exports", client)
restartedTarget, restartedCleanup := mountTestTarget(t, restartedServer)
defer restartedCleanup()
defer restartedTarget.Close()
attr, err := restartedTarget.GetAttr(handle)
require.NoError(t, err)
assert.Equal(t, uint64(client.entries["/exports/restart.txt"].GetAttributes().GetInode()), attr.Fileid)
_, restartedHandle, err := restartedTarget.Lookup("/restart.txt")
require.NoError(t, err)
assert.Equal(t, handle, restartedHandle)
readFile, err := restartedTarget.Open("/restart.txt")
require.NoError(t, err)
defer readFile.Close()
data, err := io.ReadAll(readFile)
require.NoError(t, err)
assert.Equal(t, payload, data)
}
+88
View File
@@ -0,0 +1,88 @@
package nfs
import (
"context"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"google.golang.org/grpc"
)
type filerClientExecutor func(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error
type internalClientExecutor func(streamingMode bool, fn func(nfsFilerClient) error) error
type nfsListEntriesClient interface {
Recv() (*filer_pb.ListEntriesResponse, error)
}
type nfsSubscribeMetadataClient interface {
Recv() (*filer_pb.SubscribeMetadataResponse, error)
}
type nfsFilerClient interface {
KvGet(ctx context.Context, in *filer_pb.KvGetRequest, opts ...grpc.CallOption) (*filer_pb.KvGetResponse, error)
LookupDirectoryEntry(ctx context.Context, in *filer_pb.LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error)
ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (nfsListEntriesClient, error)
SubscribeMetadata(ctx context.Context, in *filer_pb.SubscribeMetadataRequest, opts ...grpc.CallOption) (nfsSubscribeMetadataClient, error)
CreateEntry(ctx context.Context, in *filer_pb.CreateEntryRequest, opts ...grpc.CallOption) (*filer_pb.CreateEntryResponse, error)
UpdateEntry(ctx context.Context, in *filer_pb.UpdateEntryRequest, opts ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error)
DeleteEntry(ctx context.Context, in *filer_pb.DeleteEntryRequest, opts ...grpc.CallOption) (*filer_pb.DeleteEntryResponse, error)
AtomicRenameEntry(ctx context.Context, in *filer_pb.AtomicRenameEntryRequest, opts ...grpc.CallOption) (*filer_pb.AtomicRenameEntryResponse, error)
Statistics(ctx context.Context, in *filer_pb.StatisticsRequest, opts ...grpc.CallOption) (*filer_pb.StatisticsResponse, error)
}
type grpcNFSFilerClient struct {
client filer_pb.SeaweedFilerClient
}
func (c grpcNFSFilerClient) KvGet(ctx context.Context, in *filer_pb.KvGetRequest, opts ...grpc.CallOption) (*filer_pb.KvGetResponse, error) {
return c.client.KvGet(ctx, in, opts...)
}
func (c grpcNFSFilerClient) LookupDirectoryEntry(ctx context.Context, in *filer_pb.LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
return c.client.LookupDirectoryEntry(ctx, in, opts...)
}
func (c grpcNFSFilerClient) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (nfsListEntriesClient, error) {
return c.client.ListEntries(ctx, in, opts...)
}
func (c grpcNFSFilerClient) SubscribeMetadata(ctx context.Context, in *filer_pb.SubscribeMetadataRequest, opts ...grpc.CallOption) (nfsSubscribeMetadataClient, error) {
return c.client.SubscribeMetadata(ctx, in, opts...)
}
func (c grpcNFSFilerClient) CreateEntry(ctx context.Context, in *filer_pb.CreateEntryRequest, opts ...grpc.CallOption) (*filer_pb.CreateEntryResponse, error) {
return c.client.CreateEntry(ctx, in, opts...)
}
func (c grpcNFSFilerClient) UpdateEntry(ctx context.Context, in *filer_pb.UpdateEntryRequest, opts ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) {
return c.client.UpdateEntry(ctx, in, opts...)
}
func (c grpcNFSFilerClient) DeleteEntry(ctx context.Context, in *filer_pb.DeleteEntryRequest, opts ...grpc.CallOption) (*filer_pb.DeleteEntryResponse, error) {
return c.client.DeleteEntry(ctx, in, opts...)
}
func (c grpcNFSFilerClient) AtomicRenameEntry(ctx context.Context, in *filer_pb.AtomicRenameEntryRequest, opts ...grpc.CallOption) (*filer_pb.AtomicRenameEntryResponse, error) {
return c.client.AtomicRenameEntry(ctx, in, opts...)
}
func (c grpcNFSFilerClient) Statistics(ctx context.Context, in *filer_pb.StatisticsRequest, opts ...grpc.CallOption) (*filer_pb.StatisticsResponse, error) {
return c.client.Statistics(ctx, in, opts...)
}
func newFilerClientExecutor(option *Option, signature int32) filerClientExecutor {
return func(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
return pb.WithGrpcClient(streamingMode, signature, func(grpcConnection *grpc.ClientConn) error {
return fn(filer_pb.NewSeaweedFilerClient(grpcConnection))
}, option.Filer.ToGrpcAddress(), false, option.GrpcDialOption)
}
}
func newInternalClientExecutor(option *Option, signature int32) internalClientExecutor {
return func(streamingMode bool, fn func(nfsFilerClient) error) error {
return pb.WithGrpcClient(streamingMode, signature, func(grpcConnection *grpc.ClientConn) error {
return fn(grpcNFSFilerClient{client: filer_pb.NewSeaweedFilerClient(grpcConnection)})
}, option.Filer.ToGrpcAddress(), false, option.GrpcDialOption)
}
}
+147
View File
@@ -0,0 +1,147 @@
package nfs
import (
"context"
"errors"
"io"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
type chunkInvalidator interface {
UnCache(fileID string)
}
type metadataInvalidation struct {
path util.FullPath
entry *filer_pb.Entry
}
func (s *Server) runMetadataInvalidationLoop(ctx context.Context) {
if s == nil || s.chunkInvalidator == nil || s.withInternalClient == nil {
return
}
waitTime := time.Second
for ctx.Err() == nil {
err := s.followMetadataStream(ctx)
if err == nil || errors.Is(err, context.Canceled) || ctx.Err() != nil {
return
}
glog.V(0).Infof("retry nfs metadata invalidation stream for %s in %v: %v", s.exportRoot, waitTime, err)
timer := time.NewTimer(waitTime)
select {
case <-ctx.Done():
if !timer.Stop() {
<-timer.C
}
return
case <-timer.C:
}
if waitTime < util.RetryWaitTime {
waitTime += waitTime / 2
}
}
}
func (s *Server) followMetadataStream(ctx context.Context) error {
req := &filer_pb.SubscribeMetadataRequest{
ClientName: "nfs",
PathPrefix: string(s.exportRoot),
ClientId: s.signature,
ClientEpoch: 1,
ClientSupportsBatching: true,
}
return s.withInternalClient(true, func(client nfsFilerClient) error {
stream, err := client.SubscribeMetadata(ctx, req)
if err != nil {
return err
}
for {
resp, err := stream.Recv()
if err == io.EOF {
return nil
}
if err != nil {
return err
}
s.applyMetadataInvalidationResponse(resp)
}
})
}
func (s *Server) applyMetadataInvalidationResponse(resp *filer_pb.SubscribeMetadataResponse) {
if s == nil || s.chunkInvalidator == nil || resp == nil {
return
}
uncached := make(map[string]struct{})
apply := func(event *filer_pb.SubscribeMetadataResponse) {
for _, invalidation := range metadataInvalidationsForEvent(event) {
if invalidation.entry == nil || !pathVisibleFromExport(invalidation.path, s.exportRoot) {
continue
}
for _, chunk := range invalidation.entry.GetChunks() {
fileID := chunk.GetFileIdString()
if fileID == "" {
continue
}
if _, seen := uncached[fileID]; seen {
continue
}
uncached[fileID] = struct{}{}
s.chunkInvalidator.UnCache(fileID)
}
}
}
apply(resp)
for _, event := range resp.Events {
apply(event)
}
}
func metadataInvalidationsForEvent(resp *filer_pb.SubscribeMetadataResponse) []metadataInvalidation {
message := resp.GetEventNotification()
if message == nil {
return nil
}
var invalidations []metadataInvalidation
if message.OldEntry != nil && message.NewEntry != nil {
oldPath := util.NewFullPath(resp.Directory, message.OldEntry.Name)
invalidations = append(invalidations, metadataInvalidation{path: oldPath, entry: message.OldEntry})
newDir := resp.Directory
if message.NewParentPath != "" {
newDir = message.NewParentPath
}
if message.OldEntry.Name != message.NewEntry.Name || resp.Directory != newDir {
newPath := util.NewFullPath(newDir, message.NewEntry.Name)
invalidations = append(invalidations, metadataInvalidation{path: newPath, entry: message.NewEntry})
}
return invalidations
}
if message.NewEntry != nil {
newDir := resp.Directory
if message.NewParentPath != "" {
newDir = message.NewParentPath
}
newPath := util.NewFullPath(newDir, message.NewEntry.Name)
invalidations = append(invalidations, metadataInvalidation{path: newPath, entry: message.NewEntry})
}
if message.OldEntry != nil {
oldPath := util.NewFullPath(resp.Directory, message.OldEntry.Name)
invalidations = append(invalidations, metadataInvalidation{path: oldPath, entry: message.OldEntry})
}
return invalidations
}
+343
View File
@@ -0,0 +1,343 @@
package nfs
import (
"context"
"encoding/binary"
"fmt"
"net"
"os"
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/util"
)
// The upstream willscott/go-nfs library only serves the MOUNT protocol over
// TCP. Linux's mount.nfs and the in-kernel NFS client default `mountproto` to
// UDP in many configurations, so against a stock `weed nfs` deployment the
// kernel queries portmap for "MOUNT v3 UDP", gets port=0 ("not registered"),
// and either falls back inconsistently or surfaces EPROTONOSUPPORT
// ("requested NFS version or transport protocol is not supported"). The user
// either has to add `mountproto=tcp` / `mountport=2049` to their mount
// options or guess that their distro happens to fall back to TCP on its own.
//
// This responder closes that gap. It speaks just enough of MOUNT v3 to handle
// MOUNT_NULL / MOUNT_MNT / MOUNT_UMNT over UDP — the only procedures the
// kernel actually invokes during mount setup and teardown — so plain
// `mount -t nfs <host>:<export> /mnt` works without any client-side protocol
// hints. The protocol layout is intentionally identical to the TCP MOUNT
// handler in handler.go's Mount() so the two paths return the same
// filehandle and the same set of auth flavors for the same export.
//
// References: RFC 1813 §5 (NFSv3/MOUNTv3), RFC 5531 (RPC).
const (
mountUDPMaxRecord = 32 * 1024
// mountUDPRetryBackoff mirrors portmapRetryBackoff so the two
// listening goroutines back off identically under host pressure.
mountUDPRetryBackoff = 50 * time.Millisecond
// mountUDPLookupTimeout bounds any filer round-trip the UDP MOUNT
// path makes (export-root existence check, subexport lookup). The
// UDP serve loop is single-threaded, so a stalled filer call would
// otherwise block every later MOUNT packet.
mountUDPLookupTimeout = 5 * time.Second
mountVersion = 3
mountProcNull = 0
mountProcMnt = 1
mountProcUmnt = 3
// MOUNT v3 status codes (mountstat3 in RFC 1813 §5.1.1).
mnt3StatOK uint32 = 0
mnt3ErrAcces uint32 = 13
mnt3ErrNoEnt uint32 = 2
mnt3ErrNotDir uint32 = 20
mnt3ErrServerFault uint32 = 10006
// XDR opaque length cap for dirpath. RFC 1813 §5.1 limits MNTPATHLEN
// to 1024; cap a bit higher for headroom and reject anything beyond.
mountUDPMaxPathLen = 4096
// AuthFlavor numeric IDs (matches go-nfs and RFC 5531 §8).
authFlavorNull = 0
authFlavorUnix = 1
)
// mountUDPServer answers MOUNT v3 RPCs over UDP. It listens on the same port
// the NFS TCP server uses (2049 by default), since that's what we advertise
// via portmap, and shares the parent Server's exportRoot, exportID, and
// client allowlist so the UDP MOUNT path applies the same access policy as
// the TCP path.
type mountUDPServer struct {
bindIP string
port int
server *Server
udpConn *net.UDPConn
mu sync.Mutex
closed bool
done chan struct{}
wg sync.WaitGroup
}
func newMountUDPServer(bindIP string, port int, server *Server) *mountUDPServer {
return &mountUDPServer{
bindIP: bindIP,
port: port,
server: server,
done: make(chan struct{}),
}
}
func (m *mountUDPServer) Start() error {
addr := net.JoinHostPort(m.bindIP, fmt.Sprintf("%d", m.port))
udpAddr, err := net.ResolveUDPAddr("udp", addr)
if err != nil {
return fmt.Errorf("mount udp resolve %s: %w", addr, err)
}
udpConn, err := net.ListenUDP("udp", udpAddr)
if err != nil {
return fmt.Errorf("mount udp listen %s: %w", addr, err)
}
m.udpConn = udpConn
m.wg.Add(1)
go func() {
defer m.wg.Done()
m.serve()
}()
return nil
}
func (m *mountUDPServer) Close() error {
m.mu.Lock()
if m.closed {
m.mu.Unlock()
return nil
}
m.closed = true
close(m.done)
m.mu.Unlock()
if m.udpConn != nil {
_ = m.udpConn.Close()
}
m.wg.Wait()
return nil
}
func (m *mountUDPServer) isClosed() bool {
m.mu.Lock()
defer m.mu.Unlock()
return m.closed
}
func (m *mountUDPServer) serve() {
buf := make([]byte, mountUDPMaxRecord)
for {
n, addr, err := m.udpConn.ReadFromUDP(buf)
if err != nil {
if m.isClosed() {
return
}
// Transient read failure: log, back off, keep the
// responder alive — same pattern as portmap UDP.
glog.V(1).Infof("mount udp read: %v", err)
select {
case <-m.done:
return
case <-time.After(mountUDPRetryBackoff):
continue
}
}
// Apply the parent server's client allowlist before we even
// look at the RPC bytes, mirroring the TCP path's
// allowlistListener wrapping.
if m.server != nil && m.server.clientAuthorizer != nil && !m.server.clientAuthorizer.isAllowedAddr(addr) {
glog.V(1).Infof("mount udp: rejecting unauthorized client %s", addr)
continue
}
reply := m.handleCall(buf[:n], addr)
if reply == nil {
continue
}
if _, err := m.udpConn.WriteToUDP(reply, addr); err != nil {
glog.V(1).Infof("mount udp write to %s: %v", addr, err)
}
}
}
// handleCall classifies one RPC CALL message and returns the encoded reply,
// or nil if the call is malformed enough to drop silently.
func (m *mountUDPServer) handleCall(callBuf []byte, addr *net.UDPAddr) []byte {
xid, prog, vers, proc, args, err := parseRPCCall(callBuf)
if err != nil {
return nil
}
if prog != mountProgram {
return encodeAcceptedReply(xid, rpcAcceptProgUnavail, nil)
}
if vers != mountVersion {
// Mismatch — advertise the v3..v3 we actually support.
body := make([]byte, 8)
binary.BigEndian.PutUint32(body[0:4], mountVersion)
binary.BigEndian.PutUint32(body[4:8], mountVersion)
return encodeAcceptedReply(xid, rpcAcceptProgMismatch, body)
}
switch proc {
case mountProcNull:
return encodeAcceptedReply(xid, rpcAcceptSuccess, nil)
case mountProcMnt:
return m.handleMount(xid, args, addr)
case mountProcUmnt:
// Stateless server: there's nothing to forget, just acknowledge.
// The client sends back the dirpath in args; we don't need to
// validate it here because UMNT has no return data.
return encodeAcceptedReply(xid, rpcAcceptSuccess, nil)
default:
// MOUNT v3 also defines DUMP / EXPORT / UMNTALL but the kernel
// mount path doesn't invoke them. Returning PROC_UNAVAIL is
// the protocol-correct response.
return encodeAcceptedReply(xid, rpcAcceptProcUnavail, nil)
}
}
// handleMount implements MOUNT v3 MNT. RFC 1813 §5.1.4:
//
// MOUNT3args { dirpath3 dirpath; } // XDR opaque
// MOUNT3res { mountstat3 status; if OK { handle, auth_flavors[] } }
//
// Mirrors Handler.resolveMountFilesystem: exact match returns the
// synthetic root handle; under-export resolves to the subdirectory's
// handle; outside-export falls back to the synthetic root.
func (m *mountUDPServer) handleMount(xid uint32, args []byte, addr *net.UDPAddr) []byte {
if len(args) < 4 {
return encodeAcceptedReply(xid, rpcAcceptGarbageArgs, nil)
}
pathLen := binary.BigEndian.Uint32(args[0:4])
if pathLen > mountUDPMaxPathLen {
return encodeAcceptedReply(xid, rpcAcceptGarbageArgs, nil)
}
padded := (pathLen + 3) &^ 3
if uint32(len(args)) < 4+padded {
return encodeAcceptedReply(xid, rpcAcceptGarbageArgs, nil)
}
dirpath := string(args[4 : 4+pathLen])
requested := normalizeExportRoot(util.FullPath(dirpath))
flavors := []uint32{authFlavorNull, authFlavorUnix}
ctx, cancel := context.WithTimeout(context.Background(), mountUDPLookupTimeout)
defer cancel()
// Exact match and outside-export both fall back to the synthetic root
// handle. Only the second case logs; the first is the common path.
if requested == m.server.exportRoot || !requested.IsUnder(m.server.exportRoot) {
if requested != m.server.exportRoot {
glog.V(0).Infof("mount udp: client %s requested %q (outside export %q); serving configured export", addr, dirpath, m.server.exportRoot)
}
if status := m.rootMountStatus(ctx); status != mnt3StatOK {
return encodeMountStatus(xid, status)
}
return encodeMountSuccess(xid, syntheticRootHandle(m.server), flavors)
}
fh, status := m.resolveSubexportFileHandle(ctx, requested)
if status != mnt3StatOK {
return encodeMountStatus(xid, status)
}
glog.V(1).Infof("mount udp: client %s requested %q under export %q; mounting at subdirectory", addr, dirpath, m.server.exportRoot)
return encodeMountSuccess(xid, fh, flavors)
}
// rootMountStatus is the UDP analogue of Handler.lstatExportStatus:
// confirms the configured export root still exists in the filer so the
// transport-OK branches can't hand out a handle pointing at a deleted
// directory. Reuses the Server's shared rootFS instance so we don't
// construct a wrapper per MOUNT request.
func (m *mountUDPServer) rootMountStatus(ctx context.Context) uint32 {
if m.server.withInternalClient == nil {
return mnt3StatOK
}
switch _, err := m.server.rootFilesystem().fileInfoForVirtualPath(ctx, "/"); {
case err == nil:
return mnt3StatOK
case os.IsNotExist(err):
return mnt3ErrNoEnt
default:
glog.Errorf("mount udp: export root %q lookup failed: %v", m.server.exportRoot, err)
return mnt3ErrServerFault
}
}
// resolveSubexportFileHandle is the UDP analogue of the sub-fs branch in
// Handler.resolveMountFilesystem. The TCP path lets go-nfs's onMount call
// ToHandle on the returned filesystem; UDP encodes the FH itself, so the
// inode/generation lookup happens explicitly here.
//
// The UDP listener is up before serve() runs newHandler(), so a subexport
// MOUNT can land here before sharedReaderCache has been assigned. Resolve
// the rootFS first to drive Server.rootFilesystem's sync.Once and read
// the cache directly off it, so the new sub-fs always shares the same
// reader cache the TCP path uses.
func (m *mountUDPServer) resolveSubexportFileHandle(ctx context.Context, requested util.FullPath) ([]byte, uint32) {
if m.server.withInternalClient == nil {
return nil, mnt3ErrServerFault
}
rootFS := m.server.rootFilesystem()
subFS := newSeaweedFileSystem(m.server, requested, rootFS.readerCache)
info, err := subFS.fileInfoForVirtualPath(ctx, "/")
switch {
case err == nil:
case os.IsNotExist(err):
return nil, mnt3ErrNoEnt
default:
glog.Errorf("mount udp: subexport lookup %q failed: %v", requested, err)
return nil, mnt3ErrServerFault
}
if !info.entry.IsDirectory {
return nil, mnt3ErrNotDir
}
inode := info.entry.GetAttributes().GetInode()
return NewFileHandle(m.server.exportID, FileHandleKindDirectory, inode, info.generation).Encode(), mnt3StatOK
}
func syntheticRootHandle(s *Server) []byte {
return NewFileHandle(s.exportID, FileHandleKindDirectory, 0, filer.InodeIndexInitialGeneration).Encode()
}
// encodeMountStatus returns a MOUNT MNT reply carrying just an error status.
// Per RFC 1813 §5.1.4 a non-OK status terminates the response — no handle or
// flavors follow.
func encodeMountStatus(xid, status uint32) []byte {
body := make([]byte, 4)
binary.BigEndian.PutUint32(body, status)
return encodeAcceptedReply(xid, rpcAcceptSuccess, body)
}
// encodeMountSuccess builds the OK MOUNT MNT reply: status=OK, file handle
// (XDR opaque), and the supported auth_flavors list.
func encodeMountSuccess(xid uint32, handle []byte, flavors []uint32) []byte {
handleLen := uint32(len(handle))
handlePadded := (handleLen + 3) &^ 3
bodyLen := 4 + 4 + handlePadded + 4 + 4*uint32(len(flavors))
body := make([]byte, bodyLen)
binary.BigEndian.PutUint32(body[0:4], mnt3StatOK)
binary.BigEndian.PutUint32(body[4:8], handleLen)
copy(body[8:8+handleLen], handle)
// Trailing pad bytes are already zero from make().
pos := 8 + handlePadded
binary.BigEndian.PutUint32(body[pos:pos+4], uint32(len(flavors)))
pos += 4
for _, fl := range flavors {
binary.BigEndian.PutUint32(body[pos:pos+4], fl)
pos += 4
}
return encodeAcceptedReply(xid, rpcAcceptSuccess, body)
}
+431
View File
@@ -0,0 +1,431 @@
package nfs
import (
"context"
"encoding/binary"
"net"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
gonfs "github.com/willscott/go-nfs"
)
// buildMountCallFrame constructs a MOUNT v3 RPC CALL with an opaque dirpath
// argument. The shape matches RFC 5531 §9: xid + msg_type=CALL + rpcvers=2 +
// prog + vers + proc + cred(AUTH_NONE) + verf(AUTH_NONE) + arg.
func buildMountCallFrame(xid, prog, vers, proc uint32, dirpath string) []byte {
// RPC CALL header (24 bytes) + 2 × AUTH_NONE opaque_auth (16 bytes) +
// dirpath as XDR opaque (4-byte length + padded body).
dpLen := uint32(len(dirpath))
dpPadded := (dpLen + 3) &^ 3
out := make([]byte, 24+16+4+dpPadded)
binary.BigEndian.PutUint32(out[0:4], xid)
binary.BigEndian.PutUint32(out[4:8], rpcMsgCall)
binary.BigEndian.PutUint32(out[8:12], 2) // rpcvers
binary.BigEndian.PutUint32(out[12:16], prog)
binary.BigEndian.PutUint32(out[16:20], vers)
binary.BigEndian.PutUint32(out[20:24], proc)
// cred + verf both AUTH_NONE / length 0 (already zero-filled).
binary.BigEndian.PutUint32(out[40:44], dpLen)
copy(out[44:44+dpLen], dirpath)
return out
}
func newMountUDPTestServer(t *testing.T, exportPath string) (*mountUDPServer, *net.UDPConn) {
t.Helper()
return newMountUDPTestServerWithClient(t, exportPath, nil)
}
// newMountUDPTestServerWithClient wires Server.withInternalClient when
// client is non-nil, so the under-export lookup branch in handleMount
// can find directory entries.
func newMountUDPTestServerWithClient(t *testing.T, exportPath string, client *fakeNFSFilerClient) (*mountUDPServer, *net.UDPConn) {
t.Helper()
exportRoot := normalizeExportRoot(util.FullPath(exportPath))
authz, err := newClientAuthorizer(nil)
if err != nil {
t.Fatal(err)
}
srv := &Server{
option: &Option{},
exportRoot: exportRoot,
exportID: exportIDForRoot(exportRoot),
clientAuthorizer: authz,
}
if client != nil {
srv.withInternalClient = func(_ bool, fn func(nfsFilerClient) error) error {
return fn(client)
}
}
udpAddr, err := net.ResolveUDPAddr("udp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
conn, err := net.ListenUDP("udp", udpAddr)
if err != nil {
t.Fatal(err)
}
m := &mountUDPServer{
bindIP: "127.0.0.1",
port: conn.LocalAddr().(*net.UDPAddr).Port,
server: srv,
udpConn: conn,
done: make(chan struct{}),
}
m.wg.Add(1)
go func() {
defer m.wg.Done()
m.serve()
}()
t.Cleanup(func() {
_ = m.Close()
})
return m, conn
}
func sendMountUDP(t *testing.T, target *net.UDPAddr, payload []byte) []byte {
t.Helper()
c, err := net.DialUDP("udp", nil, target)
if err != nil {
t.Fatal(err)
}
defer c.Close()
if _, err := c.Write(payload); err != nil {
t.Fatal(err)
}
_ = c.SetReadDeadline(time.Now().Add(2 * time.Second))
buf := make([]byte, 4096)
n, err := c.Read(buf)
if err != nil {
t.Fatalf("read reply: %v", err)
}
return buf[:n]
}
// parseRPCReply pulls xid, accept_stat, and the body that follows accept_stat
// out of a MSG_ACCEPTED reply. Unlike the TCP path there is no fragment
// marker — the entire UDP datagram is the reply.
func parseRPCReply(t *testing.T, reply []byte) (xid, acceptStat uint32, body []byte) {
t.Helper()
if len(reply) < 24 {
t.Fatalf("reply too short: %d bytes", len(reply))
}
xid = binary.BigEndian.Uint32(reply[0:4])
if mt := binary.BigEndian.Uint32(reply[4:8]); mt != rpcMsgReply {
t.Fatalf("msg_type=%d want REPLY(1)", mt)
}
if rs := binary.BigEndian.Uint32(reply[8:12]); rs != rpcMsgAccepted {
t.Fatalf("reply_stat=%d want MSG_ACCEPTED(0)", rs)
}
acceptStat = binary.BigEndian.Uint32(reply[20:24])
body = reply[24:]
return
}
func TestMountUDPNullReturnsSuccess(t *testing.T) {
m, conn := newMountUDPTestServer(t, "/exports")
target := conn.LocalAddr().(*net.UDPAddr)
reply := sendMountUDP(t, target, buildMountCallFrame(7, mountProgram, 3, mountProcNull, ""))
xid, astat, body := parseRPCReply(t, reply)
if xid != 7 {
t.Errorf("xid=%d want 7", xid)
}
if astat != rpcAcceptSuccess {
t.Errorf("accept_stat=%d want SUCCESS(0)", astat)
}
if len(body) != 0 {
t.Errorf("NULL reply body should be empty, got %d bytes", len(body))
}
_ = m
}
func TestMountUDPMntReturnsHandleAndFlavors(t *testing.T) {
m, conn := newMountUDPTestServer(t, "/exports")
target := conn.LocalAddr().(*net.UDPAddr)
reply := sendMountUDP(t, target, buildMountCallFrame(42, mountProgram, 3, mountProcMnt, "/exports"))
xid, astat, body := parseRPCReply(t, reply)
if xid != 42 {
t.Errorf("xid=%d want 42", xid)
}
if astat != rpcAcceptSuccess {
t.Fatalf("accept_stat=%d want SUCCESS(0)", astat)
}
if len(body) < 4 {
t.Fatalf("body too short: %d bytes", len(body))
}
status := binary.BigEndian.Uint32(body[0:4])
if status != mnt3StatOK {
t.Fatalf("mountstat3=%d want OK(0)", status)
}
// fhandle3: uint32 length + padded opaque bytes.
if len(body) < 8 {
t.Fatalf("body missing handle length: %d bytes", len(body))
}
handleLen := binary.BigEndian.Uint32(body[4:8])
handlePadded := (handleLen + 3) &^ 3
if uint32(len(body)) < 8+handlePadded+4 {
t.Fatalf("body truncated: have %d, need at least %d", len(body), 8+handlePadded+4)
}
handle := body[8 : 8+handleLen]
if _, err := DecodeFileHandle(handle); err != nil {
t.Fatalf("returned handle does not decode: %v", err)
}
flavorOff := 8 + handlePadded
count := binary.BigEndian.Uint32(body[flavorOff : flavorOff+4])
if count != 2 {
t.Errorf("flavor count=%d want 2 (NULL + UNIX)", count)
}
got := []uint32{
binary.BigEndian.Uint32(body[flavorOff+4 : flavorOff+8]),
binary.BigEndian.Uint32(body[flavorOff+8 : flavorOff+12]),
}
if got[0] != authFlavorNull || got[1] != authFlavorUnix {
t.Errorf("flavors=%v want [%d %d]", got, authFlavorNull, authFlavorUnix)
}
_ = m
}
func TestMountUDPMntAcceptsAnyPath(t *testing.T) {
const exportRoot = "/buckets/data"
_, conn := newMountUDPTestServer(t, exportRoot)
target := conn.LocalAddr().(*net.UDPAddr)
dirpaths := []string{
"/",
"/buckets",
"/buckets/other",
"/wrong/path",
"",
"buckets/data",
exportRoot,
exportRoot + "/",
}
for i, dirpath := range dirpaths {
t.Run(dirpath, func(t *testing.T) {
xid := uint32(1000 + i)
reply := sendMountUDP(t, target, buildMountCallFrame(xid, mountProgram, 3, mountProcMnt, dirpath))
_, astat, body := parseRPCReply(t, reply)
if astat != rpcAcceptSuccess {
t.Fatalf("accept_stat=%d want SUCCESS(0)", astat)
}
if len(body) < 4 {
t.Fatalf("body too short: %d bytes", len(body))
}
if got := binary.BigEndian.Uint32(body[0:4]); got != mnt3StatOK {
t.Errorf("MNT(%q): mountstat3=%d want OK(0)", dirpath, got)
}
if len(body) <= 4 {
t.Errorf("MNT(%q) success body must include handle and flavors", dirpath)
}
})
}
}
func TestMountUDPSubexportMount(t *testing.T) {
const exportRoot = "/buckets"
client := &fakeNFSFilerClient{
entries: map[util.FullPath]*filer_pb.Entry{
"/buckets": testEntry("buckets", true, 100, uint32(0755), nil),
"/buckets/data": testEntry("data", true, 101, uint32(0755), nil),
"/buckets/data/nested": testEntry("nested", true, 102, uint32(0755), nil),
"/buckets/file.txt": testEntry("file.txt", false, 103, uint32(0644), []byte("hi")),
},
kv: map[string][]byte{
string(filer.InodeIndexKey(100)): testIndexRecord(t, 100, 1, "/buckets"),
string(filer.InodeIndexKey(101)): testIndexRecord(t, 101, 1, "/buckets/data"),
string(filer.InodeIndexKey(102)): testIndexRecord(t, 102, 1, "/buckets/data/nested"),
string(filer.InodeIndexKey(103)): testIndexRecord(t, 103, 1, "/buckets/file.txt"),
},
}
m, conn := newMountUDPTestServerWithClient(t, exportRoot, client)
target := conn.LocalAddr().(*net.UDPAddr)
// Build a TCP Handler from the same Server so we can compare the
// raw FH bytes both transports produce for the same subdirectory.
tcpHandler, err := m.server.newHandler()
require.NoError(t, err)
cases := []struct {
name string
dirpath string
wantStatus uint32
wantInode uint64
}{
{name: "subdirectory_one_level", dirpath: "/buckets/data", wantStatus: mnt3StatOK, wantInode: 101},
{name: "subdirectory_two_levels", dirpath: "/buckets/data/nested", wantStatus: mnt3StatOK, wantInode: 102},
{name: "subdirectory_trailing_slash", dirpath: "/buckets/data/", wantStatus: mnt3StatOK, wantInode: 101},
{name: "missing_under_export", dirpath: "/buckets/missing", wantStatus: mnt3ErrNoEnt},
{name: "deep_missing_under_export", dirpath: "/buckets/data/no-such-thing", wantStatus: mnt3ErrNoEnt},
{name: "regular_file_not_directory", dirpath: "/buckets/file.txt", wantStatus: mnt3ErrNotDir},
}
for i, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
xid := uint32(2000 + i)
reply := sendMountUDP(t, target, buildMountCallFrame(xid, mountProgram, 3, mountProcMnt, tc.dirpath))
_, astat, body := parseRPCReply(t, reply)
if astat != rpcAcceptSuccess {
t.Fatalf("accept_stat=%d want SUCCESS(0)", astat)
}
if len(body) < 4 {
t.Fatalf("body too short: %d bytes", len(body))
}
got := binary.BigEndian.Uint32(body[0:4])
if got != tc.wantStatus {
t.Fatalf("MNT(%q) status=%d want %d", tc.dirpath, got, tc.wantStatus)
}
if tc.wantStatus != mnt3StatOK {
if len(body) != 4 {
t.Errorf("MNT(%q) error body should carry only the status; got %d trailing bytes", tc.dirpath, len(body)-4)
}
return
}
if len(body) < 8 {
t.Fatalf("MNT(%q) success body missing handle length", tc.dirpath)
}
handleLen := binary.BigEndian.Uint32(body[4:8])
if uint32(len(body)) < 8+handleLen {
t.Fatalf("MNT(%q) success body truncated", tc.dirpath)
}
udpHandleBytes := body[8 : 8+handleLen]
handle, err := DecodeFileHandle(udpHandleBytes)
if err != nil {
t.Fatalf("MNT(%q) handle decode: %v", tc.dirpath, err)
}
if handle.Inode != tc.wantInode {
t.Errorf("MNT(%q) FH inode=%d want %d", tc.dirpath, handle.Inode, tc.wantInode)
}
if handle.Kind != FileHandleKindDirectory {
t.Errorf("MNT(%q) FH kind=%d want directory", tc.dirpath, handle.Kind)
}
// Transport parity: drive the TCP Handler with the same dirpath
// and confirm the bytes go-nfs's onMount would write match the
// UDP responder's bytes exactly. A regression that drifts the
// generation, exportID, or kind on one transport would fail here.
tcpStatus, tcpFS, _ := tcpHandler.Mount(context.Background(), nil, gonfs.MountRequest{Dirpath: []byte(tc.dirpath)})
require.Equal(t, gonfs.MountStatusOk, tcpStatus, "TCP Mount(%q)", tc.dirpath)
tcpHandleBytes := tcpHandler.ToHandle(tcpFS, nil)
require.NotEmpty(t, tcpHandleBytes, "TCP Mount(%q) ToHandle returned empty", tc.dirpath)
assert.Equal(t, tcpHandleBytes, udpHandleBytes, "TCP/UDP FH bytes diverge for %q", tc.dirpath)
})
}
}
func TestMountUDPRejectsWrongVersion(t *testing.T) {
// Same defence-in-depth as the TCP version filter: don't speak v1/v4
// MOUNT — return PROG_MISMATCH advertising 3..3 so the client knows
// to retry with v3.
_, conn := newMountUDPTestServer(t, "/exports")
target := conn.LocalAddr().(*net.UDPAddr)
reply := sendMountUDP(t, target, buildMountCallFrame(1, mountProgram, 4, mountProcNull, ""))
_, astat, body := parseRPCReply(t, reply)
if astat != rpcAcceptProgMismatch {
t.Fatalf("accept_stat=%d want PROG_MISMATCH(2)", astat)
}
if len(body) != 8 {
t.Fatalf("PROG_MISMATCH body=%d bytes want 8", len(body))
}
low := binary.BigEndian.Uint32(body[0:4])
high := binary.BigEndian.Uint32(body[4:8])
if low != 3 || high != 3 {
t.Errorf("supported range=(%d,%d) want (3,3)", low, high)
}
}
func TestMountUDPRejectsWrongProgram(t *testing.T) {
_, conn := newMountUDPTestServer(t, "/exports")
target := conn.LocalAddr().(*net.UDPAddr)
// 100021 is NLM, which we don't run here.
reply := sendMountUDP(t, target, buildMountCallFrame(1, 100021, 4, mountProcNull, ""))
_, astat, _ := parseRPCReply(t, reply)
if astat != rpcAcceptProgUnavail {
t.Errorf("accept_stat=%d want PROG_UNAVAIL(1)", astat)
}
}
func TestMountUDPUmntAcknowledges(t *testing.T) {
_, conn := newMountUDPTestServer(t, "/exports")
target := conn.LocalAddr().(*net.UDPAddr)
// UMNT carries a dirpath but the server is stateless and ignores it.
reply := sendMountUDP(t, target, buildMountCallFrame(8, mountProgram, 3, mountProcUmnt, "/exports"))
_, astat, body := parseRPCReply(t, reply)
if astat != rpcAcceptSuccess {
t.Errorf("accept_stat=%d want SUCCESS(0)", astat)
}
if len(body) != 0 {
t.Errorf("UMNT reply body should be empty, got %d bytes", len(body))
}
}
func TestMountUDPRejectsTruncatedMntArgs(t *testing.T) {
_, conn := newMountUDPTestServer(t, "/exports")
target := conn.LocalAddr().(*net.UDPAddr)
// Hand-craft an MNT call whose dirpath length field claims 16 bytes
// but no body follows. Using buildMountCallFrame would also emit a
// trailing length=0 from the empty-string default; we need exactly
// "length, no body" so the GARBAGE_ARGS path actually fires.
frame := make([]byte, 24+16+4) // header + auth + 4-byte length only
binary.BigEndian.PutUint32(frame[0:4], 1) // xid
binary.BigEndian.PutUint32(frame[4:8], rpcMsgCall) // msg_type
binary.BigEndian.PutUint32(frame[8:12], 2) // rpcvers
binary.BigEndian.PutUint32(frame[12:16], mountProgram)
binary.BigEndian.PutUint32(frame[16:20], 3) // mount vers
binary.BigEndian.PutUint32(frame[20:24], mountProcMnt)
// auth = two AUTH_NONE / length-0 stanzas (already zero from make).
binary.BigEndian.PutUint32(frame[40:44], 16) // dirpath length=16, no bytes follow
reply := sendMountUDP(t, target, frame)
_, astat, _ := parseRPCReply(t, reply)
if astat != rpcAcceptGarbageArgs {
t.Errorf("accept_stat=%d want GARBAGE_ARGS(4)", astat)
}
}
func TestMountUDPCloseStopsServing(t *testing.T) {
m, conn := newMountUDPTestServer(t, "/exports")
target := conn.LocalAddr().(*net.UDPAddr)
// Sanity: NULL works before close.
_ = sendMountUDP(t, target, buildMountCallFrame(1, mountProgram, 3, mountProcNull, ""))
if err := m.Close(); err != nil {
t.Fatalf("Close: %v", err)
}
// After Close the socket is shut, so a fresh send should fail to
// read a reply within the deadline rather than producing a
// well-formed response.
c, err := net.DialUDP("udp", nil, target)
if err != nil {
// Some platforms refuse the dial outright after Close — that's
// also acceptable: the server is gone either way.
return
}
defer c.Close()
_, _ = c.Write(buildMountCallFrame(2, mountProgram, 3, mountProcNull, ""))
_ = c.SetReadDeadline(time.Now().Add(200 * time.Millisecond))
buf := make([]byte, 1024)
if _, err := c.Read(buf); err == nil {
t.Error("Close should have stopped the responder, but a reply still arrived")
}
}
+447
View File
@@ -0,0 +1,447 @@
package nfs
import (
"encoding/binary"
"errors"
"fmt"
"io"
"net"
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
)
// Minimal PORTMAP v2 responder.
//
// The upstream willscott/go-nfs library serves NFSv3 and MOUNT on a single TCP
// port and deliberately does not register with portmap (RPC program 100000).
// Linux mount.nfs, however, queries portmap on port 111 before sending the
// MOUNT RPC, so the plain `mount -t nfs host:/export /mnt` command fails
// against a default `weed nfs` deployment.
//
// When enabled, this responder binds the privileged port 111 (RFC 1833) on
// both TCP and UDP and answers the subset of PORTMAP v2 calls that standard
// Linux clients make: PMAP_NULL, PMAP_GETPORT and PMAP_DUMP. It refuses
// registration from third parties (PMAP_SET / PMAP_UNSET return false) and
// only exposes the programs that weed itself serves.
//
// References: RFC 1833 (Portmap v2), RFC 5531 (RPC).
const (
portmapProgram = 100000
portmapVersion = 2
portmapPort = 111
pmapProcNull = 0
pmapProcSet = 1
pmapProcUnset = 2
pmapProcGetPort = 3
pmapProcDump = 4
ipProtoTCP = 6
ipProtoUDP = 17
nfsProgram = 100003
mountProgram = 100005
// RPC
rpcMsgCall = 0
rpcMsgReply = 1
rpcMsgAccepted = 0
rpcAcceptSuccess = 0
rpcAcceptProgUnavail = 1
rpcAcceptProgMismatch = 2
rpcAcceptProcUnavail = 3
rpcAcceptGarbageArgs = 4
rpcAuthNone = 0
// Defensive limits. Portmap messages are tiny in practice; these caps
// protect the responder from large or slow reads.
portmapMaxRecord = 64 * 1024
// Per-connection read/write deadlines on the TCP listener. The idle
// timeout bounds how long we wait for the next request on an otherwise
// quiet connection; the IO timeout bounds a single read or write once
// one is in flight. Both guard against slowloris-style stalls on the
// privileged port 111.
portmapTCPIdleTimeout = 30 * time.Second
portmapTCPIOTimeout = 10 * time.Second
// Back-off applied before retrying after a non-fatal listener error
// (e.g. EMFILE on TCP Accept, or a transient UDP read failure) so we
// don't busy-loop when the host is under pressure.
portmapRetryBackoff = 50 * time.Millisecond
)
type portmapEntry struct {
Program uint32
Version uint32
Protocol uint32
Port uint32
}
type portmapServer struct {
bindIP string
port int
entries []portmapEntry
tcpListener net.Listener
udpConn *net.UDPConn
// mu guards closed and conns. It is held only for bookkeeping, never
// across network IO.
mu sync.Mutex
closed bool
conns map[net.Conn]struct{}
// done is closed exactly once by Close() so that background loops can
// interrupt a retry-backoff sleep instead of waiting it out.
done chan struct{}
wg sync.WaitGroup
}
// newPortmapServer builds a responder advertising the NFS services the caller
// runs on nfsPort. NFS itself is TCP-only here (the upstream go-nfs library
// doesn't speak NFS UDP). MOUNT, however, is served over both TCP (via
// go-nfs) and UDP (via mountUDPServer in mount_udp.go), so we advertise
// both — that's what makes plain `mount -t nfs <host>:<export> /mnt` work
// against Linux clients whose default mountproto is UDP without needing
// mountproto=tcp / mountport=2049 mount options.
func newPortmapServer(bindIP string, port int, nfsPort uint32) *portmapServer {
if port <= 0 {
port = portmapPort
}
return &portmapServer{
bindIP: bindIP,
port: port,
done: make(chan struct{}),
entries: []portmapEntry{
{Program: nfsProgram, Version: 3, Protocol: ipProtoTCP, Port: nfsPort},
{Program: mountProgram, Version: 3, Protocol: ipProtoTCP, Port: nfsPort},
{Program: mountProgram, Version: 3, Protocol: ipProtoUDP, Port: nfsPort},
},
}
}
func (ps *portmapServer) Start() error {
addr := net.JoinHostPort(ps.bindIP, fmt.Sprintf("%d", ps.port))
tcpLn, err := net.Listen("tcp", addr)
if err != nil {
return fmt.Errorf("portmap tcp listen %s: %w", addr, err)
}
udpAddr, err := net.ResolveUDPAddr("udp", addr)
if err != nil {
_ = tcpLn.Close()
return fmt.Errorf("portmap udp resolve %s: %w", addr, err)
}
udpConn, err := net.ListenUDP("udp", udpAddr)
if err != nil {
_ = tcpLn.Close()
return fmt.Errorf("portmap udp listen %s: %w", addr, err)
}
ps.tcpListener = tcpLn
ps.udpConn = udpConn
ps.wg.Add(2)
go func() {
defer ps.wg.Done()
ps.serveTCP()
}()
go func() {
defer ps.wg.Done()
ps.serveUDP()
}()
return nil
}
func (ps *portmapServer) Close() error {
ps.mu.Lock()
if ps.closed {
ps.mu.Unlock()
return nil
}
ps.closed = true
conns := ps.conns
ps.conns = nil
close(ps.done)
ps.mu.Unlock()
var first error
if ps.tcpListener != nil {
if err := ps.tcpListener.Close(); err != nil {
first = err
}
}
if ps.udpConn != nil {
if err := ps.udpConn.Close(); err != nil && first == nil {
first = err
}
}
// Evict in-flight TCP handlers so Close() does not block on idle
// clients; their read goroutines will unwind on the closed conn.
for c := range conns {
_ = c.Close()
}
ps.wg.Wait()
return first
}
func (ps *portmapServer) isClosed() bool {
ps.mu.Lock()
defer ps.mu.Unlock()
return ps.closed
}
// addConn registers c for shutdown eviction. It returns false (and the
// caller must drop c) if the server has already started shutting down.
func (ps *portmapServer) addConn(c net.Conn) bool {
ps.mu.Lock()
defer ps.mu.Unlock()
if ps.closed {
return false
}
if ps.conns == nil {
ps.conns = make(map[net.Conn]struct{})
}
ps.conns[c] = struct{}{}
return true
}
func (ps *portmapServer) removeConn(c net.Conn) {
ps.mu.Lock()
defer ps.mu.Unlock()
delete(ps.conns, c)
}
func (ps *portmapServer) serveTCP() {
for {
conn, err := ps.tcpListener.Accept()
if err != nil {
if ps.isClosed() {
return
}
// Non-fatal (e.g. EMFILE, EINTR): log and back off rather
// than tear the listener down on a transient resource blip.
// Wake early if Close() fires during the sleep.
glog.V(1).Infof("portmap tcp accept: %v", err)
select {
case <-ps.done:
return
case <-time.After(portmapRetryBackoff):
continue
}
}
if !ps.addConn(conn) {
_ = conn.Close()
continue
}
ps.wg.Add(1)
go func(c net.Conn) {
defer ps.wg.Done()
defer ps.removeConn(c)
ps.handleTCPConn(c)
}(conn)
}
}
func (ps *portmapServer) handleTCPConn(conn net.Conn) {
defer conn.Close()
hdr := make([]byte, 4)
for {
_ = conn.SetReadDeadline(time.Now().Add(portmapTCPIdleTimeout))
if _, err := io.ReadFull(conn, hdr); err != nil {
return
}
mark := binary.BigEndian.Uint32(hdr)
// Bit 31: last-fragment flag. Portmap messages are always single
// fragment in practice; drop the connection if we see otherwise.
if mark&(1<<31) == 0 {
return
}
recLen := mark &^ (1 << 31)
if recLen == 0 || recLen > portmapMaxRecord {
return
}
buf := make([]byte, recLen)
_ = conn.SetReadDeadline(time.Now().Add(portmapTCPIOTimeout))
if _, err := io.ReadFull(conn, buf); err != nil {
return
}
reply := ps.handleCall(buf)
if reply == nil {
continue
}
out := make([]byte, 4+len(reply))
binary.BigEndian.PutUint32(out[0:4], uint32(len(reply))|(1<<31))
copy(out[4:], reply)
_ = conn.SetWriteDeadline(time.Now().Add(portmapTCPIOTimeout))
if _, err := conn.Write(out); err != nil {
return
}
}
}
func (ps *portmapServer) serveUDP() {
buf := make([]byte, portmapMaxRecord)
for {
n, addr, err := ps.udpConn.ReadFromUDP(buf)
if err != nil {
if ps.isClosed() {
return
}
// Transient read failure: log, back off, and keep the
// responder alive instead of taking UDP portmap down.
// Wake early if Close() fires during the sleep.
glog.V(1).Infof("portmap udp read: %v", err)
select {
case <-ps.done:
return
case <-time.After(portmapRetryBackoff):
continue
}
}
reply := ps.handleCall(buf[:n])
if reply == nil {
continue
}
if _, err := ps.udpConn.WriteToUDP(reply, addr); err != nil {
glog.V(1).Infof("portmap udp write to %s: %v", addr, err)
}
}
}
// handleCall parses one RPC CALL message and returns the encoded reply, or nil
// if the call is malformed enough that we should drop it silently.
func (ps *portmapServer) handleCall(callBuf []byte) []byte {
xid, prog, vers, proc, args, err := parseRPCCall(callBuf)
if err != nil {
return nil
}
if prog != portmapProgram {
return encodeAcceptedReply(xid, rpcAcceptProgUnavail, nil)
}
if vers != portmapVersion {
// Program-version mismatch: RFC 5531 says we should return the
// accepted range; keep it simple and report 2..2.
body := make([]byte, 8)
binary.BigEndian.PutUint32(body[0:4], portmapVersion)
binary.BigEndian.PutUint32(body[4:8], portmapVersion)
return encodeAcceptedReply(xid, rpcAcceptProgMismatch, body)
}
switch proc {
case pmapProcNull:
return encodeAcceptedReply(xid, rpcAcceptSuccess, nil)
case pmapProcGetPort:
if len(args) < 16 {
return encodeAcceptedReply(xid, rpcAcceptGarbageArgs, nil)
}
q := portmapEntry{
Program: binary.BigEndian.Uint32(args[0:4]),
Version: binary.BigEndian.Uint32(args[4:8]),
Protocol: binary.BigEndian.Uint32(args[8:12]),
}
port := uint32(0)
for _, e := range ps.entries {
if e.Program == q.Program && e.Version == q.Version && e.Protocol == q.Protocol {
port = e.Port
break
}
}
body := make([]byte, 4)
binary.BigEndian.PutUint32(body, port)
return encodeAcceptedReply(xid, rpcAcceptSuccess, body)
case pmapProcDump:
// Each entry is 4-byte value_follows + 16-byte mapping = 20 bytes,
// plus a 4-byte terminator value_follows=FALSE.
body := make([]byte, 0, 20*len(ps.entries)+4)
for _, e := range ps.entries {
chunk := make([]byte, 20)
binary.BigEndian.PutUint32(chunk[0:4], 1) // value_follows = TRUE
binary.BigEndian.PutUint32(chunk[4:8], e.Program)
binary.BigEndian.PutUint32(chunk[8:12], e.Version)
binary.BigEndian.PutUint32(chunk[12:16], e.Protocol)
binary.BigEndian.PutUint32(chunk[16:20], e.Port)
body = append(body, chunk...)
}
end := make([]byte, 4) // value_follows = FALSE
body = append(body, end...)
return encodeAcceptedReply(xid, rpcAcceptSuccess, body)
case pmapProcSet, pmapProcUnset:
// Don't accept third-party registrations. bool=FALSE.
body := make([]byte, 4)
return encodeAcceptedReply(xid, rpcAcceptSuccess, body)
default:
return encodeAcceptedReply(xid, rpcAcceptProcUnavail, nil)
}
}
// parseRPCCall parses the fixed portion of an RPC CALL header and returns the
// remaining procedure arguments. It skips both opaque_auth fields (cred and
// verf) so callers get a buffer starting at the procedure arguments.
func parseRPCCall(buf []byte) (xid, prog, vers, proc uint32, args []byte, err error) {
// Minimum header: xid + msg_type + rpcvers + prog + vers + proc + 2x
// (flavor + len) = 6*4 + 2*8 = 40 bytes.
const minHeader = 40
if len(buf) < minHeader {
err = fmt.Errorf("rpc call too short: %d bytes", len(buf))
return
}
xid = binary.BigEndian.Uint32(buf[0:4])
if msgType := binary.BigEndian.Uint32(buf[4:8]); msgType != rpcMsgCall {
err = fmt.Errorf("not an rpc call: msg_type=%d", msgType)
return
}
if rpcvers := binary.BigEndian.Uint32(buf[8:12]); rpcvers != 2 {
err = fmt.Errorf("unsupported rpc version %d", rpcvers)
return
}
prog = binary.BigEndian.Uint32(buf[12:16])
vers = binary.BigEndian.Uint32(buf[16:20])
proc = binary.BigEndian.Uint32(buf[20:24])
p := 24
for i := 0; i < 2; i++ {
if len(buf) < p+8 {
err = fmt.Errorf("truncated opaque_auth at offset %d", p)
return
}
authLen := binary.BigEndian.Uint32(buf[p+4 : p+8])
// Validate before applying the XDR 4-byte padding so that
// lengths near uint32 max can't wrap to a tiny padded value.
if authLen > uint32(portmapMaxRecord) {
err = errors.New("opaque_auth length exceeds limit")
return
}
padded := (authLen + 3) &^ 3
end := uint64(p) + 8 + uint64(padded)
if end > uint64(len(buf)) {
err = fmt.Errorf("truncated opaque_auth body at offset %d (len=%d)", p, authLen)
return
}
p = int(end)
}
args = buf[p:]
return
}
// encodeAcceptedReply builds a MSG_ACCEPTED reply with the given accept_stat.
// body is the already-XDR-encoded data that follows accept_stat in the reply.
// For SUCCESS it is the procedure result; it is nil for most error
// accept_stat values (PROG_UNAVAIL, PROC_UNAVAIL, GARBAGE_ARGS) but is
// non-nil for PROG_MISMATCH, which carries a struct { uint32 low; uint32
// high; } mismatch_info range per RFC 5531 §9.
func encodeAcceptedReply(xid, acceptStat uint32, body []byte) []byte {
out := make([]byte, 24+len(body))
binary.BigEndian.PutUint32(out[0:4], xid)
binary.BigEndian.PutUint32(out[4:8], rpcMsgReply)
binary.BigEndian.PutUint32(out[8:12], rpcMsgAccepted)
// verf: AUTH_NONE, zero-length opaque
binary.BigEndian.PutUint32(out[12:16], rpcAuthNone)
binary.BigEndian.PutUint32(out[16:20], 0)
binary.BigEndian.PutUint32(out[20:24], acceptStat)
copy(out[24:], body)
return out
}
+418
View File
@@ -0,0 +1,418 @@
package nfs
import (
"bytes"
"encoding/binary"
"io"
"net"
"strconv"
"testing"
"time"
)
func buildRPCCall(t *testing.T, xid, prog, vers, proc uint32, credBody, verfBody, args []byte) []byte {
t.Helper()
pad := func(b []byte) []byte {
r := len(b) % 4
if r == 0 {
return b
}
out := make([]byte, len(b)+4-r)
copy(out, b)
return out
}
buf := new(bytes.Buffer)
write := func(v uint32) {
var b [4]byte
binary.BigEndian.PutUint32(b[:], v)
buf.Write(b[:])
}
write(xid)
write(rpcMsgCall)
write(2) // rpcvers
write(prog)
write(vers)
write(proc)
// cred
write(rpcAuthNone)
write(uint32(len(credBody)))
buf.Write(pad(credBody))
// verf
write(rpcAuthNone)
write(uint32(len(verfBody)))
buf.Write(pad(verfBody))
buf.Write(args)
return buf.Bytes()
}
func parseAcceptedReply(t *testing.T, reply []byte) (xid, acceptStat uint32, body []byte) {
t.Helper()
if len(reply) < 24 {
t.Fatalf("reply too short: %d bytes", len(reply))
}
xid = binary.BigEndian.Uint32(reply[0:4])
if mt := binary.BigEndian.Uint32(reply[4:8]); mt != rpcMsgReply {
t.Fatalf("msg_type=%d, want REPLY", mt)
}
if rs := binary.BigEndian.Uint32(reply[8:12]); rs != rpcMsgAccepted {
t.Fatalf("reply_stat=%d, want ACCEPTED", rs)
}
// verf
verfLen := binary.BigEndian.Uint32(reply[16:20])
if verfLen != 0 {
t.Fatalf("unexpected verf length %d", verfLen)
}
acceptStat = binary.BigEndian.Uint32(reply[20:24])
body = reply[24:]
return
}
func newTestPortmap() *portmapServer {
return newPortmapServer("127.0.0.1", portmapPort, 2049)
}
func TestParseRPCCall_SkipsAuth(t *testing.T) {
cred := []byte("hello") // 5 bytes -> padded to 8
verf := []byte{}
args := []byte{0x01, 0x02, 0x03, 0x04}
msg := buildRPCCall(t, 42, portmapProgram, portmapVersion, pmapProcNull, cred, verf, args)
xid, prog, vers, proc, gotArgs, err := parseRPCCall(msg)
if err != nil {
t.Fatalf("parseRPCCall: %v", err)
}
if xid != 42 || prog != portmapProgram || vers != portmapVersion || proc != pmapProcNull {
t.Fatalf("header mismatch: xid=%d prog=%d vers=%d proc=%d", xid, prog, vers, proc)
}
if !bytes.Equal(gotArgs, args) {
t.Fatalf("args mismatch: got %x want %x", gotArgs, args)
}
}
func TestParseRPCCall_RejectsReply(t *testing.T) {
buf := make([]byte, 40)
binary.BigEndian.PutUint32(buf[4:8], rpcMsgReply)
if _, _, _, _, _, err := parseRPCCall(buf); err == nil {
t.Fatal("expected error on reply-typed message")
}
}
func TestParseRPCCall_TruncatedAuth(t *testing.T) {
// Claim huge cred length but provide no body.
buf := make([]byte, 40)
binary.BigEndian.PutUint32(buf[4:8], rpcMsgCall)
binary.BigEndian.PutUint32(buf[8:12], 2)
binary.BigEndian.PutUint32(buf[28:32], 1000) // cred len
if _, _, _, _, _, err := parseRPCCall(buf); err == nil {
t.Fatal("expected error on truncated auth")
}
}
func TestHandleCall_Null(t *testing.T) {
ps := newTestPortmap()
msg := buildRPCCall(t, 7, portmapProgram, portmapVersion, pmapProcNull, nil, nil, nil)
reply := ps.handleCall(msg)
xid, acc, body := parseAcceptedReply(t, reply)
if xid != 7 || acc != rpcAcceptSuccess || len(body) != 0 {
t.Fatalf("null reply xid=%d acc=%d body=%x", xid, acc, body)
}
}
func TestHandleCall_GetPort_HitAndMiss(t *testing.T) {
ps := newTestPortmap()
buildQuery := func(prog, vers, prot uint32) []byte {
args := make([]byte, 16)
binary.BigEndian.PutUint32(args[0:4], prog)
binary.BigEndian.PutUint32(args[4:8], vers)
binary.BigEndian.PutUint32(args[8:12], prot)
// port field is ignored by the server; leave zero
return args
}
cases := []struct {
name string
prog, vers, prot uint32
wantPort uint32
}{
{"nfs-v3-tcp-hit", nfsProgram, 3, ipProtoTCP, 2049},
{"mount-v3-tcp-hit", mountProgram, 3, ipProtoTCP, 2049},
{"mount-v3-udp-hit", mountProgram, 3, ipProtoUDP, 2049},
{"mount-v1-tcp-miss", mountProgram, 1, ipProtoTCP, 0},
{"nfs-v3-udp-miss", nfsProgram, 3, ipProtoUDP, 0},
{"nlm-miss", 100021, 4, ipProtoTCP, 0},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
msg := buildRPCCall(t, 11, portmapProgram, portmapVersion, pmapProcGetPort, nil, nil, buildQuery(tc.prog, tc.vers, tc.prot))
reply := ps.handleCall(msg)
xid, acc, body := parseAcceptedReply(t, reply)
if xid != 11 {
t.Fatalf("xid=%d want 11", xid)
}
if acc != rpcAcceptSuccess {
t.Fatalf("acc=%d want SUCCESS", acc)
}
if len(body) != 4 {
t.Fatalf("getport body len=%d want 4", len(body))
}
got := binary.BigEndian.Uint32(body)
if got != tc.wantPort {
t.Fatalf("port=%d want %d", got, tc.wantPort)
}
})
}
}
func TestHandleCall_Dump(t *testing.T) {
ps := newTestPortmap()
msg := buildRPCCall(t, 13, portmapProgram, portmapVersion, pmapProcDump, nil, nil, nil)
reply := ps.handleCall(msg)
_, acc, body := parseAcceptedReply(t, reply)
if acc != rpcAcceptSuccess {
t.Fatalf("acc=%d", acc)
}
var entries []portmapEntry
p := 0
for p+4 <= len(body) {
vf := binary.BigEndian.Uint32(body[p : p+4])
p += 4
if vf == 0 {
break
}
if p+16 > len(body) {
t.Fatalf("truncated entry at %d", p)
}
entries = append(entries, portmapEntry{
Program: binary.BigEndian.Uint32(body[p : p+4]),
Version: binary.BigEndian.Uint32(body[p+4 : p+8]),
Protocol: binary.BigEndian.Uint32(body[p+8 : p+12]),
Port: binary.BigEndian.Uint32(body[p+12 : p+16]),
})
p += 16
}
if len(entries) != 3 {
t.Fatalf("got %d dump entries, want 3: %+v", len(entries), entries)
}
wantSet := map[portmapEntry]bool{
{Program: nfsProgram, Version: 3, Protocol: ipProtoTCP, Port: 2049}: false,
{Program: mountProgram, Version: 3, Protocol: ipProtoTCP, Port: 2049}: false,
{Program: mountProgram, Version: 3, Protocol: ipProtoUDP, Port: 2049}: false,
}
for _, e := range entries {
if _, ok := wantSet[e]; !ok {
t.Fatalf("unexpected dump entry %+v", e)
}
wantSet[e] = true
}
for e, seen := range wantSet {
if !seen {
t.Fatalf("missing dump entry %+v", e)
}
}
}
func TestHandleCall_UnknownProg(t *testing.T) {
ps := newTestPortmap()
msg := buildRPCCall(t, 1, 999999, 1, 0, nil, nil, nil)
reply := ps.handleCall(msg)
_, acc, _ := parseAcceptedReply(t, reply)
if acc != rpcAcceptProgUnavail {
t.Fatalf("acc=%d want PROG_UNAVAIL", acc)
}
}
func TestHandleCall_VersionMismatch(t *testing.T) {
ps := newTestPortmap()
msg := buildRPCCall(t, 1, portmapProgram, 42, pmapProcNull, nil, nil, nil)
reply := ps.handleCall(msg)
_, acc, body := parseAcceptedReply(t, reply)
if acc != rpcAcceptProgMismatch {
t.Fatalf("acc=%d want PROG_MISMATCH", acc)
}
if len(body) != 8 {
t.Fatalf("mismatch body len=%d want 8", len(body))
}
lo := binary.BigEndian.Uint32(body[0:4])
hi := binary.BigEndian.Uint32(body[4:8])
if lo != portmapVersion || hi != portmapVersion {
t.Fatalf("mismatch range lo=%d hi=%d", lo, hi)
}
}
func TestHandleCall_UnknownProc(t *testing.T) {
ps := newTestPortmap()
msg := buildRPCCall(t, 1, portmapProgram, portmapVersion, 42, nil, nil, nil)
reply := ps.handleCall(msg)
_, acc, _ := parseAcceptedReply(t, reply)
if acc != rpcAcceptProcUnavail {
t.Fatalf("acc=%d want PROC_UNAVAIL", acc)
}
}
func TestHandleCall_SetRefused(t *testing.T) {
ps := newTestPortmap()
args := make([]byte, 16) // mapping struct
msg := buildRPCCall(t, 1, portmapProgram, portmapVersion, pmapProcSet, nil, nil, args)
reply := ps.handleCall(msg)
_, acc, body := parseAcceptedReply(t, reply)
if acc != rpcAcceptSuccess {
t.Fatalf("acc=%d", acc)
}
if len(body) != 4 || binary.BigEndian.Uint32(body) != 0 {
t.Fatalf("PMAP_SET must return FALSE, got %x", body)
}
}
// pickFreePort asks the OS for an unused high port by opening and closing a
// listener on it. Used so the end-to-end tests can run in parallel without
// stepping on the privileged default port 111.
func pickFreePort(t *testing.T) int {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
defer ln.Close()
return ln.Addr().(*net.TCPAddr).Port
}
func TestPortmapServer_UDPGetPort(t *testing.T) {
port := pickFreePort(t)
ps := newPortmapServer("127.0.0.1", port, 2049)
if err := ps.Start(); err != nil {
t.Fatalf("start: %v", err)
}
t.Cleanup(func() { _ = ps.Close() })
args := make([]byte, 16)
binary.BigEndian.PutUint32(args[0:4], nfsProgram)
binary.BigEndian.PutUint32(args[4:8], 3)
binary.BigEndian.PutUint32(args[8:12], ipProtoTCP)
msg := buildRPCCall(t, 99, portmapProgram, portmapVersion, pmapProcGetPort, nil, nil, args)
conn, err := net.Dial("udp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)))
if err != nil {
t.Fatalf("dial udp: %v", err)
}
defer conn.Close()
_ = conn.SetDeadline(time.Now().Add(2 * time.Second))
if _, err := conn.Write(msg); err != nil {
t.Fatalf("write: %v", err)
}
buf := make([]byte, 4096)
n, err := conn.Read(buf)
if err != nil {
t.Fatalf("read: %v", err)
}
xid, acc, body := parseAcceptedReply(t, buf[:n])
if xid != 99 || acc != rpcAcceptSuccess || len(body) != 4 {
t.Fatalf("bad reply xid=%d acc=%d body=%x", xid, acc, body)
}
if got := binary.BigEndian.Uint32(body); got != 2049 {
t.Fatalf("udp getport port=%d want 2049", got)
}
}
func TestPortmapServer_CloseEvictsIdleTCPConn(t *testing.T) {
port := pickFreePort(t)
ps := newPortmapServer("127.0.0.1", port, 2049)
if err := ps.Start(); err != nil {
t.Fatalf("start: %v", err)
}
conn, err := net.Dial("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)))
if err != nil {
_ = ps.Close()
t.Fatalf("dial: %v", err)
}
defer conn.Close()
// Issue one call and read its reply so the server-side connection is
// definitely registered before we trigger shutdown.
msg := buildRPCCall(t, 1, portmapProgram, portmapVersion, pmapProcNull, nil, nil, nil)
var mark [4]byte
binary.BigEndian.PutUint32(mark[:], uint32(len(msg))|(1<<31))
_ = conn.SetDeadline(time.Now().Add(2 * time.Second))
if _, err := conn.Write(mark[:]); err != nil {
t.Fatalf("write mark: %v", err)
}
if _, err := conn.Write(msg); err != nil {
t.Fatalf("write msg: %v", err)
}
if _, err := io.ReadFull(conn, mark[:]); err != nil {
t.Fatalf("read mark: %v", err)
}
rlen := binary.BigEndian.Uint32(mark[:]) &^ (1 << 31)
if _, err := io.ReadFull(conn, make([]byte, rlen)); err != nil {
t.Fatalf("read body: %v", err)
}
// Close must return long before the TCP idle deadline (30s) — in
// other words, the server must actively close the idle conn rather
// than wait for the deadline or for the client to disconnect.
done := make(chan error, 1)
go func() { done <- ps.Close() }()
select {
case err := <-done:
if err != nil {
t.Fatalf("Close: %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("Close did not return within 2s; in-flight conn not evicted")
}
_ = conn.SetReadDeadline(time.Now().Add(1 * time.Second))
if _, err := conn.Read(make([]byte, 4)); err == nil {
t.Fatal("expected read error on client conn after server Close")
}
}
func TestPortmapServer_TCPGetPort(t *testing.T) {
port := pickFreePort(t)
ps := newPortmapServer("127.0.0.1", port, 2049)
if err := ps.Start(); err != nil {
t.Fatalf("start: %v", err)
}
t.Cleanup(func() { _ = ps.Close() })
args := make([]byte, 16)
binary.BigEndian.PutUint32(args[0:4], mountProgram)
binary.BigEndian.PutUint32(args[4:8], 3)
binary.BigEndian.PutUint32(args[8:12], ipProtoTCP)
msg := buildRPCCall(t, 123, portmapProgram, portmapVersion, pmapProcGetPort, nil, nil, args)
conn, err := net.Dial("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)))
if err != nil {
t.Fatalf("dial tcp: %v", err)
}
defer conn.Close()
_ = conn.SetDeadline(time.Now().Add(2 * time.Second))
// record mark: last-fragment bit + length
var mark [4]byte
binary.BigEndian.PutUint32(mark[:], uint32(len(msg))|(1<<31))
if _, err := conn.Write(mark[:]); err != nil {
t.Fatalf("write mark: %v", err)
}
if _, err := conn.Write(msg); err != nil {
t.Fatalf("write msg: %v", err)
}
var rmark [4]byte
if _, err := io.ReadFull(conn, rmark[:]); err != nil {
t.Fatalf("read mark: %v", err)
}
rlen := binary.BigEndian.Uint32(rmark[:]) &^ (1 << 31)
buf := make([]byte, rlen)
if _, err := io.ReadFull(conn, buf); err != nil {
t.Fatalf("read body: %v", err)
}
xid, acc, body := parseAcceptedReply(t, buf)
if xid != 123 || acc != rpcAcceptSuccess || len(body) != 4 {
t.Fatalf("bad reply xid=%d acc=%d body=%x", xid, acc, body)
}
if got := binary.BigEndian.Uint32(body); got != 2049 {
t.Fatalf("tcp getport port=%d want 2049", got)
}
}
+377
View File
@@ -0,0 +1,377 @@
package nfs
import (
"bufio"
"encoding/binary"
"errors"
"io"
"net"
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
)
// The upstream willscott/go-nfs library dispatches RPC calls by (program,
// procedure) only — it does not validate the RPC program version. That means
// a Linux client speaking NFSv4 (program 100003 vers 4) lands on the same
// handler map as NFSv3: proc=1 routes to NFSv3 SETATTR, which parses the
// NFSv4 COMPOUND args as if they were SETATTR3args and writes a malformed
// reply. The client cannot decode that reply, the kernel returns
// EPROTONOSUPPORT, and mount.nfs prints "requested NFS version or transport
// protocol is not supported" without ever falling back to v3.
//
// The default Linux mount.nfs path is to try NFSv4 first, so this affects
// every plain `mount -t nfs <host>:<export> /mnt` against a `weed nfs`
// deployment. To make the v4→v3 fallback work, we wrap the listener so the
// first RPC frame on each new TCP connection is inspected: if the program is
// NFS or MOUNT and the version is not 3, we synthesize a PROG_MISMATCH reply
// (with the supported version range 3..3) directly to the socket and close
// the connection. The client then retries with v3 and proceeds normally.
//
// Clients keep the same program/version for the lifetime of a TCP connection
// in practice, so we only need to check the first frame; subsequent frames
// flow through to go-nfs unchanged. This avoids vendoring go-nfs while still
// producing protocol-correct rejections.
// RPC numeric constants used here (rpcMsgCall, rpcMsgReply, rpcMsgAccepted,
// rpcAcceptProgMismatch, rpcAuthNone, nfsProgram, mountProgram) are defined
// alongside the portmap responder in portmap.go to keep one source of truth
// per package.
const (
// rpcVersionFilterPeekTimeout bounds how long we wait for the first frame
// header on a new connection before giving up and letting go-nfs handle
// the (possibly half-open) socket.
rpcVersionFilterPeekTimeout = 10 * time.Second
// peeked length: 4-byte fragment marker + 24 bytes of fixed RPC header
// (xid + msg_type + rpcvers + prog + vers + proc).
rpcVersionFilterPeekLen = 28
// rpcVersionFilterAcceptBackoff is how long the accept loop sleeps
// after a transient Accept() error (EMFILE, EAGAIN, ECONNABORTED,
// etc.) before retrying. Mirrors portmapRetryBackoff in portmap.go so
// both NFS-listening goroutines back off identically under host
// resource pressure.
rpcVersionFilterAcceptBackoff = 50 * time.Millisecond
supportedNFSVer = 3
)
// versionFilterListener moves the per-connection RPC peek off the
// Listener.Accept() critical path. Peeking inline would let one slow or idle
// client (or a TCP three-way handshake without any RPC payload) hold
// rpcVersionFilterPeekTimeout — i.e. up to 10 seconds — of head-of-line
// blocking against every other connect, since gonfs.Serve only calls Accept
// serially. Instead, a background goroutine runs the inner Accept() loop and
// hands each raw conn to its own short-lived goroutine that does the peek;
// validated conns are sent on acceptCh and the wrapper's Accept() reads from
// that channel. Rejected conns never reach the channel — PROG_MISMATCH is
// already on the wire by the time the per-conn goroutine returns.
type versionFilterListener struct {
inner net.Listener
acceptCh chan net.Conn
// closed is signalled either by Close() or by the accept loop after the
// inner listener returns a terminal error. After it fires Accept() will
// stop blocking and return acceptErr (or net.ErrClosed if none).
closed chan struct{}
closeOnce sync.Once
mu sync.Mutex
acceptErr error
// inFlight tracks raw (pre-peek) conns that are currently in
// handleConn so Close() can break their Peek() deadline by closing
// them, instead of waiting up to rpcVersionFilterPeekTimeout per
// idle client for the timeout to fire on its own.
inFlight map[net.Conn]struct{}
startOnce sync.Once
wg sync.WaitGroup
}
func newVersionFilterListener(inner net.Listener) net.Listener {
return &versionFilterListener{
inner: inner,
acceptCh: make(chan net.Conn),
closed: make(chan struct{}),
}
}
// start lazily kicks off the background accept loop the first time someone
// calls Accept(). This matches the behaviour of the embedded-listener form we
// replaced — no goroutines spawn just from constructing the wrapper.
func (l *versionFilterListener) start() {
l.startOnce.Do(func() {
l.wg.Add(1)
go l.acceptLoop()
})
}
func (l *versionFilterListener) Accept() (net.Conn, error) {
l.start()
select {
case c := <-l.acceptCh:
return c, nil
case <-l.closed:
return nil, l.terminalErr()
}
}
func (l *versionFilterListener) Close() error {
l.signalClose()
err := l.inner.Close()
// Eagerly close any raw conns currently blocked in filterFirstRPCFrame's
// Peek so handleConn returns promptly. Without this, an idle client
// (TCP handshake without any RPC payload) holds Close() up to
// rpcVersionFilterPeekTimeout — 10s of stop-the-world per such conn.
l.evictInFlight()
l.wg.Wait()
return err
}
func (l *versionFilterListener) Addr() net.Addr {
return l.inner.Addr()
}
func (l *versionFilterListener) signalClose() {
l.closeOnce.Do(func() {
close(l.closed)
})
}
func (l *versionFilterListener) terminalErr() error {
l.mu.Lock()
defer l.mu.Unlock()
if l.acceptErr != nil {
return l.acceptErr
}
return net.ErrClosed
}
// trackInFlight records a raw conn that's about to be peeked, so Close()
// can break its Peek() deadline by closing it. Returns false if shutdown
// has already started; the caller must close the conn and bail.
func (l *versionFilterListener) trackInFlight(c net.Conn) bool {
l.mu.Lock()
defer l.mu.Unlock()
select {
case <-l.closed:
return false
default:
}
if l.inFlight == nil {
l.inFlight = make(map[net.Conn]struct{})
}
l.inFlight[c] = struct{}{}
return true
}
func (l *versionFilterListener) untrackInFlight(c net.Conn) {
l.mu.Lock()
defer l.mu.Unlock()
delete(l.inFlight, c)
}
// evictInFlight closes every conn currently in handleConn so their
// in-flight Peek() returns immediately. delete(nil-map, k) is a no-op,
// so handleConn's deferred untrackInFlight is safe even after we've
// nilled the map here.
func (l *versionFilterListener) evictInFlight() {
l.mu.Lock()
conns := l.inFlight
l.inFlight = nil
l.mu.Unlock()
for c := range conns {
_ = c.Close()
}
}
func (l *versionFilterListener) acceptLoop() {
defer l.wg.Done()
defer l.signalClose()
for {
conn, err := l.inner.Accept()
if err != nil {
// Permanent: the inner listener has been closed (Close(),
// shutdown, or an unrecoverable error from the OS). Surface
// the error to Accept() and stop.
if errors.Is(err, net.ErrClosed) {
l.mu.Lock()
if l.acceptErr == nil {
l.acceptErr = err
}
l.mu.Unlock()
return
}
// Transient (EMFILE, EAGAIN, ECONNABORTED on accept,
// timeouts if a deadline is ever set): treating these as
// terminal would tear the whole NFS server down on a
// resource blip. Back off briefly and retry, mirroring the
// pattern in portmap.go's serveTCP.
glog.V(1).Infof("nfs version filter: transient accept error: %v", err)
select {
case <-l.closed:
return
case <-time.After(rpcVersionFilterAcceptBackoff):
continue
}
}
l.wg.Add(1)
go l.handleConn(conn)
}
}
// handleConn runs the version peek for a single accepted conn. Because each
// conn has its own goroutine, a slow client only blocks itself; concurrent
// peeks proceed in parallel up to whatever the runtime can schedule. If
// Close() fires before the peek completes we drop the validated conn so we
// don't leak a socket past shutdown.
func (l *versionFilterListener) handleConn(conn net.Conn) {
defer l.wg.Done()
if !l.trackInFlight(conn) {
// Shutdown beat us: don't start the Peek that we'd then
// have to break, just close the raw conn.
_ = conn.Close()
return
}
defer l.untrackInFlight(conn)
wrapped, accepted := filterFirstRPCFrame(conn)
if !accepted {
// Already replied with PROG_MISMATCH and closed conn.
return
}
select {
case l.acceptCh <- wrapped:
case <-l.closed:
_ = wrapped.Close()
}
}
// peekedConn returns the bytes that filterFirstRPCFrame already buffered when
// it peeked the first RPC header, then transparently reads from the
// underlying connection. Writes go straight to the socket; the bufio reader
// only buffers the read side.
type peekedConn struct {
net.Conn
reader io.Reader
}
func (c *peekedConn) Read(p []byte) (int, error) {
return c.reader.Read(p)
}
// filterFirstRPCFrame inspects the first RPC frame on conn and decides whether
// to pass it through to go-nfs. Returns (wrappedConn, true) if the frame is
// for a supported (program, version) — including programs we don't recognize,
// since go-nfs handles its own PROG_UNAVAIL response. Returns (nil, false) if
// we already replied with PROG_MISMATCH and closed conn.
//
// On peek failure (early close, deadline) we pass the connection through:
// returning an error here would silently drop legitimate clients on a flaky
// link, and go-nfs has its own per-frame error handling.
func filterFirstRPCFrame(conn net.Conn) (net.Conn, bool) {
r := bufio.NewReader(conn)
deadlineErr := conn.SetReadDeadline(time.Now().Add(rpcVersionFilterPeekTimeout))
hdr, peekErr := r.Peek(rpcVersionFilterPeekLen)
// Always clear the deadline before returning to go-nfs; failing to do so
// would make every subsequent Read() time out at the same instant.
if deadlineErr == nil {
_ = conn.SetReadDeadline(time.Time{})
}
if peekErr != nil {
return &peekedConn{Conn: conn, reader: r}, true
}
fragMark := binary.BigEndian.Uint32(hdr[0:4])
if fragMark&(1<<31) == 0 {
// Multi-fragment record: portmap-style filtering of the first frame
// would need reassembly. Fall through to go-nfs which handles this.
return &peekedConn{Conn: conn, reader: r}, true
}
// Peek(28) can read across record boundaries — the first fragment may
// be shorter than the fixed RPC CALL header (24 bytes after the marker)
// with the remaining bytes belonging to the *next* RPC. Indexing into
// hdr[16:24] without first checking the fragment length would parse
// fields from a different RPC and either spuriously reject or pass it.
// Pass through if the first fragment can't possibly hold a full header
// and let go-nfs surface the framing error.
if fragLen := fragMark &^ uint32(1<<31); fragLen < 24 {
return &peekedConn{Conn: conn, reader: r}, true
}
xid := binary.BigEndian.Uint32(hdr[4:8])
if msgType := binary.BigEndian.Uint32(hdr[8:12]); msgType != rpcMsgCall {
// Not a CALL — odd, but pass through.
return &peekedConn{Conn: conn, reader: r}, true
}
if rpcVers := binary.BigEndian.Uint32(hdr[12:16]); rpcVers != 2 {
// ONC RPC v2 is the only version we and go-nfs speak; if the
// rpcvers field is anything else the rest of the header is
// untrusted (could be a non-RPC protocol that happens to share
// the port, or simply garbled traffic). Don't synthesize a
// PROG_MISMATCH that lies about supporting NFS — pass it
// through and let go-nfs / RFC 5531 §9 RPC_MISMATCH handling
// in the upstream library do the right thing.
return &peekedConn{Conn: conn, reader: r}, true
}
prog := binary.BigEndian.Uint32(hdr[16:20])
vers := binary.BigEndian.Uint32(hdr[20:24])
switch prog {
case nfsProgram, mountProgram:
default:
// Unknown program: let go-nfs reply PROG_UNAVAIL itself.
return &peekedConn{Conn: conn, reader: r}, true
}
if vers == supportedNFSVer {
return &peekedConn{Conn: conn, reader: r}, true
}
glog.V(1).Infof("nfs: rejecting client %s with PROG_MISMATCH: prog=%d vers=%d (supported=%d)",
conn.RemoteAddr(), prog, vers, supportedNFSVer)
if err := writeProgMismatchTCP(conn, xid, supportedNFSVer, supportedNFSVer); err != nil {
glog.V(1).Infof("nfs: write PROG_MISMATCH to %s: %v", conn.RemoteAddr(), err)
}
_ = conn.Close()
return nil, false
}
// writeProgMismatchTCP encodes a single-frame TCP RPC reply carrying
// MSG_ACCEPTED + PROG_MISMATCH along with the supported version range, per
// RFC 5531 section 9. The frame layout is:
//
// uint32 fragment_header (last-fragment | length)
// uint32 xid
// uint32 msg_type=REPLY(1)
// uint32 reply_stat=MSG_ACCEPTED(0)
// uint32 verf_flavor=AUTH_NONE(0)
// uint32 verf_len=0
// uint32 accept_stat=PROG_MISMATCH(2)
// uint32 low
// uint32 high
const progMismatchBodyLen = 32
func writeProgMismatchTCP(w io.Writer, xid, low, high uint32) error {
out := make([]byte, 4+progMismatchBodyLen)
binary.BigEndian.PutUint32(out[0:4], uint32(progMismatchBodyLen)|(1<<31))
binary.BigEndian.PutUint32(out[4:8], xid)
binary.BigEndian.PutUint32(out[8:12], rpcMsgReply)
binary.BigEndian.PutUint32(out[12:16], rpcMsgAccepted)
binary.BigEndian.PutUint32(out[16:20], rpcAuthNone)
binary.BigEndian.PutUint32(out[20:24], 0) // verf opaque length (always zero for AUTH_NONE)
binary.BigEndian.PutUint32(out[24:28], rpcAcceptProgMismatch)
binary.BigEndian.PutUint32(out[28:32], low)
binary.BigEndian.PutUint32(out[32:36], high)
_, err := w.Write(out)
return err
}
+561
View File
@@ -0,0 +1,561 @@
package nfs
import (
"encoding/binary"
"errors"
"io"
"net"
"sync"
"testing"
"time"
)
// buildRPCCallFrame constructs a single TCP-framed RPC CALL header without
// procedure arguments — enough for the version filter to decide whether to
// reject the connection. The frame layout matches RFC 5531 (Open Network
// Computing RPC v2): a 4-byte fragment marker (last-fragment bit set on a
// 40-byte body) followed by xid + msg_type=CALL + rpcvers=2 + prog + vers +
// proc + two empty AUTH_NONE opaque_auth structs.
func buildRPCCallFrame(xid, prog, vers, proc uint32) []byte {
const bodyLen = 40
frame := make([]byte, 4+bodyLen)
binary.BigEndian.PutUint32(frame[0:4], uint32(bodyLen)|(1<<31))
binary.BigEndian.PutUint32(frame[4:8], xid)
binary.BigEndian.PutUint32(frame[8:12], 0) // msg_type CALL
binary.BigEndian.PutUint32(frame[12:16], 2)
binary.BigEndian.PutUint32(frame[16:20], prog)
binary.BigEndian.PutUint32(frame[20:24], vers)
binary.BigEndian.PutUint32(frame[24:28], proc)
// cred + verf both AUTH_NONE / length 0
return frame
}
// readPROGMismatchReply parses a TCP-framed PROG_MISMATCH reply produced by
// writeProgMismatchTCP and returns the xid plus the supported (low, high)
// version range advertised by the server.
func readPROGMismatchReply(t *testing.T, conn net.Conn) (xid, low, high uint32) {
t.Helper()
_ = conn.SetReadDeadline(time.Now().Add(2 * time.Second))
buf := make([]byte, 4+progMismatchBodyLen)
n, err := io.ReadFull(conn, buf)
if err != nil {
t.Fatalf("read reply: %v (got %d bytes)", err, n)
}
frag := binary.BigEndian.Uint32(buf[0:4])
if frag&(1<<31) == 0 {
t.Fatalf("reply frame missing last-fragment bit: %x", frag)
}
if got := frag &^ (1 << 31); got != progMismatchBodyLen {
t.Fatalf("reply body length=%d want %d", got, progMismatchBodyLen)
}
xid = binary.BigEndian.Uint32(buf[4:8])
if mt := binary.BigEndian.Uint32(buf[8:12]); mt != 1 {
t.Fatalf("reply msg_type=%d want REPLY(1)", mt)
}
if rs := binary.BigEndian.Uint32(buf[12:16]); rs != 0 {
t.Fatalf("reply reply_stat=%d want MSG_ACCEPTED(0)", rs)
}
if as := binary.BigEndian.Uint32(buf[24:28]); as != 2 {
t.Fatalf("reply accept_stat=%d want PROG_MISMATCH(2)", as)
}
low = binary.BigEndian.Uint32(buf[28:32])
high = binary.BigEndian.Uint32(buf[32:36])
return
}
func TestVersionFilterRejectsNFSv4WithProgMismatch(t *testing.T) {
innerListener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer innerListener.Close()
listener := newVersionFilterListener(innerListener)
// In a real server, accepted conns are passed to go-nfs. We just need
// to drive Accept() so the filter runs; the test never sees a wrapped
// conn because the v4 frame is rejected.
accepted := make(chan net.Conn, 1)
go func() {
for {
c, aerr := listener.Accept()
if aerr != nil {
return
}
accepted <- c
}
}()
conn, err := net.Dial("tcp", innerListener.Addr().String())
if err != nil {
t.Fatalf("dial: %v", err)
}
defer conn.Close()
// NFSv4 NULL: the first probe Linux mount.nfs sends when trying v4.
if _, err := conn.Write(buildRPCCallFrame(0xdeadbeef, nfsProgram, 4, 0)); err != nil {
t.Fatalf("write: %v", err)
}
xid, low, high := readPROGMismatchReply(t, conn)
const wantXID uint32 = 0xdeadbeef
if xid != wantXID {
t.Errorf("xid=%x want %x", xid, wantXID)
}
if low != supportedNFSVer || high != supportedNFSVer {
t.Errorf("supported range=(%d,%d) want (%d,%d)", low, high, supportedNFSVer, supportedNFSVer)
}
// Filter must close the connection after replying so the client knows
// not to send another RPC on this socket. Insist on io.EOF specifically:
// "any error" would let a stuck (but still-open) connection pass this
// check via a deadline timeout, which is exactly the regression we want
// to catch.
_ = conn.SetReadDeadline(time.Now().Add(time.Second))
one := make([]byte, 1)
n, err := conn.Read(one)
switch {
case err == nil:
t.Errorf("expected EOF after PROG_MISMATCH but read returned %d bytes", n)
case !errors.Is(err, io.EOF):
t.Errorf("expected io.EOF after PROG_MISMATCH, got %v (likely a regression where the filter replies but does not close)", err)
}
select {
case c := <-accepted:
c.Close()
t.Error("rejected connection should not be returned to caller")
case <-time.After(100 * time.Millisecond):
}
}
func TestVersionFilterRejectsMOUNTv4WithProgMismatch(t *testing.T) {
innerListener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer innerListener.Close()
listener := newVersionFilterListener(innerListener)
go func() {
for {
c, aerr := listener.Accept()
if aerr != nil {
return
}
c.Close()
}
}()
conn, err := net.Dial("tcp", innerListener.Addr().String())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
if _, err := conn.Write(buildRPCCallFrame(42, mountProgram, 4, 0)); err != nil {
t.Fatal(err)
}
xid, low, high := readPROGMismatchReply(t, conn)
if xid != 42 {
t.Errorf("xid=%d want 42", xid)
}
if low != supportedNFSVer || high != supportedNFSVer {
t.Errorf("supported range=(%d,%d) want (3,3)", low, high)
}
}
func TestVersionFilterPassesThroughNFSv3(t *testing.T) {
innerListener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer innerListener.Close()
listener := newVersionFilterListener(innerListener)
got := make(chan []byte, 1)
go func() {
c, aerr := listener.Accept()
if aerr != nil {
return
}
defer c.Close()
buf := make([]byte, 44)
_, rerr := io.ReadFull(c, buf)
if rerr != nil {
return
}
got <- buf
}()
conn, err := net.Dial("tcp", innerListener.Addr().String())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
frame := buildRPCCallFrame(7, nfsProgram, 3, 0)
if _, err := conn.Write(frame); err != nil {
t.Fatal(err)
}
select {
case received := <-got:
if string(received) != string(frame) {
t.Error("v3 frame was modified or partially consumed by filter")
}
case <-time.After(2 * time.Second):
t.Fatal("v3 frame not delivered to inner accept handler")
}
}
func TestVersionFilterPassesThroughUnknownProgram(t *testing.T) {
// The filter should only police NFS / MOUNT versions; other programs
// reach go-nfs which already responds PROG_UNAVAIL itself. Otherwise
// adding a new program (e.g. NLM) here would require updating the
// filter, which would defeat the point of using it as a thin shim.
innerListener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer innerListener.Close()
listener := newVersionFilterListener(innerListener)
delivered := make(chan struct{}, 1)
go func() {
c, aerr := listener.Accept()
if aerr != nil {
return
}
defer c.Close()
buf := make([]byte, 44)
if _, rerr := io.ReadFull(c, buf); rerr == nil {
delivered <- struct{}{}
}
}()
conn, err := net.Dial("tcp", innerListener.Addr().String())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
// Program 100021 is NLM, which weed nfs doesn't run; let go-nfs handle
// the unsupported-program reply.
if _, err := conn.Write(buildRPCCallFrame(99, 100021, 4, 0)); err != nil {
t.Fatal(err)
}
select {
case <-delivered:
case <-time.After(2 * time.Second):
t.Fatal("unknown-program frame should pass through filter")
}
}
// transientErrListener wraps a real net.Listener but injects a configurable
// number of transient Accept() errors before delegating. It exists only to
// regression-test the version filter's transient-retry behaviour without
// having to provoke real EMFILE conditions on the host.
type transientErrListener struct {
inner net.Listener
mu sync.Mutex
remaining int
}
type fakeAcceptError struct{}
func (fakeAcceptError) Error() string { return "fake transient accept error" }
func (l *transientErrListener) Accept() (net.Conn, error) {
l.mu.Lock()
if l.remaining > 0 {
l.remaining--
l.mu.Unlock()
return nil, fakeAcceptError{}
}
l.mu.Unlock()
return l.inner.Accept()
}
func (l *transientErrListener) Close() error { return l.inner.Close() }
func (l *transientErrListener) Addr() net.Addr { return l.inner.Addr() }
func TestVersionFilterRetriesTransientAcceptErrors(t *testing.T) {
// Regression test: previously the accept loop exited on any error
// from the inner listener, which meant a single transient EMFILE /
// EAGAIN under host resource pressure would tear the entire NFS
// server down. Inject a few fake transient errors and assert the
// filter still delivers the next real connection.
innerListener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer innerListener.Close()
injected := &transientErrListener{inner: innerListener, remaining: 3}
listener := newVersionFilterListener(injected)
delivered := make(chan struct{}, 1)
go func() {
c, aerr := listener.Accept()
if aerr != nil {
return
}
defer c.Close()
buf := make([]byte, 44)
if _, rerr := io.ReadFull(c, buf); rerr == nil {
delivered <- struct{}{}
}
}()
conn, err := net.Dial("tcp", innerListener.Addr().String())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
if _, err := conn.Write(buildRPCCallFrame(1, nfsProgram, 3, 0)); err != nil {
t.Fatal(err)
}
// 3 transient errors × ~50ms backoff plus normal accept latency. Allow
// a generous bound so flakes on slow CI don't surface here, but still
// tight enough to catch a regression to "any error is terminal".
select {
case <-delivered:
case <-time.After(2 * time.Second):
t.Fatal("filter did not retry transient Accept() errors and recover")
}
}
func TestVersionFilterCloseReturnsPromptlyWithIdlePeekConns(t *testing.T) {
// Regression test: Close() used to wait on every handleConn goroutine
// via wg.Wait, but those goroutines could be stuck in
// filterFirstRPCFrame's Peek() until rpcVersionFilterPeekTimeout (10s)
// fired. An idle client that completed a TCP handshake but never sent
// a byte would stretch shutdown by up to that timeout per conn.
// Close() now eagerly closes any tracked in-flight raw conns, which
// forces Peek() to return immediately and lets handleConn finish.
//
// Black-box test: only observes Close() latency. With the regression
// in place Close() would block ~10s; with the fix it returns in well
// under a second.
innerListener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
listener := newVersionFilterListener(innerListener)
// Drive Accept once so the background accept loop is running.
go func() { _, _ = listener.Accept() }()
const idleConns = 4
dialed := make([]net.Conn, 0, idleConns)
defer func() {
for _, c := range dialed {
_ = c.Close()
}
}()
for i := 0; i < idleConns; i++ {
c, err := net.Dial("tcp", innerListener.Addr().String())
if err != nil {
t.Fatal(err)
}
dialed = append(dialed, c)
}
// Give handleConn time to invoke Peek for each idle conn — without
// this the test could race ahead and Close() while no goroutine has
// actually started peeking yet, masking the regression.
time.Sleep(100 * time.Millisecond)
// Close() must finish in well under rpcVersionFilterPeekTimeout (10s).
// 2s is a generous bound that still clearly distinguishes "broke the
// peek by closing the conn" from "waited for the peek deadline".
start := time.Now()
if err := listener.Close(); err != nil {
t.Errorf("Close: %v", err)
}
elapsed := time.Since(start)
if elapsed > 2*time.Second {
t.Errorf("Close took %v with %d idle pre-peek conns; should be sub-second once they're forcibly closed", elapsed, idleConns)
}
}
func TestVersionFilterPassesThroughNonV2RPC(t *testing.T) {
// Anything that isn't ONC RPC v2 isn't ours to classify — even if the
// bytes at hdr[16:24] happen to look like nfsProgram + vers=4, we
// shouldn't synthesize a PROG_MISMATCH advertising NFSv3 support for
// what could be a completely different protocol sharing the port.
innerListener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer innerListener.Close()
listener := newVersionFilterListener(innerListener)
delivered := make(chan struct{}, 1)
go func() {
c, aerr := listener.Accept()
if aerr != nil {
return
}
defer c.Close()
buf := make([]byte, 44)
if _, rerr := io.ReadFull(c, buf); rerr == nil {
delivered <- struct{}{}
}
}()
conn, err := net.Dial("tcp", innerListener.Addr().String())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
// Build a CALL frame, then overwrite the rpcvers field with 99.
// Without the rpcvers guard the filter would still parse prog=NFS,
// vers=4 from the same buffer and reject with PROG_MISMATCH.
frame := buildRPCCallFrame(0xfeedbeef, nfsProgram, 4, 0)
binary.BigEndian.PutUint32(frame[12:16], 99) // bogus rpcvers
if _, err := conn.Write(frame); err != nil {
t.Fatal(err)
}
// Try to read a PROG_MISMATCH reply with a short deadline — none
// should arrive because the filter shouldn't pretend to know what
// this protocol is.
_ = conn.SetReadDeadline(time.Now().Add(500 * time.Millisecond))
hdr := make([]byte, 4)
if n, err := io.ReadFull(conn, hdr); err == nil && n == 4 {
if got := binary.BigEndian.Uint32(hdr); got == uint32(progMismatchBodyLen)|(1<<31) {
t.Fatal("filter sent PROG_MISMATCH for a non-v2 RPC frame")
}
}
// And the connection should reach the inner accept handler.
select {
case <-delivered:
case <-time.After(2 * time.Second):
t.Fatal("non-v2 RPC frame should pass through filter to inner accept")
}
}
func TestVersionFilterIgnoresShortFirstFragment(t *testing.T) {
// Peek(28) can read past the first fragment's body when the body is
// shorter than the 24-byte fixed RPC CALL header. Without a length
// check, the prog/vers fields would be sourced from bytes belonging to
// the *next* RPC (or a syntactic accident), and the filter could
// spuriously reject the connection. Send a 12-byte first fragment whose
// trailing peek-region bytes look like an NFSv4 CALL header, and assert
// the filter does NOT emit a PROG_MISMATCH reply.
innerListener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer innerListener.Close()
listener := newVersionFilterListener(innerListener)
go func() {
for {
c, aerr := listener.Accept()
if aerr != nil {
return
}
c.Close()
}
}()
conn, err := net.Dial("tcp", innerListener.Addr().String())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
const shortBody = 12
payload := make([]byte, 4+24)
binary.BigEndian.PutUint32(payload[0:4], shortBody|(1<<31)) // last-fragment, body=12
// Bytes 4..16 are the actual fragment body (12 bytes — too short for a
// CALL header; the filter must not look at them as one).
// Bytes 16..28 sit past the fragment in the peek window. If we were to
// (incorrectly) read prog/vers from hdr[16:24], we'd see NFS+v4 here.
binary.BigEndian.PutUint32(payload[16:20], nfsProgram)
binary.BigEndian.PutUint32(payload[20:24], 4)
if _, err := conn.Write(payload); err != nil {
t.Fatal(err)
}
// If the filter erroneously rejected, it would send a 36-byte TCP RPC
// reply (4-byte frag marker + 32-byte PROG_MISMATCH body) within ms.
// Wait briefly and assert nothing PROG_MISMATCH-shaped came back.
_ = conn.SetReadDeadline(time.Now().Add(500 * time.Millisecond))
hdr := make([]byte, 4)
n, err := io.ReadFull(conn, hdr)
if err == nil && n == 4 {
if got := binary.BigEndian.Uint32(hdr); got == uint32(progMismatchBodyLen)|(1<<31) {
t.Fatal("filter sent PROG_MISMATCH on a short fragment whose trailing peek bytes only superficially resembled a v4 call")
}
}
// Anything else (timeout, EOF, or unrelated bytes) is fine — we only
// care that the filter did NOT misclassify the short fragment.
}
func TestVersionFilterDoesNotHeadOfLineBlockOnSlowConn(t *testing.T) {
// Regression test: the previous implementation peeked the first RPC
// frame inline in Accept(), so an idle TCP-only connect would block
// every later Accept() call for up to rpcVersionFilterPeekTimeout.
// The peek now runs in a per-conn goroutine; a fast follow-up connect
// must reach the inner accept handler well before the slow conn's
// peek deadline.
innerListener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer innerListener.Close()
listener := newVersionFilterListener(innerListener)
delivered := make(chan struct{}, 1)
go func() {
c, aerr := listener.Accept()
if aerr != nil {
return
}
defer c.Close()
buf := make([]byte, 44)
if _, rerr := io.ReadFull(c, buf); rerr == nil {
delivered <- struct{}{}
}
}()
// Slow client: connect, never write. Holds a goroutine inside the
// filter peeking until the deadline, but must not block the next conn.
slowConn, err := net.Dial("tcp", innerListener.Addr().String())
if err != nil {
t.Fatal(err)
}
defer slowConn.Close()
// Fast client: send a valid v3 frame straight away; this conn must be
// delivered to the inner accept handler without waiting for slowConn.
fastConn, err := net.Dial("tcp", innerListener.Addr().String())
if err != nil {
t.Fatal(err)
}
defer fastConn.Close()
if _, err := fastConn.Write(buildRPCCallFrame(11, nfsProgram, 3, 0)); err != nil {
t.Fatal(err)
}
// Bound the wait well below rpcVersionFilterPeekTimeout (10s) so a
// regression to inline peeking would clearly time out here.
select {
case <-delivered:
case <-time.After(2 * time.Second):
t.Fatal("fast conn should not be head-of-line blocked by slow conn's peek")
}
}
+250
View File
@@ -0,0 +1,250 @@
package nfs
import (
"context"
"errors"
"fmt"
"net"
"sync"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
gonfs "github.com/willscott/go-nfs"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
type Option struct {
Filer pb.ServerAddress
BindIp string
Port int
FilerRootPath string
ReadOnly bool
AllowedClients []string
VolumeServerAccess string
GrpcDialOption grpc.DialOption
// PortmapBind, when non-empty, enables a built-in portmap v2 responder
// on <PortmapBind>:111 advertising the NFS v3 and MOUNT v3 services at
// Port. Empty (the default) disables portmap; clients must then bypass
// portmap with mount -o port=,mountport=,proto=tcp,mountproto=tcp.
PortmapBind string
}
type Server struct {
option *Option
exportRoot util.FullPath
exportID uint32
signature int32
handleLimit int
clientAuthorizer *clientAuthorizer
sharedReaderCache *filer.ReaderCache
chunkInvalidator chunkInvalidator
filerClient *wdclient.FilerClient
newUploader func() (chunkUploader, error)
withFilerClient filerClientExecutor
withInternalClient internalClientExecutor
rootFSOnce sync.Once
rootFS *seaweedFileSystem
}
func NewServer(option *Option) (*Server, error) {
if option == nil {
return nil, errors.New("nfs option is required")
}
if option.Port <= 0 {
return nil, fmt.Errorf("nfs port must be positive: %d", option.Port)
}
if option.FilerRootPath == "" {
option.FilerRootPath = "/"
}
if option.VolumeServerAccess == "" {
option.VolumeServerAccess = "direct"
}
if option.GrpcDialOption == nil {
option.GrpcDialOption = grpc.WithTransportCredentials(insecure.NewCredentials())
}
clientAuthorizer, err := newClientAuthorizer(option.AllowedClients)
if err != nil {
return nil, err
}
var filerClient *wdclient.FilerClient
if option.VolumeServerAccess != "filerProxy" {
var opts *wdclient.FilerClientOption
if option.VolumeServerAccess == "publicUrl" {
opts = &wdclient.FilerClientOption{UrlPreference: wdclient.PreferPublicUrl}
}
filerClient = wdclient.NewFilerClient([]pb.ServerAddress{option.Filer}, option.GrpcDialOption, "", opts)
}
exportRoot := normalizeExportRoot(util.FullPath(option.FilerRootPath))
signature := util.RandomInt32()
return &Server{
option: option,
exportRoot: exportRoot,
exportID: exportIDForRoot(exportRoot),
signature: signature,
handleLimit: 1 << 20,
clientAuthorizer: clientAuthorizer,
filerClient: filerClient,
newUploader: newChunkUploader,
withFilerClient: newFilerClientExecutor(option, signature),
withInternalClient: newInternalClientExecutor(option, signature),
}, nil
}
func (s *Server) Start() error {
listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", s.option.BindIp, s.option.Port))
if err != nil {
return fmt.Errorf("listen nfs on %s:%d: %w", s.option.BindIp, s.option.Port, err)
}
// MOUNT v3 over UDP runs alongside the TCP NFS listener on the same
// port. The kernel default for mountproto is UDP in many setups, so
// without this responder a plain `mount -t nfs <host>:<export> /mnt`
// gets EPROTONOSUPPORT during the MOUNT phase even though the TCP
// NFS path is fine.
mountUDP := newMountUDPServer(s.option.BindIp, s.option.Port, s)
if err := mountUDP.Start(); err != nil {
_ = listener.Close()
return fmt.Errorf("start mount udp: %w", err)
}
defer func() {
_ = mountUDP.Close()
}()
glog.V(0).Infof("MOUNT v3 UDP responder listening on %s:%d", s.option.BindIp, s.option.Port)
var portmap *portmapServer
if s.option.PortmapBind != "" {
portmap = newPortmapServer(s.option.PortmapBind, portmapPort, uint32(s.option.Port))
if pmErr := portmap.Start(); pmErr != nil {
_ = listener.Close()
return fmt.Errorf("start portmap: %w", pmErr)
}
glog.V(0).Infof("NFS portmap responder listening on %s:%d (NFS v3 tcp=%d, MOUNT v3 tcp=%d, MOUNT v3 udp=%d)",
s.option.PortmapBind, portmapPort, s.option.Port, s.option.Port, s.option.Port)
defer func() {
if portmap != nil {
_ = portmap.Close()
}
}()
}
s.logMountHint()
return s.serve(listener)
}
// logMountHint prints a copy-pasteable Linux mount command so operators can
// see at startup how to mount the export from a client.
//
// With -portmap.bind set, MOUNT is now answered over both TCP and UDP, so a
// plain `mount -t nfs host:/export /mnt` works — there is no longer any
// kernel-default mountproto path that fails. Without -portmap.bind the
// client still has to bypass portmap entirely via the explicit
// port=/mountport=/proto=/mountproto= options.
func (s *Server) logMountHint() {
exportPath := string(s.exportRoot)
if s.option.PortmapBind != "" {
glog.V(0).Infof("mount example: mount -t nfs -o nfsvers=3,nolock <host>:%s <mountpoint>", exportPath)
glog.V(0).Infof("(MOUNT v3 is served over both TCP and UDP, so no mountproto override is needed.)")
return
}
glog.V(0).Infof("mount example (bypasses portmap): mount -t nfs -o nfsvers=3,nolock,noacl,port=%d,mountport=%d,proto=tcp,mountproto=tcp <host>:%s <mountpoint>",
s.option.Port, s.option.Port, exportPath)
glog.V(0).Infof("tip: pass -portmap.bind to enable the built-in portmap responder on port 111 so plain `mount -t nfs host:%s /mnt` works.", exportPath)
}
func (s *Server) serve(listener net.Listener) error {
if s.filerClient != nil {
defer s.filerClient.Close()
}
if s.clientAuthorizer != nil && s.clientAuthorizer.enabled {
listener = &allowlistListener{
Listener: listener,
authorizer: s.clientAuthorizer,
}
}
listener = newVersionFilterListener(listener)
handler, err := s.newHandler()
if err != nil {
_ = listener.Close()
return err
}
followCtx, followCancel := context.WithCancel(context.Background())
defer followCancel()
followDone := make(chan struct{})
go func() {
defer close(followDone)
s.runMetadataInvalidationLoop(followCtx)
}()
defer func() {
followCancel()
<-followDone
}()
glog.V(0).Infof("Start Seaweed NFS Server filer=%s bind=%s export=%s exportId=%d readOnly=%t allowedClients=%d volumeServerAccess=%s",
s.option.Filer,
listener.Addr(),
s.exportRoot,
s.exportID,
s.option.ReadOnly,
len(s.option.AllowedClients),
s.option.VolumeServerAccess,
)
return gonfs.Serve(listener, handler)
}
func (s *Server) newHandler() (*Handler, error) {
if s == nil {
return nil, errors.New("nfs server is not configured")
}
return &Handler{
server: s,
rootFS: s.rootFilesystem(),
}, nil
}
// rootFilesystem returns a single seaweedFileSystem rooted at the
// configured export, building it on first call. Both the TCP handler
// (via newHandler) and the UDP MOUNT path use the same instance so
// they share the chunk reader cache and don't reconstruct a wrapper
// per request.
func (s *Server) rootFilesystem() *seaweedFileSystem {
s.rootFSOnce.Do(func() {
s.rootFS = newSeaweedFileSystem(s, s.exportRoot, s.sharedReaderCache)
if s.sharedReaderCache == nil {
s.sharedReaderCache = s.rootFS.readerCache
}
if s.chunkInvalidator == nil {
s.chunkInvalidator = s.sharedReaderCache
}
})
return s.rootFS
}
func (s *Server) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
if s == nil || s.withFilerClient == nil {
return errors.New("nfs filer client is not configured")
}
return s.withFilerClient(streamingMode, fn)
}
func (s *Server) LookupFn() wdclient.LookupFileIdFunctionType {
if s == nil {
return nil
}
if s.option != nil && s.option.VolumeServerAccess == "filerProxy" {
return func(ctx context.Context, fileID string) ([]string, error) {
return []string{fmt.Sprintf("http://%s/?proxyChunkId=%s", s.option.Filer.ToHttpAddress(), fileID)}, nil
}
}
if s.filerClient != nil {
return s.filerClient.GetLookupFileIdFunction()
}
return nil
}
File diff suppressed because it is too large Load Diff
+40
View File
@@ -0,0 +1,40 @@
package nfs
import (
"io"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
type chunkUploader interface {
UploadWithRetry(
filerClient filer_pb.FilerClient,
assignRequest *filer_pb.AssignVolumeRequest,
uploadOption *operation.UploadOption,
genFileUrlFn func(host, fileId string) string,
reader io.Reader,
) (fileId string, uploadResult *operation.UploadResult, err error, data []byte)
}
type operationChunkUploader struct {
uploader *operation.Uploader
}
func (u operationChunkUploader) UploadWithRetry(
filerClient filer_pb.FilerClient,
assignRequest *filer_pb.AssignVolumeRequest,
uploadOption *operation.UploadOption,
genFileUrlFn func(host, fileId string) string,
reader io.Reader,
) (string, *operation.UploadResult, error, []byte) {
return u.uploader.UploadWithRetry(filerClient, assignRequest, uploadOption, genFileUrlFn, reader)
}
func newChunkUploader() (chunkUploader, error) {
uploader, err := operation.NewUploader()
if err != nil {
return nil, err
}
return operationChunkUploader{uploader: uploader}, nil
}
+5 -6
View File
@@ -42,11 +42,11 @@ type VolumeServer struct {
currentMaster pb.ServerAddress
currentMasterLock sync.RWMutex
pulsePeriod time.Duration
dataCenter string
rack string
store *storage.Store
guard *security.Guard
grpcDialOption grpc.DialOption
dataCenter string
rack string
store *storage.Store
guard *security.Guard
grpcDialOption grpc.DialOption
needleMapKind storage.NeedleMapKind
ldbTimout int64
@@ -140,7 +140,6 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
handleStaticResources(adminMux)
adminMux.HandleFunc("/status", requestIDMiddleware(vs.statusHandler))
adminMux.HandleFunc("/healthz", requestIDMiddleware(vs.healthzHandler))
adminMux.HandleFunc("/readyz", requestIDMiddleware(vs.healthzHandler))
if signingKey == "" || enableUiAccess {
// only expose the volume server details for safe environments
adminMux.HandleFunc("/ui/index.html", requestIDMiddleware(vs.uiStatusHandler))
+3 -13
View File
@@ -448,7 +448,6 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(dataNodeVolumeIdToVIn
// MasterClient.GetLocations, so iterating per replica here (as the old
// code did) would issue N*N delete RPCs for N replicas.
if applyPurging {
var skippedVolumeIds []uint32
for volumeId, orphanReplicaFileIds := range volumeIdOrphanFileIds {
if len(orphanReplicaFileIds) == 0 {
continue
@@ -459,19 +458,11 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(dataNodeVolumeIdToVIn
}
// Call out to a closure per volume so the deferred "mark
// readonly again" fires between volumes instead of piling up
// until findExtraChunksInVolumeServers returns. Per-volume
// failures (e.g. a replica stuck read-only) don't halt the
// rest of the run; the volume is remembered and its deletes
// are skipped.
// until findExtraChunksInVolumeServers returns.
if err := c.purgeOneVolume(volumeId, orphanReplicaFileIds, volumeReplicaCounts[volumeId], readOnlyServerReplicas[volumeId]); err != nil {
fmt.Fprintf(c.writer, "skip purging volume %d: %v\n", volumeId, err)
skippedVolumeIds = append(skippedVolumeIds, volumeId)
return err
}
}
if len(skippedVolumeIds) > 0 {
sort.Slice(skippedVolumeIds, func(i, j int) bool { return skippedVolumeIds[i] < skippedVolumeIds[j] })
fmt.Fprintf(c.writer, "skipped purge on %d volume(s): %v\n", len(skippedVolumeIds), skippedVolumeIds)
}
}
if !applyPurging {
@@ -519,8 +510,7 @@ func (c *commandVolumeFsck) purgeOneVolume(volumeId uint32, orphanReplicaFileIds
needleVID := needle.VolumeId(volumeId)
for _, server := range readOnlyReplicas {
if err := markVolumeWritable(c.env.option.GrpcDialOption, needleVID, server, true, false); err != nil {
// Replicas flipped writable earlier roll back via the defer.
return fmt.Errorf("mark %v writable: %v", server, err)
return fmt.Errorf("mark volume %d on %v read/write: %v", volumeId, server, err)
}
fmt.Fprintf(c.writer, "temporarily marked %d on server %v writable for forced purge\n", volumeId, server)
defer markVolumeWritable(c.env.option.GrpcDialOption, needleVID, server, false, false)
+15 -83
View File
@@ -52,9 +52,8 @@ This command:
1) marks the volume readonly on replicas (if not already)
2) allocates a temporary copy on a third location
3) merges replicas in append timestamp order, skipping duplicates
4) verifies the merged copy is not short before touching the replicas
5) replaces the original replicas with the merged volume
6) restores writable state if it was writable before
4) replaces the original replicas with the merged volume
5) restores writable state if it was writable before
`
}
@@ -174,27 +173,17 @@ func (c *commandVolumeMerge) Do(args []string, commandEnv *CommandEnv, writer io
return mergeErr
}
// Verify the merged copy before overwriting any replica. A short or empty
// merge stamped over every replica at once is unrecoverable; on failure the
// originals are left intact and the bad merged copy is cleaned up.
if err = verifyMergedVolume(commandEnv.option.GrpcDialOption, volumeId, targetServer, replicas); err != nil {
return err
}
// The merged copy is now the only authoritative copy. Keep it until every
// replica is rebuilt from it so a mid-loop failure can still be finished.
cleanupTarget = false
for i, replica := range replicas {
for _, replica := range replicas {
sourceServer := pb.NewServerAddressFromDataNode(replica.location.dataNode)
if _, err = copyVolume(commandEnv.option.GrpcDialOption, writer, volumeId, targetServer, sourceServer, "", 0, false); err != nil {
return fmt.Errorf("rebuild replica %d/%d on %s from merged volume %d: %w; merged copy kept on %s, re-run to finish", i+1, len(replicas), sourceServer, volumeId, err, targetServer)
return err
}
}
if err = deleteVolume(commandEnv.option.GrpcDialOption, volumeId, targetServer, false, false); err != nil {
return err
}
cleanupTarget = false
fmt.Fprintf(writer, "merged volume %d from %d replicas via %s\n", volumeId, len(replicas), targetServer)
return nil
@@ -480,13 +469,19 @@ func ensureVolumeReadonly(commandEnv *CommandEnv, replicas []*VolumeReplica) ([]
var writableReplicaIndices []int
for i, replica := range replicas {
server := pb.NewServerAddressFromDataNode(replica.location.dataNode)
status, err := readVolumeStatus(commandEnv.option.GrpcDialOption, server, needle.VolumeId(replica.info.Id))
err := operation.WithVolumeServerClient(false, server, commandEnv.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
resp, err := client.VolumeStatus(context.Background(), &volume_server_pb.VolumeStatusRequest{VolumeId: replica.info.Id})
if err != nil {
return err
}
if !resp.IsReadOnly {
writableReplicaIndices = append(writableReplicaIndices, i)
}
return nil
})
if err != nil {
return nil, err
}
if !status.IsReadOnly {
writableReplicaIndices = append(writableReplicaIndices, i)
}
}
if len(writableReplicaIndices) > 0 {
if err := markReplicasWritable(commandEnv.option.GrpcDialOption, replicas, false, false); err != nil {
@@ -496,69 +491,6 @@ func ensureVolumeReadonly(commandEnv *CommandEnv, replicas []*VolumeReplica) ([]
return writableReplicaIndices, nil
}
// verifyMergedVolume checks the freshly merged copy is at least as complete as the
// most complete source replica before the originals are overwritten.
func verifyMergedVolume(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, targetServer pb.ServerAddress, replicas []*VolumeReplica) error {
merged, err := readVolumeStatus(grpcDialOption, targetServer, volumeId)
if err != nil {
return fmt.Errorf("read merged volume %d on %s: %w", volumeId, targetServer, err)
}
replicaLive := make(map[pb.ServerAddress]uint64, len(replicas))
for _, replica := range replicas {
server := pb.NewServerAddressFromDataNode(replica.location.dataNode)
status, err := readVolumeStatus(grpcDialOption, server, volumeId)
if err != nil {
return fmt.Errorf("read replica volume %d on %s: %w", volumeId, server, err)
}
replicaLive[server] = liveNeedleCount(status)
}
return evaluateMergedVolume(volumeId, liveNeedleCount(merged), replicaLive)
}
// evaluateMergedVolume reports why a merged copy with mergedLive live needles is
// unsafe to overwrite replicas whose live counts are replicaLive. The merge is the
// union of every replica's needles, so the merged copy must be non-empty and hold
// at least as many live needles as the most complete replica.
func evaluateMergedVolume(volumeId needle.VolumeId, mergedLive uint64, replicaLive map[pb.ServerAddress]uint64) error {
if mergedLive == 0 {
return fmt.Errorf("merged volume %d is empty; keeping original replicas", volumeId)
}
var maxLive uint64
var maxFrom pb.ServerAddress
for server, live := range replicaLive {
if live > maxLive {
maxLive, maxFrom = live, server
}
}
if mergedLive < maxLive {
return fmt.Errorf("merged volume %d has %d live needles, fewer than replica %s with %d; refusing to overwrite replicas", volumeId, mergedLive, maxFrom, maxLive)
}
return nil
}
func liveNeedleCount(status *volume_server_pb.VolumeStatusResponse) uint64 {
if status == nil || status.FileCount < status.FileDeletedCount {
return 0
}
return status.FileCount - status.FileDeletedCount
}
func readVolumeStatus(grpcDialOption grpc.DialOption, server pb.ServerAddress, volumeId needle.VolumeId) (*volume_server_pb.VolumeStatusResponse, error) {
var resp *volume_server_pb.VolumeStatusResponse
err := operation.WithVolumeServerClient(false, server, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
r, statusErr := client.VolumeStatus(context.Background(), &volume_server_pb.VolumeStatusRequest{VolumeId: uint32(volumeId)})
if statusErr != nil {
return statusErr
}
resp = r
return nil
})
if err == nil && resp == nil {
return nil, fmt.Errorf("empty volume status response from %s", server)
}
return resp, err
}
func isReplicaServer(target pb.ServerAddress, replicas []*VolumeReplica) bool {
for _, replica := range replicas {
if pb.NewServerAddressFromDataNode(replica.location.dataNode).Equals(target) {

Some files were not shown because too many files have changed in this diff Show More