mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-10-22 06:22:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into pr11709-merge

Browse Source 
# Conflicts:
#	weed/command/filer_remote_sync_dir.go
#	weed/command/filer_remote_sync_dir_test.go
This commit is contained in:
Chris Lu
2026-10-12 03:14:35 -07:00
parent 16110fd387 41260a55f6
commit d9d2dec9ac
134 changed files with 7594 additions and 2790 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/helm_ci.yml
+35
View File
@@ -95,6 +95,41 @@ jobs:
grep -A 5 "^ lifecycle:" /tmp/s3-rollout.yaml | grep -q -- "- sleep"
echo "S3 rollout settings render correctly"
echo "=== Testing the S3 config checksum is stable across renders ==="
# s3-secret.yaml keeps its generated keys with lookup. Hashing the rendered
# Secret drew new keys for the checksum whenever lookup found nothing (on
# install), so the first upgrade changed the checksum and restarted the
# filer and S3 pods although nothing had changed.
s3_checksum() { # <template> <helm args...>: the checksum/s3config of one workload
local template=$1; shift
local sum
sum=$(helm template test $CHART_DIR --show-only "$template" "$@" \
| grep 'checksum/s3config:' | awk '{print $2}')
test -n "$sum" || { echo "no checksum/s3config in $template" >&2; return 1; }
echo "$sum"
}
s3_auth_args=(--set s3.enabled=true --set s3.enableAuth=true)
s3_sum=$(s3_checksum templates/s3/s3-deployment.yaml "${s3_auth_args[@]}")
filer_sum=$(s3_checksum templates/filer/filer-statefulset.yaml "${s3_auth_args[@]}")
test "$s3_sum" = "$filer_sum" # one Secret, one checksum
test "$s3_sum" = "$(s3_checksum templates/s3/s3-deployment.yaml "${s3_auth_args[@]}")"
s3_creds_sum=$(s3_checksum templates/s3/s3-deployment.yaml "${s3_auth_args[@]}" \
--set s3.credentials.admin.accessKey=a --set s3.credentials.admin.secretKey=b)
test "$s3_sum" != "$s3_creds_sum"
filer_audit_sum=$(s3_checksum templates/filer/filer-statefulset.yaml "${s3_auth_args[@]}" \
--set-json 'filer.s3.auditLogConfig={"fluent_port":24224}')
test "$filer_sum" != "$filer_audit_sum"
# Settings the Secret does not read must not restart the S3 and filer pods.
test "$filer_sum" = "$(s3_checksum templates/filer/filer-statefulset.yaml "${s3_auth_args[@]}" \
--set s3.replicas=3 --set s3.resources.limits.cpu=1)"
all_in_one_args=(--set allInOne.enabled=true --set allInOne.s3.enabled=true --set allInOne.s3.enableAuth=true)
aio_sum=$(s3_checksum templates/all-in-one/all-in-one-deployment.yaml "${all_in_one_args[@]}")
test "$aio_sum" = "$(s3_checksum templates/all-in-one/all-in-one-deployment.yaml "${all_in_one_args[@]}")"
aio_creds_sum=$(s3_checksum templates/all-in-one/all-in-one-deployment.yaml "${all_in_one_args[@]}" \
--set s3.credentials.admin.accessKey=a --set s3.credentials.admin.secretKey=b)
test "$aio_sum" != "$aio_creds_sum"
echo "S3 config checksum is stable and follows only what the S3 Secret reads"
echo "=== Testing S3 credentials from an existing secret ==="
credential_args=(
--set s3.credentials.admin.existingSecret=minio-root
go.mod
+7 -7
View File
@@ -125,7 +125,7 @@ require (
github.com/aws/aws-sdk-go-v2 v1.47.1
github.com/aws/aws-sdk-go-v2/config v1.33.6
github.com/aws/aws-sdk-go-v2/credentials v1.20.6
github.com/aws/aws-sdk-go-v2/service/s3 v1.109.1
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1
github.com/cespare/xxhash/v2 v2.3.0
github.com/cognusion/imaging v1.0.4
github.com/fluent/fluent-logger-golang v1.10.1
@@ -156,7 +156,7 @@ require (
github.com/tikv/client-go/v2 v2.0.7
github.com/twmb/avro v1.9.0
github.com/xeipuuv/gojsonschema v1.2.0
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.2
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.3
github.com/ydb-platform/ydb-go-sdk/v3 v3.151.1
go.etcd.io/etcd/client/pkg/v3 v3.7.1
go.uber.org/atomic v1.12.0
@@ -324,9 +324,9 @@ require (
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.39.14 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.24 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect
@@ -469,10 +469,10 @@ require (
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/winfsp/cgofuse v1.6.1-0.20260126094232-f2c4fccdb286
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
github.com/yandex-cloud/go-genproto v0.0.0-20240819112322-98a264d392f6 // indirect
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260810122915-65bfd5c4b705 // indirect
github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
github.com/ydb-platform/ydb-go-yc v0.12.5 // indirect
github.com/ydb-platform/ydb-go-yc-metadata v0.6.2 // indirect
github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zeebo/blake3 v0.2.4 // indirect
go.sum
+14 -1197
View File
File diff suppressed because it is too large Load Diff
k8s/charts/seaweedfs/templates/all-in-one/all-in-one-deployment.yaml
+1 -1
View File
@@ -43,7 +43,7 @@ spec:
{{- $configSecret := (lookup "v1" "Secret" .Release.Namespace $existingS3ConfigSecret) | default dict }}
{{- $_ := set $podAnnotations "checksum/s3config" ($configSecret | toYaml | sha256sum) }}
{{- else }}
{{- $_ := set $podAnnotations "checksum/s3config" (include (print .Template.BasePath "/s3/s3-secret.yaml") . | sha256sum) }}
{{- $_ := set $podAnnotations "checksum/s3config" (include "seaweedfs.s3.configChecksum" .) }}
{{- end }}
{{- $_ := set $podAnnotations "checksum/master-config" (include (print .Template.BasePath "/master/master-configmap.yaml") . | sha256sum) }}
annotations:
k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml
+1 -1
View File
@@ -48,7 +48,7 @@ spec:
{{- $configSecret := (lookup "v1" "Secret" .Release.Namespace .Values.filer.s3.existingConfigSecret) | default dict }}
{{- $_ := set $podAnnotations "checksum/s3config" ($configSecret | toYaml | sha256sum) }}
{{- else }}
{{- $_ := set $podAnnotations "checksum/s3config" (include (print .Template.BasePath "/s3/s3-secret.yaml") . | sha256sum) }}
{{- $_ := set $podAnnotations "checksum/s3config" (include "seaweedfs.s3.configChecksum" .) }}
{{- end }}
annotations:
{{- toYaml $podAnnotations | nindent 8 }}
k8s/charts/seaweedfs/templates/s3/s3-deployment.yaml
+1 -1
View File
@@ -44,7 +44,7 @@ spec:
{{- $configSecret := (lookup "v1" "Secret" .Release.Namespace .Values.s3.existingConfigSecret) | default dict }}
{{- $_ := set $podAnnotations "checksum/s3config" ($configSecret | toYaml | sha256sum) }}
{{- else }}
{{- $_ := set $podAnnotations "checksum/s3config" (include (print .Template.BasePath "/s3/s3-secret.yaml") . | sha256sum) }}
{{- $_ := set $podAnnotations "checksum/s3config" (include "seaweedfs.s3.configChecksum" .) }}
{{- end }}
annotations:
{{- toYaml $podAnnotations | nindent 8 }}
k8s/charts/seaweedfs/templates/shared/_helpers.tpl
+25
View File
@@ -692,3 +692,28 @@ Usage: {{ include "seaweedfs.service.loadBalancerFields" .Values.s3.service }}
{{- end }}
{{- end }}
{{- end -}}
{{/*
Checksum that restarts the S3-serving pods when the generated S3 config changes. It hashes the
values s3-secret.yaml is built from, not the rendered Secret: that template keeps its generated keys
with lookup, so rendering it again for a checksum draws new random keys whenever lookup finds
nothing (on install), and the first upgrade then changed the checksum and restarted the pods
although nothing had changed. Only the values that template reads are hashed, so scaling the S3
gateway or changing its resources does not restart the filer. Keep this list in step with
s3-secret.yaml.
Keys the chart generated are not part of it. Deleting the Secret to rotate them regenerates them on
the next upgrade without restarting the pods; restart them afterwards (kubectl rollout restart).
Folding the stored keys in would bring the first-upgrade restart back, since lookup finds nothing
on install.
*/}}
{{- define "seaweedfs.s3.configChecksum" -}}
{{- dict
"secretName" (include "seaweedfs.fullname" .)
"enabled" (list .Values.s3.enabled .Values.filer.s3.enabled .Values.allInOne.enabled .Values.allInOne.s3.enabled)
"enableAuth" (list .Values.s3.enableAuth .Values.filer.s3.enableAuth .Values.allInOne.s3.enableAuth)
"existingConfigSecret" (list .Values.s3.existingConfigSecret .Values.filer.s3.existingConfigSecret .Values.allInOne.s3.existingConfigSecret)
"credentials" .Values.s3.credentials
"legacySecret" (list .Values.s3.reuseLegacySecret .Values.s3.legacySecretName)
"auditLogConfig" (list .Values.s3.auditLogConfig .Values.filer.s3.auditLogConfig)
| toJson | sha256sum -}}
{{- end -}}
note/star_history.svg
+1059 -1070
View File
File diff suppressed because it is too large Load Diff
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 54 KiB

After

Width:  |  Height:  |  Size: 54 KiB

seaweed-volume/src/server/grpc_server.rs
+50 -1
View File
@@ -986,7 +986,12 @@ impl VolumeServer for VolumeGrpcService {
let original_cookie = n.cookie;
if !is_ec_volume {
let store = self.state.store.read().unwrap();
match store.read_volume_needle(file_id.volume_id, &mut n) {
let mut read_option = crate::server::handlers::delete_read_option();
match store.read_volume_needle_with_option(
file_id.volume_id,
&mut n,
&mut read_option,
) {
Ok(_) => {}
Err(e) => {
results.push(volume_server_pb::DeleteResult {
@@ -10997,6 +11002,50 @@ mod tests {
(r.status, r.error.clone())
}
/// A batch delete only needs the needle's cookie, which is stored in the
/// header, so it must not read the payload.
#[tokio::test]
async fn batch_delete_reads_needle_meta_only() {
use crate::storage::volume::needle_read_hook;
use std::sync::Mutex;
const ID: u64 = 0x6e7a_0b01;
let (service, _tmp) = make_local_service_with_volume("", None);
let data = vec![7u8; 512 * 1024];
let mut n = Needle {
id: NeedleId(ID),
cookie: Cookie(0x3344),
data: data.clone(),
data_size: data.len() as u32,
..Needle::default()
};
service
.state
.store
.write()
.unwrap()
.write_volume_needle(VolumeId(1), &mut n, false)
.unwrap();
let reads = Arc::new(Mutex::new(Vec::new()));
let _hook = needle_read_hook::register(NeedleId(ID), {
let reads = reads.clone();
move |len| reads.lock().unwrap().push(len)
});
let (status, _) = batch_delete_1(&service, ID, 0x3345, false).await;
assert_eq!(status, 400);
let (status, error) = batch_delete_1(&service, ID, 0x3344, false).await;
assert_eq!(status, 202, "{error}");
let reads = reads.lock().unwrap().clone();
assert!(
!reads.is_empty() && reads.iter().all(|&len| len < 4096),
"a batch delete must not read the needle's payload ({} bytes): {reads:?}",
data.len()
);
}
const GO_EC_NEEDLE_NOT_FOUND: &str =
"locate in local ec volume: FindNeedleFromEcx: needle not found";
seaweed-volume/src/server/handlers.rs
+93 -5
View File
@@ -161,9 +161,10 @@ impl SourceReadRequest {
/// HEAD, a range, or a stream.
fn served_from_source(&self, n: &Needle) -> bool {
// HEAD is meta-only unless the payload is needed to expand a manifest.
if self.is_head {
return !(n.is_chunk_manifest() && !self.bypass_cm);
// HEAD is meta-only unless the payload is needed to expand a
// manifest or to answer an image transform's length.
return !self.has_image_ops && !(n.is_chunk_manifest() && !self.bypass_cm);
}
self.direct(n) && (self.has_range || n.data_size > STREAMING_THRESHOLD)
}
@@ -171,8 +172,10 @@ impl SourceReadRequest {
/// Meta first only if the needle could be served from source; its data
/// size is below its index size.
fn try_meta_first(&self, size: Size) -> bool {
self.is_head
|| (!self.has_image_ops && (self.has_range || size.0 > STREAMING_THRESHOLD as i32))
if self.has_image_ops {
return false;
}
self.is_head || self.has_range || size.0 > STREAMING_THRESHOLD as i32
}
}
@@ -3058,6 +3061,20 @@ struct DeleteResult {
size: i64,
}
/// Read option for deletes: read only the needle's header and trailer.
///
/// A delete needs to know that the needle exists, what its cookie is, and
/// whether it is a chunk manifest. None of that requires the payload. If the
/// needle turns out to be a chunk manifest, the read loads its data as well,
/// because the delete needs the manifest to delete the chunks.
pub(crate) fn delete_read_option() -> ReadOption {
ReadOption {
attempt_meta_only: true,
must_meta_only: true,
..ReadOption::default()
}
}
pub async fn delete_handler(
State(state): State<Arc<VolumeServerState>>,
request: Request<Body>,
@@ -3226,7 +3243,7 @@ pub async fn delete_handler(
let original_cookie = cookie;
{
let store = state.store.read().unwrap();
match store.read_volume_needle(vid, &mut n) {
match store.read_volume_needle_with_option(vid, &mut n, &mut delete_read_option()) {
Ok(_) => {}
Err(_) => {
let result = DeleteResult { size: 0 };
@@ -5523,6 +5540,17 @@ mod tests {
(status, headers, body)
}
/// The public router does not accept DELETE requests, so this calls the
/// handler directly, the same way the admin router does.
async fn send_delete(state: &Arc<VolumeServerState>, path: &str) -> StatusCode {
let request = Request::builder()
.method(Method::DELETE)
.uri(path)
.body(Body::empty())
.unwrap();
delete_handler(State(state.clone()), request).await.status()
}
/// A GET whose needle read is parked must not hold the store lock: a
/// writer, here a real append to the same volume, must get through.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -5627,6 +5655,43 @@ mod tests {
);
}
/// A delete needs the needle's cookie and its chunk-manifest flag. Both
/// are stored in the header and trailer, so the delete must not read the
/// payload.
#[tokio::test]
async fn test_delete_reads_needle_meta_only() {
use crate::storage::volume::needle_read_hook;
use std::sync::Mutex;
const ID: u64 = 0x6e7a_0a01;
let tmp = tempfile::TempDir::new().unwrap();
let state = volume_test_state(&tmp);
let data: Vec<u8> = (0..512 * 1024).map(|i| (i % 251) as u8).collect();
let path = put_test_needle(&state, ID, &data);
let reads = Arc::new(Mutex::new(Vec::new()));
let _hook = needle_read_hook::register(NeedleId(ID), {
let reads = reads.clone();
move |len| reads.lock().unwrap().push(len)
});
let wrong_cookie = format!("/1,{:x}{:08x}", ID, TEST_COOKIE + 1);
assert_eq!(
send_delete(&state, &wrong_cookie).await,
StatusCode::BAD_REQUEST
);
assert_eq!(send_delete(&state, &path).await, StatusCode::ACCEPTED);
let reads = reads.lock().unwrap().clone();
assert!(
!reads.is_empty() && reads.iter().all(|&len| len < 4096),
"a delete must not read the needle's payload ({} bytes): {reads:?}",
data.len()
);
let (status, _, _) = send_read(&state, Method::GET, &path, None).await;
assert_eq!(status, StatusCode::NOT_FOUND);
}
/// HEAD and ranged reads keep answering from the needle meta and the
/// data file, and a missing or wrong-cookie needle is still a 404.
#[tokio::test]
@@ -6105,6 +6170,29 @@ mod tests {
assert_eq!(taken(&reads), vec![1]);
}
/// A HEAD asking for an image transform needs the payload too: a
/// meta-only read could not produce the transformed length.
#[tokio::test]
async fn test_head_with_image_op_reads_full() {
let tmp = tempfile::TempDir::new().unwrap();
let state = volume_test_state(&tmp);
let png = solid_png(8, 8);
let stored = put_test_needle(&state, 0x6e7a_0a30, &png);
let crop = format!(
"{}.png?crop_x1=0&crop_y1=0&crop_x2=2&crop_y2=2",
stored
);
let (get_status, _, cropped) = send_read(&state, Method::GET, &crop, None).await;
assert_eq!(get_status, StatusCode::OK);
assert_ne!(cropped.len(), png.len(), "the crop must change the length");
let (status, headers, body) = send_read(&state, Method::HEAD, &crop, None).await;
assert_eq!(status, StatusCode::OK);
assert!(body.is_empty());
assert_eq!(headers[header::CONTENT_LENGTH], cropped.len().to_string());
}
/// Go takes the transform extension from the URL first — including a fid
/// suffix like /vid,fid.png — before the resolved filename's.
#[tokio::test]
seaweed-volume/src/storage/store.rs
+14
View File
@@ -736,6 +736,20 @@ impl Store {
vol.read_needle(n)
}
/// Read a needle from a volume using the given read option.
///
/// On return, `read_option.is_meta_only` tells the caller whether the
/// payload was skipped.
pub fn read_volume_needle_with_option(
&self,
vid: VolumeId,
n: &mut Needle,
read_option: &mut ReadOption,
) -> Result<i32, VolumeError> {
let (_, vol) = self.find_volume(vid).ok_or(VolumeError::NotFound)?;
vol.read_needle_with_option(n, read_option)
}
/// Read a needle from a volume, optionally reading deleted needles.
pub fn read_volume_needle_opt(
&self,
seaweed-volume/src/storage/volume.rs
+137 -6
View File
@@ -2009,12 +2009,29 @@ impl Volume {
return Ok(0);
}
match self.read_needle_data_at_unlocked(
n,
nv.offset.to_actual_offset(),
read_size,
read_option,
) {
// With `must_meta_only`, read the header and trailer first. If the
// needle is compressed or is a chunk manifest, the caller needs its
// data, so read the payload as well.
//
// A version 1 needle has no data size field and no metadata after its
// data, so the metadata reader would treat the whole payload as
// metadata. It is always read in full.
let offset = nv.offset.to_actual_offset();
// `is_meta_only` is a result: clear it in case the option is reused.
read_option.is_meta_only = false;
let meta_first = read_option.must_meta_only && self.version() != VERSION_1;
let mut read = if meta_first {
self.read_needle_meta_at_unlocked(n, offset, read_size)
} else {
Ok(())
};
if read.is_ok() && meta_first && !n.is_compressed() && !n.is_chunk_manifest() {
read_option.is_meta_only = true;
}
if read.is_ok() && !read_option.is_meta_only {
read = self.read_needle_data_at_unlocked(n, offset, read_size, read_option);
}
match read {
Ok(()) => self.check_read_write_error(None),
Err(VolumeError::Io(ref e)) => {
self.check_read_write_error(Some(e));
@@ -6080,6 +6097,120 @@ mod tests {
assert_eq!(read_n.cookie, Cookie(0x12345678));
}
/// With `must_meta_only` set, the payload is skipped whatever the needle's
/// size. This is the option a delete uses. A chunk manifest is the
/// exception: its data is still read, because a delete needs it to find
/// the chunks.
#[test]
fn test_read_needle_must_meta_only() {
let tmp = TempDir::new().unwrap();
let mut v = make_test_volume(tmp.path().to_str().unwrap());
let data = vec![7u8; 8192];
let mut plain = Needle {
id: NeedleId(1),
cookie: Cookie(0x1234),
data: data.clone(),
data_size: data.len() as u32,
..Needle::default()
};
v.write_needle(&mut plain, true, false).unwrap();
let mut manifest = Needle {
id: NeedleId(2),
data: b"{}".to_vec(),
data_size: 2,
..Needle::default()
};
manifest.set_is_chunk_manifest();
v.write_needle(&mut manifest, true, false).unwrap();
let read = |id: u64, read_option: &mut ReadOption| {
let mut n = Needle {
id: NeedleId(id),
..Needle::default()
};
v.read_needle_with_option(&mut n, read_option).unwrap();
n
};
let meta_only = || ReadOption {
attempt_meta_only: true,
must_meta_only: true,
..ReadOption::default()
};
let mut read_option = meta_only();
let n = read(1, &mut read_option);
assert!(read_option.is_meta_only);
assert!(n.data.is_empty(), "{} bytes of data read", n.data.len());
assert_eq!(n.cookie, Cookie(0x1234));
assert_eq!(n.data_size as usize, data.len());
// Without `must_meta_only`, the needle is read in full.
let mut read_option = ReadOption::default();
let n = read(1, &mut read_option);
assert!(!read_option.is_meta_only);
assert_eq!(n.data, data);
let mut read_option = meta_only();
let n = read(2, &mut read_option);
assert!(!read_option.is_meta_only);
assert_eq!(n.data, b"{}");
// The option is reused: a metadata-only result from the first read
// must not carry over and skip the chunk manifest's data.
let mut read_option = meta_only();
read(1, &mut read_option);
assert!(read_option.is_meta_only);
let n = read(2, &mut read_option);
assert!(!read_option.is_meta_only);
assert_eq!(n.data, b"{}");
}
/// A version 1 needle is a header, the data and a checksum: it has no data
/// size field and no metadata after the data, so the metadata reader would
/// take the whole payload for metadata and reject anything over 128 KiB.
/// A read with `must_meta_only` must fall back to a full read.
#[test]
fn test_read_needle_must_meta_only_version_1() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let spec = VolumeSpec {
version: VERSION_1,
..VolumeSpec::default()
};
let mut v = Volume::new(dir, dir, VolumeId(1), NeedleMapKind::InMemory, &spec).unwrap();
assert_eq!(v.version(), VERSION_1);
for (id, len) in [(1u64, 11usize), (2, 256 * 1024)] {
let data: Vec<u8> = b"hello world".iter().copied().cycle().take(len).collect();
// The server only writes version 2 and 3 records, so build the
// version 1 record by hand.
let size = Size(len as i32);
let mut blob = vec![0u8; NEEDLE_HEADER_SIZE];
Cookie(0x1234).to_bytes(&mut blob[0..COOKIE_SIZE]);
NeedleId(id).to_bytes(&mut blob[COOKIE_SIZE..COOKIE_SIZE + NEEDLE_ID_SIZE]);
size.to_bytes(&mut blob[COOKIE_SIZE + NEEDLE_ID_SIZE..NEEDLE_HEADER_SIZE]);
blob.extend_from_slice(&data);
blob.extend_from_slice(&CRC::new(&data).0.to_be_bytes());
blob.resize(get_actual_size(size, VERSION_1) as usize, 0);
v.write_needle_blob_and_index(NeedleId(id), &blob, size)
.unwrap();
let mut n = Needle {
id: NeedleId(id),
..Needle::default()
};
let mut read_option = ReadOption {
attempt_meta_only: true,
must_meta_only: true,
..ReadOption::default()
};
v.read_needle_with_option(&mut n, &mut read_option).unwrap();
assert!(!read_option.is_meta_only);
assert_eq!(n.cookie, Cookie(0x1234));
assert_eq!(n.data, data, "{len} byte needle");
}
}
/// A durable write goes down the same path and lands the same data; the
/// flush is extra work, not different work.
#[test]
seaweed-worker/Cargo.lock
Generated
+11 -2
View File
@@ -485,6 +485,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e"
dependencies = [
"aws-lc-sys",
"untrusted 0.7.1",
"zeroize",
]
@@ -5124,7 +5125,7 @@ dependencies = [
"cfg-if",
"getrandom 0.2.17",
"libc",
"untrusted",
"untrusted 0.9.0",
"windows-sys 0.52.0",
]
@@ -5260,7 +5261,7 @@ dependencies = [
"aws-lc-rs",
"ring",
"rustls-pki-types",
"untrusted",
"untrusted 0.9.0",
]
[[package]]
@@ -5342,7 +5343,9 @@ version = "0.1.0"
dependencies = [
"anyhow",
"async-trait",
"aws-lc-rs",
"axum",
"base64 0.22.1",
"prometheus",
"prost",
"prost-types",
@@ -6213,6 +6216,12 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
[[package]]
name = "untrusted"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
[[package]]
name = "untrusted"
version = "0.9.0"
seaweed-worker/crates/core/Cargo.toml
+4
View File
@@ -12,6 +12,10 @@ name = "seaweed_worker_core"
# Helpers the Rust volume server needs as well. A path dependency because
# the two trees are separate cargo workspaces with no common root manifest.
seaweed-common = { path = "../../../seaweed-common" }
# Already in the graph through tonic's `tls-aws-lc` feature; named here so the
# admin token can sign with the same HMAC the Go worker uses.
aws-lc-rs = "1"
base64 = "0.22"
anyhow.workspace = true
async-trait.workspace = true
prost.workspace = true
seaweed-worker/crates/core/src/config.rs
+11
View File
@@ -17,6 +17,15 @@ pub struct WorkerOptions {
/// section of security.toml. None means plaintext, which is the default the
/// Go worker also takes when no certificates are configured.
pub tls: Option<TlsOptions>,
/// The shared secret the admin stream requires when the cluster configures
/// `jwt.filer_signing.key`; mirrored from that config value. Empty means no
/// token is attached, which only works against admins that either bound
/// their worker listener to loopback or authenticate with mTLS instead.
pub signing_key: Option<String>,
/// Seconds the minted admin token lives, mirroring
/// `jwt.filer_signing.expires_after_seconds`. Zero mints a token without an
/// expiry, matching the Go default.
pub signing_key_expires_after_seconds: u64,
}
/// Certificates for the control stream. All three are required together: the
@@ -44,6 +53,8 @@ impl Default for WorkerOptions {
max_detection_concurrency: 1,
max_execution_concurrency: 1,
tls: None,
signing_key: None,
signing_key_expires_after_seconds: 0,
}
}
}
seaweed-worker/crates/core/src/lib.rs
+1
View File
@@ -13,6 +13,7 @@ pub mod metrics;
pub mod registry;
pub mod senders;
pub mod stream;
pub mod token;
/// Generated plugin.proto types.
pub mod pb {
seaweed-worker/crates/core/src/stream.rs
+13 -4
View File
@@ -176,10 +176,19 @@ async fn serve_once(
metadata: Default::default(),
}))?;
let mut inbound = client
.worker_stream(UnboundedReceiverStream::new(rx))
.await?
.into_inner();
let mut request = tonic::Request::new(UnboundedReceiverStream::new(rx));
if let Some(token) = crate::token::admin_token(
options.signing_key.as_deref().unwrap_or("").as_bytes(),
options.signing_key_expires_after_seconds,
) {
request.metadata_mut().insert(
"authorization",
format!("Bearer {token}")
.parse()
.context("minted admin token is not valid metadata")?,
);
}
let mut inbound = client.worker_stream(request).await?.into_inner();
let heartbeat = spawn_heartbeat(
sender.clone(),
seaweed-worker/crates/core/src/token.rs
+61
View File
@@ -0,0 +1,61 @@
use std::time::{SystemTime, UNIX_EPOCH};
use aws_lc_rs::hmac;
use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine};
/// Mints the HS256 Bearer token the admin control stream requires when
/// jwt.filer_signing.key is configured, matching the SeaweedFilerAdminClaims
/// the Go worker sends: registered claims only, no session id.
pub fn admin_token(signing_key: &[u8], expires_after_secs: u64) -> Option<String> {
if signing_key.is_empty() {
return None;
}
let claims = if expires_after_secs > 0 {
let exp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or_default()
+ expires_after_secs;
format!("{{\"exp\":{exp}}}")
} else {
"{}".to_string()
};
let signing_input = format!(
"{}.{}",
URL_SAFE_NO_PAD.encode(b"{\"alg\":\"HS256\",\"typ\":\"JWT\"}"),
URL_SAFE_NO_PAD.encode(claims.as_bytes())
);
let key = hmac::Key::new(hmac::HMAC_SHA256, signing_key);
let signature = hmac::sign(&key, signing_input.as_bytes());
Some(format!(
"{signing_input}.{}",
URL_SAFE_NO_PAD.encode(signature.as_ref())
))
}
#[cfg(test)]
mod tests {
use super::admin_token;
#[test]
fn no_key_no_token() {
assert!(admin_token(b"", 0).is_none());
}
#[test]
fn minted_token_verifies_shape() {
let token = admin_token(b"secret", 60).unwrap();
let parts: Vec<&str> = token.split('.').collect();
assert_eq!(parts.len(), 3);
assert!(!parts.iter().any(|p| p.is_empty()));
}
#[test]
fn no_expiry_has_empty_claims() {
let token = admin_token(b"secret", 0).unwrap();
let payload = token.split('.').nth(1).unwrap();
use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine};
let claims = URL_SAFE_NO_PAD.decode(payload).unwrap();
assert_eq!(claims, b"{}");
}
}
seaweed-worker/crates/lance/src/main.rs
+14
View File
@@ -61,6 +61,18 @@ struct Args {
#[arg(long)]
tls_server_name: Option<String>,
/// Shared secret for the admin control stream, the `jwt.filer_signing.key`
/// value the Go worker reads from its config file. Required by admins that
/// authenticate the worker stream with a signing key.
#[arg(long, env = "WEED_FILER_SIGNING_KEY")]
filer_signing_key: Option<String>,
/// Lifetime of the token minted from the signing key, mirroring
/// `jwt.filer_signing.expires_after_seconds`. Zero mints a token with no
/// expiry.
#[arg(long, default_value = "0")]
filer_signing_key_expires_after_seconds: u64,
/// Serve /health, /ready and /metrics on this port, the way
/// `weed worker -metricsPort` does. Zero, the default, serves nothing.
#[arg(long, default_value = "0", env = "WEED_METRICS_PORT")]
@@ -131,6 +143,8 @@ async fn main() -> Result<()> {
max_detection_concurrency: args.max_concurrency,
max_execution_concurrency: args.max_concurrency,
tls,
signing_key: args.filer_signing_key,
signing_key_expires_after_seconds: args.filer_signing_key_expires_after_seconds,
..Default::default()
};
test/fuse_dlm/dlm_concurrent_write_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build linux || darwin
package fuse_dlm
import (
test/fuse_dlm/framework_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build linux || darwin
package fuse_dlm
import (
test/fuse_dlm/posix_lock_ring_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build linux || darwin
package fuse_dlm
import (
test/fuse_p2p/peer_chunk_sharing_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build linux || darwin
package fuse_p2p
import (
unmaintained/check_disk_size/check_disk_size.go
+2
View File
@@ -1,3 +1,5 @@
//go:build !windows && !openbsd && !netbsd && !plan9 && !solaris
package main
import (
weed/admin/dash/worker_grpc_server.go
+4
View File
@@ -188,6 +188,10 @@ func (s *WorkerGrpcServer) Stop() error {
func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStreamServer) error {
ctx := stream.Context()
if err := security.CheckGrpcAdminPeer(ctx, security.SigningKey(util.GetViper().GetString("jwt.filer_signing.key"))); err != nil {
return err
}
// get client address
address := findClientAddress(ctx)
weed/admin/plugin/plugin.go
+6
View File
@@ -14,6 +14,8 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
@@ -267,6 +269,10 @@ func (r *Plugin) Shutdown() {
}
func (r *Plugin) WorkerStream(stream plugin_pb.PluginControlService_WorkerStreamServer) error {
if err := security.CheckGrpcAdminPeer(stream.Context(), security.SigningKey(util.GetViper().GetString("jwt.filer_signing.key"))); err != nil {
return err
}
first, err := stream.Recv()
if err != nil {
if errors.Is(err, io.EOF) {
weed/cluster/master_client.go
+24 -5
View File
@@ -10,10 +10,13 @@ import (
"google.golang.org/grpc"
)
func ListExistingPeerUpdates(master pb.ServerAddress, grpcDialOption grpc.DialOption, filerGroup string, clientType string) (existingNodes []*master_pb.ClusterNodeUpdate) {
// ListExistingPeerUpdates lists the cluster nodes one master currently tracks.
// An empty result is only a successful empty answer; a failed RPC returns the
// error so callers do not mistake "unknown" for "absent".
func ListExistingPeerUpdates(ctx context.Context, master pb.ServerAddress, grpcDialOption grpc.DialOption, filerGroup string, clientType string) (existingNodes []*master_pb.ClusterNodeUpdate, err error) {
if grpcErr := pb.WithMasterClient(context.Background(), false, master, grpcDialOption, false, func(client master_pb.SeaweedClient) error {
resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
err = pb.WithMasterClient(ctx, false, master, grpcDialOption, false, func(client master_pb.SeaweedClient) error {
resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{
ClientType: clientType,
FilerGroup: filerGroup,
})
@@ -31,8 +34,24 @@ func ListExistingPeerUpdates(master pb.ServerAddress, grpcDialOption grpc.DialOp
})
}
return nil
}); grpcErr != nil {
glog.V(0).Infof("connect to %s: %v", master, grpcErr)
})
if err != nil {
glog.V(0).Infof("connect to %s: %v", master, err)
}
return
}
// LookupClusterLeader asks one master for the raft leader it currently sees.
// Any reachable master answers, including followers; it returns "" while the
// cluster is mid-election.
func LookupClusterLeader(ctx context.Context, master pb.ServerAddress, grpcDialOption grpc.DialOption) (leader pb.ServerAddress, err error) {
err = pb.WithMasterClient(ctx, false, master, grpcDialOption, false, func(client master_pb.SeaweedClient) error {
resp, err := client.GetMasterConfiguration(ctx, &master_pb.GetMasterConfigurationRequest{})
if err != nil {
return err
}
leader = pb.ServerAddress(resp.Leader)
return nil
})
return
}
weed/command/filer.go
+9
View File
@@ -382,8 +382,10 @@ func (fo *FilerOptions) startFiler() {
}
}
announce := make(chan struct{})
fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
Masters: fo.masters,
AnnounceCh: announce,
FilerGroup: *fo.filerGroup,
Collection: *fo.collection,
DefaultReplication: *fo.defaultReplicaPlacement,
@@ -500,6 +502,13 @@ func (fo *FilerOptions) startFiler() {
go grpcS.Serve(grpcL)
pb.ServeGrpcOnLocalSocket(grpcS, grpcPort)
// Membership in the master's filer list routes S3 and peer traffic here;
// announce once the gRPC port is serving and the metadata replay has
// caught up with the peers found at startup, so the filer does not take
// lock-ring ownership while a peer's older events can still land on top
// of its fresh writes. A cap bounds the wait.
go fs.AnnounceOnceCaughtUp(announce, time.Now())
// Helper to gracefully stop the gRPC server, waiting for active RPCs.
gracefulTimeout := fo.gracefulStopTimeout
if gracefulTimeout <= 0 {
weed/command/filer_remote_gateway_buckets.go
+8 -1
View File
@@ -359,7 +359,14 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
if message.OldEntry.RemoteEntry != nil && filer.IsSameData(message.OldEntry, message.NewEntry) {
glog.V(2).Infof("update meta: %+v", resp)
oldDest := toRemoteStorageLocation(oldBucket, util.NewFullPath(resp.Directory, message.OldEntry.Name), oldRemoteStorageMountLocation)
return client.UpdateFileMetadata(oldDest, message.OldEntry, message.NewEntry)
if err := client.UpdateFileMetadata(oldDest, message.OldEntry, message.NewEntry); err != nil {
return err
}
remoteEntry := message.NewEntry.RemoteEntry
if remoteEntry == nil {
remoteEntry = message.OldEntry.RemoteEntry
}
return updateLocalEntry(option, message.NewParentPath, message.NewEntry, remoteEntry)
} else {
newDest := toRemoteStorageLocation(newBucket, util.NewFullPath(message.NewParentPath, message.NewEntry.Name), newRemoteStorageMountLocation)
remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewParentPath, message.NewEntry, newDest)
weed/command/filer_remote_sync_dir.go
+10 -4
View File
@@ -363,7 +363,10 @@ func processUpdateEvent(
}
if remoteEntry != nil {
glog.V(2).Infof("update meta: %+v", resp)
return client.UpdateFileMetadata(dest, message.OldEntry, remoteWriteEntry(message.NewEntry, storageClass))
if err := client.UpdateFileMetadata(dest, message.OldEntry, remoteWriteEntry(message.NewEntry, storageClass)); err != nil {
return err
}
return updateLocalEntry(filerClient, message.NewParentPath, message.NewEntry, remoteEntry)
}
glog.V(0).Infof("never replicated, uploading %s", remote_storage.FormatLocation(dest))
}
@@ -735,10 +738,12 @@ func shouldSendToRemote(entry *filer_pb.Entry) bool {
if entry.RemoteEntry == nil {
return true
}
if entry.RemoteEntry.RemoteMtime < entry.Attributes.Mtime {
return true
if entry.RemoteEntry.RemoteMtime != entry.Attributes.Mtime {
return entry.RemoteEntry.RemoteMtime < entry.Attributes.Mtime
}
return false
// Equal seconds cannot tell a rewrite from the version already
// uploaded; the sync stamp can, the same check remote.uncache uses.
return entry.RemoteEntry.LastLocalSyncTsNs < entry.Attributes.Mtime*1e9+int64(entry.Attributes.MtimeNs)
}
// remoteWriteEntry returns the entry as remote storage should see it: the
@@ -767,6 +772,7 @@ func remoteWriteEntry(entry *filer_pb.Entry, storageClass string) *filer_pb.Entr
// in the log and stamps the current entry, so the stale stamp is skipped the
// same way a superseded upload is.
func updateLocalEntry(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.Entry, remoteEntry *filer_pb.RemoteEntry) error {
remoteEntry = proto.Clone(remoteEntry).(*filer_pb.RemoteEntry)
remoteEntry.LastLocalSyncTsNs = time.Now().UnixNano()
expected := proto.Clone(entry).(*filer_pb.Entry)
entry.RemoteEntry = remoteEntry
weed/command/filer_remote_sync_dir_test.go
+108 -25
View File
@@ -421,6 +421,7 @@ type stubFilerClient struct {
updateErr error
lookups int
updates int
updated *filer_pb.UpdateEntryRequest
}
func (c *stubFilerClient) LookupDirectoryEntry(context.Context, *filer_pb.LookupDirectoryEntryRequest, ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
@@ -431,8 +432,9 @@ func (c *stubFilerClient) LookupDirectoryEntry(context.Context, *filer_pb.Lookup
return &filer_pb.LookupDirectoryEntryResponse{Entry: c.entry}, nil
}
func (c *stubFilerClient) UpdateEntry(context.Context, *filer_pb.UpdateEntryRequest, ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) {
func (c *stubFilerClient) UpdateEntry(_ context.Context, req *filer_pb.UpdateEntryRequest, _ ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) {
c.updates++
c.updated = req
if c.updateErr != nil {
return &filer_pb.UpdateEntryResponse{}, c.updateErr
}
@@ -678,9 +680,10 @@ func TestRetriedWriteFileStopsWhenSuperseded(t *testing.T) {
type recordingRemote struct {
remote_storage.RemoteStorageClient
deletes []*remote_pb.RemoteStorageLocation
writes []*remote_pb.RemoteStorageLocation
written [][]byte
deletes []*remote_pb.RemoteStorageLocation
writes []*remote_pb.RemoteStorageLocation
metadataUpdates []*remote_pb.RemoteStorageLocation
written [][]byte
// chunk ids of each written entry: which version of the file went up
writtenChunks [][]string
deleteErr error
@@ -734,16 +737,22 @@ func (r *recordingRemote) DeleteFile(loc *remote_pb.RemoteStorageLocation) error
return r.deleteErr
}
func (r *recordingRemote) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry, newEntry *filer_pb.Entry) error {
r.metadataUpdates = append(r.metadataUpdates, loc)
return nil
}
// TestRenameWithInheritedRemoteEntryWritesNewKey reproduces #11261: a rename
// arrives as an update whose NewEntry carries the RemoteEntry inherited from
// the source. shouldSendToRemote returns false for it (RemoteMtime >= Mtime),
// so the old code skipped the event and never wrote the new key, while the
// filer had already deleted the old object. A path change must always write.
// the source. shouldSendToRemote returns false for it (the sync stamp covers
// the entry's mtime), so the old code skipped the event and never wrote the
// new key, while the filer had already deleted the old object. A path change
// must always write.
func TestRenameWithInheritedRemoteEntryWritesNewKey(t *testing.T) {
const mountedDir = "/buckets"
mountLoc := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/"}
replicated := &filer_pb.RemoteEntry{StorageName: "b2", RemoteETag: "abc", RemoteSize: 2048, RemoteMtime: 1786096669}
replicated := &filer_pb.RemoteEntry{StorageName: "b2", RemoteETag: "abc", RemoteSize: 2048, RemoteMtime: 1786096669, LastLocalSyncTsNs: 1786096669*1e9 + 1}
oldEntry := &filer_pb.Entry{Name: "probe.bin", Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}, RemoteEntry: replicated}
newEntry := &filer_pb.Entry{
Name: "probe.bin",
@@ -1200,9 +1209,9 @@ func TestUpdateLocalEntrySkipsDeletedEntry(t *testing.T) {
// TestSupersededRenameUploadsCurrentEntry: a rename A -> B queued behind a
// rewrite of B. The rename's snapshot is superseded (B's chunks changed) and
// the old key A is already deleted. The rewrite's own event uploads B unless
// shouldSendToRemote skips it on the inherited RemoteEntry (same-second
// mtimes); only then does the rename upload what the filer holds now, so the
// content goes up exactly once.
// the entry already reads synced (the stamp covers its mtime, or the remote
// is newer); only then does the rename upload what the filer holds now, so
// the content goes up exactly once.
func TestSupersededRenameUploadsCurrentEntry(t *testing.T) {
const mountedDir = "/buckets"
mountLoc := &remote_pb.RemoteStorageLocation{Name: "gcs", Bucket: "bucket", Path: "/"}
@@ -1218,9 +1227,9 @@ func TestSupersededRenameUploadsCurrentEntry(t *testing.T) {
wantDelete := &remote_pb.RemoteStorageLocation{Name: "gcs", Bucket: "bucket", Path: "/b/dir/a.txt"}
wantWrite := &remote_pb.RemoteStorageLocation{Name: "gcs", Bucket: "bucket", Path: "/b/dir/b.txt"}
t.Run("rewrite hidden by the inherited stamp: upload once here", func(t *testing.T) {
// RemoteMtime inherited from a.txt equals the rewrite's mtime.
current := entryWith("b.txt", &filer_pb.RemoteEntry{StorageName: "gcs", RemoteMtime: 1786096669, RemoteSize: 1024}, chunk("3,02", "e2"))
t.Run("rewrite already synced: upload once here", func(t *testing.T) {
// The stamp covers the rewrite's mtime, so its own event skips.
current := entryWith("b.txt", &filer_pb.RemoteEntry{StorageName: "gcs", RemoteMtime: 1786096669, RemoteSize: 1024, LastLocalSyncTsNs: 1786096669*1e9 + 1}, chunk("3,02", "e2"))
remote := &recordingRemote{}
filerClient := &stubFilerClient{entry: current}
if err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, resp); err != nil {
@@ -1324,7 +1333,7 @@ func TestSupersededRenameUploadsCurrentEntry(t *testing.T) {
}
})
t.Run("snapshot remote-only but rewritten since: the rewrite's bytes go up, nothing is copied", func(t *testing.T) {
t.Run("snapshot remote-only but rewritten since: the rewrite's own event uploads", func(t *testing.T) {
remoteOnly := &filer_pb.RemoteEntry{StorageName: "gcs", RemoteMtime: 1786096669, RemoteSize: 7}
snapshot := &filer_pb.SubscribeMetadataResponse{
Directory: "/buckets/b/dir",
@@ -1334,21 +1343,17 @@ func TestSupersededRenameUploadsCurrentEntry(t *testing.T) {
NewEntry: entryWith("b.txt", remoteOnly),
},
}
// rewritten with the inherited stamp still hiding it from shouldSendToRemote
current := entryWith("b.txt", remoteOnly, chunk("3,09", "e9"))
// rewritten since the stamp; its own event uploads, not this one
rewritten := proto.Clone(remoteOnly).(*filer_pb.RemoteEntry)
rewritten.LastLocalSyncTsNs = 1
current := entryWith("b.txt", rewritten, chunk("3,09", "e9"))
remote := &recordingRemote{objects: map[string][]byte{"/b/dir/a.txt": []byte("payload")}}
filerClient := &stubFilerClient{entry: current}
if err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, snapshot); err != nil {
t.Fatalf("err = %v, want nil", err)
}
if len(remote.writes) != 1 || !proto.Equal(remote.writes[0], wantWrite) {
t.Fatalf("writes = %+v, want one upload of the rewritten b.txt", remote.writes)
}
if len(remote.writtenChunks[0]) != 1 || remote.writtenChunks[0][0] != "3,09" {
t.Errorf("uploaded chunks = %v, want the rewrite's chunk 3,09, not the chunkless snapshot", remote.writtenChunks[0])
}
if string(remote.written[0]) == "payload" {
t.Error("the old object's bytes were copied over a rewritten entry")
if len(remote.writes) != 0 {
t.Errorf("writes = %+v, want none: the rewrite event behind this one uploads b.txt", remote.writes)
}
if len(remote.deletes) != 1 {
t.Errorf("deletes = %+v, want the old key deleted", remote.deletes)
@@ -1628,3 +1633,81 @@ func (c *pathFilerClient) LookupDirectoryEntry(_ context.Context, req *filer_pb.
func (c *pathFilerClient) WithFilerClient(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error {
return fn(c)
}
func TestShouldSendToRemote(t *testing.T) {
const mtime = 1786096669
tests := []struct {
name string
mtime int64
mtimeNs int32
remote *filer_pb.RemoteEntry
want bool
}{
{name: "no RemoteEntry", mtime: mtime, want: true},
{name: "remote older", mtime: mtime, remote: &filer_pb.RemoteEntry{RemoteMtime: mtime - 1}, want: true},
{name: "remote newer", mtime: mtime, remote: &filer_pb.RemoteEntry{RemoteMtime: mtime + 1}, want: false},
{name: "same second, no sync stamp", mtime: mtime, remote: &filer_pb.RemoteEntry{RemoteMtime: mtime}, want: true},
{name: "same second, synced at the second", mtime: mtime, remote: &filer_pb.RemoteEntry{RemoteMtime: mtime, LastLocalSyncTsNs: mtime * 1e9}, want: false},
{name: "same second, synced after", mtime: mtime, remote: &filer_pb.RemoteEntry{RemoteMtime: mtime, LastLocalSyncTsNs: mtime*1e9 + 1}, want: false},
{name: "same second, rewritten after the sync", mtime: mtime, mtimeNs: 337, remote: &filer_pb.RemoteEntry{RemoteMtime: mtime, LastLocalSyncTsNs: mtime*1e9 + 100}, want: true},
{name: "same second, rewrite synced", mtime: mtime, mtimeNs: 337, remote: &filer_pb.RemoteEntry{RemoteMtime: mtime, LastLocalSyncTsNs: mtime*1e9 + 400}, want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
entry := &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: tt.mtime, MtimeNs: tt.mtimeNs}, RemoteEntry: tt.remote}
if got := shouldSendToRemote(entry); got != tt.want {
t.Errorf("shouldSendToRemote = %v, want %v", got, tt.want)
}
})
}
}
// A metadata-only update pushed to the remote must advance the sync stamp.
// Without the restamp a same-second edit stays dirty forever and every
// following event re-sends the metadata update.
func TestMetadataOnlyUpdateRestampsSyncTs(t *testing.T) {
const mountedDir = "/buckets"
mountLoc := &remote_pb.RemoteStorageLocation{Name: "gcs", Bucket: "bucket", Path: "/"}
replicated := &filer_pb.RemoteEntry{StorageName: "gcs", RemoteETag: "abc", RemoteSize: 2048, RemoteMtime: 1786096669, LastLocalSyncTsNs: 1}
oldEntry := entryWith("a.txt", replicated, chunk("3,01", "e1"))
newEntry := entryWith("a.txt", replicated, chunk("3,01", "e1"))
resp := &filer_pb.SubscribeMetadataResponse{
Directory: "/buckets/b/dir",
EventNotification: &filer_pb.EventNotification{
OldEntry: oldEntry,
NewParentPath: "/buckets/b/dir",
NewEntry: newEntry,
},
}
remote := &recordingRemote{}
filerClient := &stubFilerClient{entry: newEntry}
if err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, resp); err != nil {
t.Fatal(err)
}
if len(remote.metadataUpdates) != 1 {
t.Fatalf("metadata updates = %+v, want 1", remote.metadataUpdates)
}
if len(remote.writes) != 0 {
t.Errorf("writes = %+v, want none: the content did not change", remote.writes)
}
if filerClient.updates != 1 {
t.Fatalf("stamped %d times, want 1: the entry is restamped after the metadata update", filerClient.updates)
}
stamp := filerClient.updated.Entry.GetRemoteEntry().GetLastLocalSyncTsNs()
if stamp <= 1 {
t.Errorf("LastLocalSyncTsNs = %d, want it advanced past the previous stamp", stamp)
}
expected := filerClient.updated.Condition.Clauses[0].ExpectedEntry
if got := expected.GetRemoteEntry().GetLastLocalSyncTsNs(); got != 1 {
t.Errorf("precondition stamp = %d, want the stored stamp 1 or IF_ENTRY_EQUAL rejects the write", got)
}
if got := replicated.GetLastLocalSyncTsNs(); got != 1 {
t.Errorf("shared RemoteEntry stamp = %d, want 1: stamping must not mutate the caller's RemoteEntry", got)
}
if shouldSendToRemote(filerClient.updated.Entry) {
t.Error("the restamped entry should read synced")
}
}
weed/filer/filechunks.go
+6
View File
@@ -79,6 +79,9 @@ func CompactFileChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileId
}
func SeparateGarbageChunks(visibles *IntervalList[*VisibleInterval], chunks []*filer_pb.FileChunk) (compacted []*filer_pb.FileChunk, garbage []*filer_pb.FileChunk) {
if visibles == nil {
return chunks, nil
}
fileIds := make(map[string]bool)
for x := visibles.Front(); x != nil; x = x.Next {
interval := x.Value
@@ -225,6 +228,9 @@ func ViewFromVisibleIntervals(visibles *IntervalList[*VisibleInterval], offset i
}
chunkViews = NewIntervalList[*ChunkView]()
if visibles == nil {
return chunkViews
}
for x := visibles.Front(); x != nil; x = x.Next {
chunk := x.Value
weed/filer/filechunks_test.go
+33
View File
@@ -583,3 +583,36 @@ func TestCompactFileChunks3(t *testing.T) {
t.Fatalf("unexpected compacted: %d", len(compacted))
}
}
func TestSeparateGarbageChunksNilVisibles(t *testing.T) {
chunks := []*filer_pb.FileChunk{
{Offset: 10, Size: 100, FileId: "abc", ModifiedTsNs: 50},
}
compacted, garbage := SeparateGarbageChunks(nil, chunks)
if len(compacted) != 1 || len(garbage) != 0 {
t.Fatalf("expected fallback to original chunks with nil garbage")
}
}
func TestViewFromVisibleIntervalsNilVisibles(t *testing.T) {
views := ViewFromVisibleIntervals(nil, 0, 100)
if views == nil || views.Front() != nil {
t.Fatalf("expected empty interval list")
}
}
func TestViewFromChunksPropagatesError(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
chunks := []*filer_pb.FileChunk{
{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 50, IsChunkManifest: true},
}
views, err := ViewFromChunks(ctx, nil, chunks, 0, math.MaxInt64)
if err == nil {
t.Fatal("expected error on cancelled context")
}
if views != nil {
t.Fatalf("expected nil views on error, got %v", views)
}
}
weed/filer/filer.go
+108 -2
View File
@@ -82,6 +82,11 @@ type Filer struct {
persistedLogCache *persistedLogCache
metaLogInflight metaLogInflight
remoteTombstones *remoteDeletionTombstones
// EntryLockTable serializes mutations per entry path. The filer server
// holds it across local read-modify-write mutations and the meta
// aggregator across a replayed peer change's check-and-apply, so the two
// cannot interleave on the same path.
EntryLockTable *util.LockTable[util.FullPath]
// remoteTombstonesDone, when non-nil, is closed once the startup tombstone
// rebuild finishes; lazy remote reads wait on it so a pending delete
// cannot resurrect in the gap.
@@ -110,6 +115,12 @@ type Filer struct {
// overwriting the unread ledger with a partial set.
deletionLedgerBlocked atomic.Bool
deletionLedgerFlush chan struct{}
// metaTsPending holds per-path commit records whose KV write failed,
// keyed by path; ordering reads consult it until the retry lands them
// (see meta_path_ts.go). metaTsRepairing marks paths with a live
// repair goroutine so failures neither strand a record nor pile on.
metaTsPending sync.Map
metaTsRepairing sync.Map
// flushCtx is the context metadata-log flushes append under. It stays
// live for the process lifetime and is cancelled once during Shutdown,
// bounding every retry - in-flight and queued alike - to one deadline.
@@ -133,6 +144,7 @@ func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerH
DeletionRetryQueue: NewDeletionRetryQueue(),
persistedLogCache: newPersistedLogCache(persistedLogCacheMaxBytes),
remoteTombstones: newRemoteDeletionTombstones(),
EntryLockTable: util.NewLockTable[util.FullPath](),
}
f.flushCtx, f.flushCancel = context.WithCancel(context.Background())
if f.UniqueFilerId < 0 {
@@ -254,8 +266,96 @@ func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*maste
}
func (f *Filer) ListExistingPeerUpdates(ctx context.Context) (existingNodes []*master_pb.ClusterNodeUpdate) {
return cluster.ListExistingPeerUpdates(f.GetMaster(ctx), f.GrpcDialOption, f.MasterClient.FilerGroup, cluster.FilerType)
// peerLookupTimeout bounds each master query so one unresponsive master
// cannot stall the peer scan.
const peerLookupTimeout = 10 * time.Second
// leaderSettleTimeout bounds how long an empty peer list from the leader is
// re-polled: a freshly elected leader has not yet received the filers'
// reconnects, so an immediate empty answer can just mean membership has not
// migrated yet. A persistent empty answer is then checked against the other
// masters below before it is accepted as an empty cluster.
const leaderSettleTimeout = 10 * time.Second
const leaderSettleInterval = time.Second
// ListExistingPeerUpdates returns the peer list to bootstrap from. The
// leader's list is authoritative: after a leader change an old master keeps
// its registrations until its streams end, so the longest answer is not
// necessarily the current one. The leader's answer wins whenever it can be
// found and queried; only when the leader is unreachable does the scan fall
// back to the fullest answer from the other masters. It returns an error when
// no master answers, so a failed lookup is not mistaken for an empty cluster.
func (f *Filer) ListExistingPeerUpdates(ctx context.Context) (existingNodes []*master_pb.ClusterNodeUpdate, err error) {
masters := f.MasterClient.ListMasters()
var queryErrs []error
var leader pb.ServerAddress
for _, master := range masters {
if ctx.Err() != nil {
break
}
lookupCtx, cancel := context.WithTimeout(ctx, peerLookupTimeout)
l, lookErr := cluster.LookupClusterLeader(lookupCtx, master, f.GrpcDialOption)
cancel()
if lookErr != nil {
queryErrs = append(queryErrs, lookErr)
continue
}
if l != "" {
leader = l
break
}
}
if leader != "" {
settleDeadline := time.Now().Add(leaderSettleTimeout)
for {
lookupCtx, cancel := context.WithTimeout(ctx, peerLookupTimeout)
nodes, listErr := cluster.ListExistingPeerUpdates(lookupCtx, leader, f.GrpcDialOption, f.MasterClient.FilerGroup, cluster.FilerType)
cancel()
if listErr != nil {
queryErrs = append(queryErrs, fmt.Errorf("list peers from leader %s: %w", leader, listErr))
break
}
if len(nodes) > 0 {
return nodes, nil
}
if ctx.Err() != nil || !time.Now().Before(settleDeadline) {
break
}
select {
case <-ctx.Done():
case <-time.After(leaderSettleInterval):
}
}
}
answered := false
for _, master := range masters {
if ctx.Err() != nil {
break
}
lookupCtx, cancel := context.WithTimeout(ctx, peerLookupTimeout)
nodes, listErr := cluster.ListExistingPeerUpdates(lookupCtx, master, f.GrpcDialOption, f.MasterClient.FilerGroup, cluster.FilerType)
cancel()
if listErr != nil {
queryErrs = append(queryErrs, listErr)
continue
}
answered = true
if len(nodes) > len(existingNodes) {
existingNodes = nodes
}
}
if !answered {
if ctxErr := ctx.Err(); ctxErr != nil {
return nil, ctxErr
}
if err = errors.Join(queryErrs...); err == nil {
err = fmt.Errorf("no masters configured")
}
return nil, fmt.Errorf("list existing peers: %w", err)
}
return existingNodes, nil
}
func (f *Filer) SetStore(store FilerStore) (isFresh bool) {
@@ -825,6 +925,12 @@ func (f *Filer) Shutdown() {
f.LocalMetaLogBuffer.ShutdownLogBuffer()
// The final metadata-log flush still needs the store to append its entry.
f.LocalMetaLogBuffer.WaitForShutdown()
// Stop peer subscriptions and let each persist its last applied offset;
// the replay on the next start otherwise resumes from a stale offset and
// applies those old events over newer local state.
if f.MetaAggregator != nil {
f.MetaAggregator.Shutdown()
}
// Persist the deletion ledger one last time before the store closes, so a
// clean shutdown leaves the recovery set exactly consistent with reality.
f.snapshotDeletionLedger()
weed/filer/filer_lazy_remote_test.go
+10
View File
@@ -35,6 +35,7 @@ type stubFilerStore struct {
kv map[string][]byte
insertErr error
findErr error
kvPutErr error
kvGetErr error
kvDeleteErr error
kvGetHook func(key []byte)
@@ -49,6 +50,12 @@ func newStubFilerStore() *stubFilerStore {
}
}
func (s *stubFilerStore) setKvPutErr(err error) {
s.mu.Lock()
defer s.mu.Unlock()
s.kvPutErr = err
}
func (s *stubFilerStore) GetName() string { return "stub" }
func (s *stubFilerStore) Initialize(util.Configuration, string) error { return nil }
func (s *stubFilerStore) Shutdown() {}
@@ -60,6 +67,9 @@ func (s *stubFilerStore) RollbackTransaction(context.Context) error { return nil
func (s *stubFilerStore) KvPut(_ context.Context, key []byte, value []byte) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.kvPutErr != nil {
return s.kvPutErr
}
s.kv[string(key)] = append([]byte(nil), value...)
return nil
}
weed/filer/filer_notify.go
+4
View File
@@ -71,6 +71,10 @@ func (f *Filer) notifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry
}
f.logMetaEvent(ctx, event)
// Record the commit timestamp so a peer replay ordered by the entry lock
// this mutation may still be under sees the local change, including as a
// tombstone after the entry itself is gone.
f.noteCommittedEvent(event)
if sink := metadataEventSinkFromContext(ctx); sink != nil {
sink.Record(event)
}
weed/filer/meta_aggregator.go
+211 -54
View File
@@ -63,6 +63,11 @@ type MetaAggregator struct {
flushAdvanced chan struct{}
remotePeerArrived chan struct{}
peerWatermarksLock sync.Mutex
peerWg sync.WaitGroup
// stopped is set under peerChansLock by Shutdown; OnPeerUpdate then
// rejects adds so no subscription can outlive the shutdown wait and run
// against a store Filer.Shutdown has already closed.
stopped bool
}
// MetaAggregator only aggregates data "on the fly". The logs are not re-persisted to disk.
@@ -87,6 +92,13 @@ func (ma *MetaAggregator) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, star
ma.peerChansLock.Lock()
defer ma.peerChansLock.Unlock()
// After Shutdown the master connection may still deliver an update; a
// subscription started now would have no stop signal and could run past
// the store close.
if ma.stopped {
return
}
address := pb.ServerAddress(update.Address)
if update.IsAdd {
// the peer is already followed, restarting would only lose the events
@@ -102,7 +114,11 @@ func (ma *MetaAggregator) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, star
// Account for the peer before its stream signals; keep prior values
// on reconnect.
ma.initPeerWatermark(address)
go ma.loopSubscribeToOneFiler(ma.filer, ma.self, address, startFrom, stopChan)
ma.peerWg.Add(1)
go func() {
defer ma.peerWg.Done()
ma.loopSubscribeToOneFiler(ma.filer, ma.self, address, startFrom, stopChan)
}()
} else {
if prevChan, found := ma.peerChans[address]; found {
close(prevChan)
@@ -273,6 +289,21 @@ func lowWatermarkOf(watermarks map[pb.ServerAddress]int64) int64 {
return low
}
// PeersCaughtUpTo reports whether every listed peer's stream has delivered
// an event or idle heartbeat at or past tsNs, i.e. no pre-tsNs backlog can
// still arrive from it. A peer no longer tracked counts as caught up: it
// cannot deliver a backlog anymore.
func (ma *MetaAggregator) PeersCaughtUpTo(peers []pb.ServerAddress, tsNs int64) bool {
ma.peerWatermarksLock.Lock()
defer ma.peerWatermarksLock.Unlock()
for _, peer := range peers {
if ts, found := ma.peerWatermarks[peer]; found && ts < tsNs {
return false
}
}
return true
}
func (ma *MetaAggregator) HasRemotePeers() bool {
ma.peerChansLock.Lock()
defer ma.peerChansLock.Unlock()
@@ -329,6 +360,42 @@ func (ma *MetaAggregator) HasPeer(address pb.ServerAddress) bool {
return false
}
// Shutdown stops every peer subscription and waits for each to return,
// letting the deferred offset save persist the last applied position before
// the store closes. Without it a graceful restart resumes from the last
// periodically-saved offset — up to a minute stale — and replays those
// events over newer local state.
func (ma *MetaAggregator) Shutdown() {
ma.peerChansLock.Lock()
ma.stopped = true
chans := ma.peerChans
ma.peerChans = make(map[pb.ServerAddress]chan struct{})
for _, stopChan := range chans {
close(stopChan)
}
ma.peerChansLock.Unlock()
// Wait without a cap: every blocking point in the subscribe loop is
// stop-aware, so a peer still running here is genuinely wedged and
// returning anyway would let Filer.Shutdown close the store under it and
// skip its deferred offset save.
done := make(chan struct{})
go func() {
ma.peerWg.Wait()
close(done)
}()
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
for {
select {
case <-done:
return
case <-ticker.C:
glog.Warningf("meta aggregator shutdown: peer subscriptions still stopping")
}
}
}
func (ma *MetaAggregator) loopSubscribeToOneFiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress, startFrom time.Time, stopChan chan struct{}) {
lastTsNs := startFrom.UnixNano()
for {
@@ -353,7 +420,12 @@ func (ma *MetaAggregator) loopSubscribeToOneFiler(f *Filer, self pb.ServerAddres
if lastTsNs < nextLastTsNs {
lastTsNs = nextLastTsNs
}
time.Sleep(1733 * time.Millisecond)
select {
case <-stopChan:
glog.V(0).Infof("stop subscribing peer %s meta change", peer)
return
case <-time.After(1733 * time.Millisecond):
}
}
}
@@ -369,11 +441,34 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
*/
var maybeReplicateMetadataChange func(*filer_pb.SubscribeMetadataResponse)
var maybeReplicateMetadataChange func(*filer_pb.SubscribeMetadataResponse) error
lastPersistTime := time.Now()
lastTsNs := startFrom
// replayFloorTsNs is the earliest event this session gave up replaying.
// The persisted offset and the reconnect resume point must stay behind
// it so the event is redelivered on the next subscribe instead of being
// skipped forever.
var replayFloorTsNs int64
persistableTsNs := func(tsNs int64) int64 {
if replayFloorTsNs > 0 && replayFloorTsNs-1 < tsNs {
return replayFloorTsNs - 1
}
return tsNs
}
peerSignature, err := ma.readFilerStoreSignature(peer)
// Everything below derives from ctx, cancelled when the peer is removed
// or the aggregator shuts down, so no stage can block past its stop.
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go func() {
select {
case <-stopChan:
cancel()
case <-ctx.Done():
}
}()
peerSignature, err := ma.readFilerStoreSignature(ctx, peer)
if err != nil {
return lastTsNs, fmt.Errorf("connecting to peer filer %s: %v", peer, err)
}
@@ -393,7 +488,7 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
// different concept and can be set to arbitrary values.
preTraverseTime := time.Now()
glog.V(0).Infof("no previous offset for peer %s, starting full metadata sync", peer)
if traverseErr := ma.traversePeerMetadata(f, peer); traverseErr != nil {
if traverseErr := ma.traversePeerMetadata(ctx, f, peer); traverseErr != nil {
return lastTsNs, fmt.Errorf("initial metadata sync from %s: %v", peer, traverseErr)
}
lastTsNs = preTraverseTime.Add(-time.Minute).UnixNano()
@@ -405,11 +500,12 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
return lastTsNs, fmt.Errorf("read offset for peer %s: %w", peer, err)
}
defer func(prevTsNs int64) {
if lastTsNs != prevTsNs && lastTsNs != lastPersistTime.UnixNano() {
if err := ma.updateOffset(f, peer, peerSignature, lastTsNs); err == nil {
glog.V(0).Infof("last sync time with %s at %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
saveTsNs := persistableTsNs(lastTsNs)
if saveTsNs != prevTsNs && lastTsNs != lastPersistTime.UnixNano() {
if err := ma.updateOffset(f, peer, peerSignature, saveTsNs); err == nil {
glog.V(0).Infof("last sync time with %s at %v (%d)", peer, time.Unix(0, saveTsNs), saveTsNs)
} else {
glog.Errorf("failed to save last sync time with %s at %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
glog.Errorf("failed to save last sync time with %s at %v (%d)", peer, time.Unix(0, saveTsNs), saveTsNs)
}
}
}(lastTsNs)
@@ -417,11 +513,23 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
glog.V(0).Infof("follow peer: %v, last %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
var counter int64
var synced bool
maybeReplicateMetadataChange = func(event *filer_pb.SubscribeMetadataResponse) {
replicateMetadataChange(f.Store, peer, event)
maybeReplicateMetadataChange = func(event *filer_pb.SubscribeMetadataResponse) error {
if err := replicateMetadataChange(f, peer, event, stopChan); err != nil {
if errors.Is(err, errPeerSubscriptionStopped) {
return err
}
// A failed event becomes a barrier the persisted offset and
// the reconnect resume point cannot pass, while the stream
// keeps flowing: blocking on an event that can never replay
// would stall every later event from this peer.
if replayFloorTsNs == 0 || event.TsNs < replayFloorTsNs {
replayFloorTsNs = event.TsNs
}
return nil
}
counter++
if lastPersistTime.Add(time.Minute).Before(time.Now()) {
if err := ma.updateOffset(f, peer, peerSignature, event.TsNs); err == nil {
if err := ma.updateOffset(f, peer, peerSignature, persistableTsNs(event.TsNs)); err == nil {
if event.TsNs < time.Now().Add(-2*time.Minute).UnixNano() {
glog.V(0).Infof("sync with %s progressed to: %v %0.2f/sec", peer, time.Unix(0, event.TsNs), float64(counter)/60.0)
} else if !synced {
@@ -434,6 +542,7 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
glog.V(0).Infof("failed to update offset for %v: %v", peer, err)
}
}
return nil
}
}
@@ -450,7 +559,9 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
return err
}
if maybeReplicateMetadataChange != nil {
maybeReplicateMetadataChange(event)
if err := maybeReplicateMetadataChange(event); err != nil {
return err
}
}
return nil
}
@@ -461,17 +572,8 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
glog.V(0).Infof("subscribing remote %s meta change: %v, clientId:%d", peer, time.Unix(0, lastTsNs), ma.filer.UniqueFilerId)
err = pb.WithFilerClient(true, 0, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Stop the stream promptly on removal; the blocking Recv below would
// otherwise only notice when the stream errors on its own.
go func() {
select {
case <-stopChan:
cancel()
case <-ctx.Done():
}
}()
// ctx is cancelled on stopChan by the watch above; the blocking Recv
// below stops promptly on removal or shutdown.
atomic.AddInt32(&ma.filer.UniqueFilerEpoch, 1)
// Construct a log file reader that reads chunks via the peer filer's LookupVolume.
lookupFn := LookupFn(filerClient{client})
@@ -496,6 +598,14 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
}
processOne := func(event *filer_pb.SubscribeMetadataResponse) error {
// Bail promptly on stop; a cancelled stream may still deliver
// queued events while unwinding, and replaying them now only
// delays the offset save the shutdown is waiting on.
select {
case <-stopChan:
return errPeerSubscriptionStopped
default:
}
if err := processEventFn(event); err != nil {
glog.V(0).Infof("SubscribeLocalMetadata process %v: %v", event, err)
return fmt.Errorf("process %v: %w", event, err)
@@ -572,19 +682,37 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
}
}
})
return lastTsNs, err
return persistableTsNs(lastTsNs), err
}
// replicateMetadataChange retries transient Replay failures with bounded
// backoff. A failure that outlives the retry budget is counted and logged,
// then skipped: blocking on an event that can never replay would stall every
// later event from this peer, which is worse than one entry staying stale.
func replicateMetadataChange(store FilerStore, peer pb.ServerAddress, event *filer_pb.SubscribeMetadataResponse) {
err := util.Retry("replicate metadata change from "+string(peer), func() error {
return Replay(store, event)
})
if err == nil {
return
// backoff and reports the outcome. errPeerSubscriptionStopped is returned
// when the subscription stops mid-retry so the caller aborts and the offset
// save keeps the last completed event. A failure that outlives the retry
// budget is counted and logged, then returned: the caller pins the peer's
// offset at the event so a restart replays it rather than skipping it.
func replicateMetadataChange(f *Filer, peer pb.ServerAddress, event *filer_pb.SubscribeMetadataResponse, stopChan <-chan struct{}) error {
waitTime := time.Second
var err error
for waitTime < util.RetryWaitTime {
select {
case <-stopChan:
return errPeerSubscriptionStopped
default:
}
if err = f.Replay(event); err == nil {
return nil
}
if !util.IsTransientError(err) {
break
}
glog.V(0).Infof("retry replicate metadata change from %s: %v", peer, err)
select {
case <-stopChan:
return errPeerSubscriptionStopped
case <-time.After(waitTime):
}
waitTime += waitTime / 2
}
stats.FilerMetaAggregatorReplayFailures.WithLabelValues(string(peer)).Inc()
name := event.GetEventNotification().GetNewEntry().GetName()
@@ -592,15 +720,20 @@ func replicateMetadataChange(store FilerStore, peer pb.ServerAddress, event *fil
name = event.GetEventNotification().GetOldEntry().GetName()
}
glog.Errorf("giving up replicating metadata change from %s for %s/%s (ts=%d): %v", peer, event.Directory, name, event.TsNs, err)
return err
}
// errPeerSubscriptionStopped ends event processing once the subscription's
// stop channel closed.
var errPeerSubscriptionStopped = errors.New("peer subscription stopped")
// traversePeerMetadata does a full BFS traversal of a peer filer's metadata
// and inserts all entries into the local store. This is used when a filer
// connects to a peer for the first time and needs to bootstrap pre-existing data.
func (ma *MetaAggregator) traversePeerMetadata(f *Filer, peer pb.ServerAddress) error {
// The ctx cancels on the peer's stop channel so a shutdown does not have to
// out-wait a long traversal.
func (ma *MetaAggregator) traversePeerMetadata(ctx context.Context, f *Filer, peer pb.ServerAddress) error {
return pb.WithFilerClient(true, 0, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
stream, err := client.TraverseBfsMetadata(ctx, &filer_pb.TraverseBfsMetadataRequest{
Directory: "/",
ExcludedPrefixes: []string{SystemLogDir},
@@ -620,22 +753,17 @@ func (ma *MetaAggregator) traversePeerMetadata(f *Filer, peer pb.ServerAddress)
if resp.Entry == nil {
continue
}
fullpath := util.Join(resp.Directory, resp.Entry.Name)
entry := FromPbEntry(resp.Directory, resp.Entry)
if insertErr := f.Store.InsertEntry(context.Background(), entry); insertErr != nil {
// Entry may already exist (root dir, or partial previous bootstrap).
existing, findErr := f.Store.FindEntry(context.Background(), entry.FullPath)
if findErr != nil {
return fmt.Errorf("insert entry %s: %w", fullpath, insertErr)
}
// Only overwrite if the peer's entry is newer.
if entry.Attr.Mtime.After(existing.Attr.Mtime) {
if updateErr := f.Store.UpdateEntry(context.Background(), entry); updateErr != nil {
return fmt.Errorf("update entry %s: %w", fullpath, updateErr)
}
} else {
glog.V(1).Infof("skip older peer entry %s (peer mtime %v <= local mtime %v)", fullpath, entry.Attr.Mtime, existing.Attr.Mtime)
}
// Order on the sender's per-path commit timestamp, not the
// entry's mtime — mtime is client-settable and does not track
// when the peer committed the change. Peers that predate the
// field send 0 and fall back to mtime.
peerTsNs := resp.EntryTsNs
if peerTsNs == 0 {
peerTsNs = entry.Attr.Mtime.UnixNano()
}
if applyErr := f.applyTraversedEntry(ctx, entry, peerTsNs); applyErr != nil {
return applyErr
}
count++
if count%10000 == 0 {
@@ -647,9 +775,38 @@ func (ma *MetaAggregator) traversePeerMetadata(f *Filer, peer pb.ServerAddress)
})
}
func (ma *MetaAggregator) readFilerStoreSignature(peer pb.ServerAddress) (sig int32, err error) {
// applyTraversedEntry merges one bootstrap entry under the path's replay
// locks, keeping local state when its commit is newer than the sender's —
// most importantly a delete, whose tombstone must not let the bootstrap
// resurrect the path. An applied entry records the sender's commit time so a
// later peer replay orders against it rather than the entry's mtime.
func (f *Filer) applyTraversedEntry(ctx context.Context, entry *Entry, peerTsNs int64) error {
unlock := f.lockReplayPaths(entry.FullPath, "")
defer unlock()
localTsNs, err := f.localCommitTsNs(ctx, entry.FullPath)
if err != nil {
return fmt.Errorf("check local state for %s: %w", entry.FullPath, err)
}
if localTsNs >= peerTsNs {
return nil
}
if insertErr := f.Store.InsertEntry(ctx, entry); insertErr != nil {
// Entry already exists (root dir, or partial previous bootstrap);
// the lock kept the check above current, so the peer commit is newer.
if _, findErr := f.Store.FindEntry(ctx, entry.FullPath); findErr != nil {
return fmt.Errorf("insert entry %s: %w", entry.FullPath, insertErr)
}
if updateErr := f.Store.UpdateEntry(ctx, entry); updateErr != nil {
return fmt.Errorf("update entry %s: %w", entry.FullPath, updateErr)
}
}
f.notePathChange(entry.FullPath, peerTsNs, false)
return nil
}
func (ma *MetaAggregator) readFilerStoreSignature(ctx context.Context, peer pb.ServerAddress) (sig int32, err error) {
err = pb.WithFilerClient(false, 0, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
resp, err := client.GetFilerConfiguration(ctx, &filer_pb.GetFilerConfigurationRequest{})
if err != nil {
return err
}
weed/filer/meta_aggregator_replay_test.go
+601 -2
View File
@@ -15,6 +15,14 @@ import (
"github.com/seaweedfs/seaweedfs/weed/util"
)
// newReplayTestFiler builds the minimum Filer Replay needs over a stub store.
func newReplayTestFiler(store FilerStore) *Filer {
return &Filer{
Store: NewFilerStoreWrapper(store),
EntryLockTable: util.NewLockTable[util.FullPath](),
}
}
// flakyReplayStore fails InsertEntry for its first failUntil calls, then
// delegates to the embedded stubFilerStore.
type flakyReplayStore struct {
@@ -73,8 +81,11 @@ func TestReplicateMetadataChangeRetriesTransientFailure(t *testing.T) {
err: errors.New("i/o timeout talking to store"),
}
peer := pb.ServerAddress("peer-transient:1")
f := newReplayTestFiler(store)
replicateMetadataChange(store, peer, quotaChangeEvent("my-bucket", wantQuota))
if err := replicateMetadataChange(f, peer, quotaChangeEvent("my-bucket", wantQuota), nil); err != nil {
t.Fatalf("replicateMetadataChange: %v", err)
}
inserted, err := store.FindEntry(context.Background(), util.NewFullPath("/buckets", "my-bucket"))
if err != nil {
@@ -97,10 +108,14 @@ func TestReplicateMetadataChangeGivesUpLoudlyOnPermanentFailure(t *testing.T) {
err: errors.New("entry checksum mismatch"),
}
peer := pb.ServerAddress("peer-permanent:1")
f := newReplayTestFiler(store)
before := replayFailureCount(t, peer)
replicateMetadataChange(store, peer, quotaChangeEvent("poison-bucket", 65536<<20))
err := replicateMetadataChange(f, peer, quotaChangeEvent("poison-bucket", 65536<<20), nil)
if err == nil {
t.Fatal("expected the exhausted replay error so the caller can pin the peer offset at the failed event")
}
if got := store.attemptCount(); got != 1 {
t.Fatalf("attempts = %d, want exactly 1: a non-transient error must fail fast, not retry", got)
@@ -109,3 +124,587 @@ func TestReplicateMetadataChangeGivesUpLoudlyOnPermanentFailure(t *testing.T) {
t.Fatalf("FilerMetaAggregatorReplayFailures[%s] = %v, want %v", peer, got, before+1)
}
}
func overwriteEvent(dir, name string, tsNs int64, content string) *filer_pb.SubscribeMetadataResponse {
return &filer_pb.SubscribeMetadataResponse{
Directory: dir,
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: name},
NewEntry: &filer_pb.Entry{Name: name, Content: []byte(content)},
},
TsNs: tsNs,
}
}
func deleteEvent(dir, name string, tsNs int64) *filer_pb.SubscribeMetadataResponse {
return &filer_pb.SubscribeMetadataResponse{
Directory: dir,
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: name},
},
TsNs: tsNs,
}
}
func TestReplaySkipsEventOlderThanLocalEntry(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
now := time.Now()
path := util.NewFullPath("/dir", "file")
store.entries[string(path)] = &Entry{
FullPath: path,
Attr: Attr{Mtime: now},
Content: []byte("newer local write"),
}
if err := f.Replay(overwriteEvent("/dir", "file", now.Add(-time.Minute).UnixNano(), "old replayed version")); err != nil {
t.Fatalf("Replay: %v", err)
}
entry, err := store.FindEntry(context.Background(), path)
if err != nil {
t.Fatalf("FindEntry: %v", err)
}
if string(entry.Content) != "newer local write" {
t.Fatalf("content = %q, want local write kept", entry.Content)
}
}
func TestReplayAppliesEventNewerThanLocalEntry(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
now := time.Now()
path := util.NewFullPath("/dir", "file")
store.entries[string(path)] = &Entry{
FullPath: path,
Attr: Attr{Mtime: now.Add(-time.Minute)},
Content: []byte("old local write"),
}
if err := f.Replay(overwriteEvent("/dir", "file", now.UnixNano(), "newer peer write")); err != nil {
t.Fatalf("Replay: %v", err)
}
entry, err := store.FindEntry(context.Background(), path)
if err != nil {
t.Fatalf("FindEntry: %v", err)
}
if string(entry.Content) != "newer peer write" {
t.Fatalf("content = %q, want peer write applied", entry.Content)
}
}
func TestReplaySkipsDeleteOlderThanLocalEntry(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
now := time.Now()
path := util.NewFullPath("/dir", "file")
store.entries[string(path)] = &Entry{FullPath: path, Attr: Attr{Mtime: now}}
if err := f.Replay(deleteEvent("/dir", "file", now.Add(-time.Minute).UnixNano())); err != nil {
t.Fatalf("Replay: %v", err)
}
if _, err := store.FindEntry(context.Background(), path); err != nil {
t.Fatalf("newer local entry must survive a stale delete: %v", err)
}
}
func TestReplayAppliesNewerDelete(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
now := time.Now()
path := util.NewFullPath("/dir", "file")
store.entries[string(path)] = &Entry{FullPath: path, Attr: Attr{Mtime: now.Add(-time.Minute)}}
if err := f.Replay(deleteEvent("/dir", "file", now.UnixNano())); err != nil {
t.Fatalf("Replay: %v", err)
}
if _, err := store.FindEntry(context.Background(), path); err == nil {
t.Fatal("newer delete must remove the local entry")
}
}
// A move event touches two paths; the local check applies to each side
// independently, so a newer destination is kept while the source is removed.
func TestReplayMoveChecksEachPathSeparately(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
now := time.Now()
oldPath := util.NewFullPath("/a", "file")
newPath := util.NewFullPath("/b", "file")
store.entries[string(oldPath)] = &Entry{FullPath: oldPath, Attr: Attr{Mtime: now.Add(-time.Minute)}}
store.entries[string(newPath)] = &Entry{FullPath: newPath, Attr: Attr{Mtime: now}, Content: []byte("kept")}
event := &filer_pb.SubscribeMetadataResponse{
Directory: "/a",
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "file"},
NewEntry: &filer_pb.Entry{Name: "file"},
NewParentPath: "/b",
},
TsNs: now.Add(-time.Minute).UnixNano(),
}
if err := f.Replay(event); err != nil {
t.Fatalf("Replay: %v", err)
}
if _, err := store.FindEntry(context.Background(), oldPath); err == nil {
t.Fatal("old source entry should be deleted")
}
entry, err := store.FindEntry(context.Background(), newPath)
if err != nil {
t.Fatalf("FindEntry: %v", err)
}
if string(entry.Content) != "kept" {
t.Fatalf("content = %q, want newer destination kept", entry.Content)
}
}
// A local delete leaves a recorded tombstone, so a replayed create older than
// the delete must not resurrect the path even though no entry remains to
// compare against.
func TestReplaySkipsCreateOlderThanLocalDelete(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
now := time.Now()
path := util.NewFullPath("/dir", "file")
// The delete removed the entry; its commit record survives.
f.notePathChange(path, now.UnixNano(), true)
createEvent := &filer_pb.SubscribeMetadataResponse{
Directory: "/dir",
EventNotification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "file", Content: []byte("stale resurrect")},
},
TsNs: now.Add(-time.Minute).UnixNano(),
}
if err := f.Replay(createEvent); err != nil {
t.Fatalf("Replay: %v", err)
}
if _, err := store.FindEntry(context.Background(), path); err == nil {
t.Fatal("stale replayed create resurrected a locally deleted entry")
}
}
// A deleted directory covers its children: a bulk drop emits no per-child
// events, so the tombstone on the ancestor is what a replayed child create
// must check.
func TestReplaySkipsCreateUnderDeletedAncestor(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
now := time.Now()
f.notePathChange("/dir", now.UnixNano(), true)
createEvent := &filer_pb.SubscribeMetadataResponse{
Directory: "/dir",
EventNotification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "child", Content: []byte("stale")},
},
TsNs: now.Add(-time.Minute).UnixNano(),
}
if err := f.Replay(createEvent); err != nil {
t.Fatalf("Replay: %v", err)
}
if _, err := store.FindEntry(context.Background(), "/dir/child"); err == nil {
t.Fatal("stale replayed create resurrected a child of a deleted directory")
}
}
// A recorded commit timestamp orders replay, not the entry's mtime: a rename
// keeps the source's mtime on the destination, and a mounted client can set
// mtime arbitrarily, so an entry whose mtime postdates the event must still
// be overwritten when its commit record is older.
func TestReplayAppliesWhenCommitRecordIsOlderThanFutureMtime(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
now := time.Now()
path := util.NewFullPath("/b", "x")
store.entries[string(path)] = &Entry{
FullPath: path,
Attr: Attr{Mtime: now.Add(time.Hour)}, // future mtime, e.g. client-set or kept from a rename source
Content: []byte("old destination"),
}
// The last committed change predates the event.
f.notePathChange(path, now.Add(-time.Minute).UnixNano(), false)
event := &filer_pb.SubscribeMetadataResponse{
Directory: "/a",
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "x"},
NewEntry: &filer_pb.Entry{Name: "x", Content: []byte("renamed in")},
NewParentPath: "/b",
},
TsNs: now.UnixNano(),
}
if err := f.Replay(event); err != nil {
t.Fatalf("Replay: %v", err)
}
entry, err := store.FindEntry(context.Background(), path)
if err != nil {
t.Fatalf("FindEntry: %v", err)
}
if string(entry.Content) != "renamed in" {
t.Fatalf("content = %q, want the rename destination applied", entry.Content)
}
}
// A store read failure is not a not-found: the replayed change must not be
// applied on the guess that no newer local entry exists.
func TestReplayPropagatesLookupError(t *testing.T) {
store := newStubFilerStore()
store.findErr = errors.New("store read failed")
f := newReplayTestFiler(store)
err := f.Replay(overwriteEvent("/dir", "file", time.Now().UnixNano(), "v"))
if err == nil {
t.Fatal("expected Replay to propagate the FindEntry failure")
}
}
// The check and the mutation share the per-path entry lock, so a local write
// holding that lock cannot be split by a replayed change.
func TestReplaySerializesWithEntryLock(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
now := time.Now()
path := util.NewFullPath("/dir", "file")
store.entries[string(path)] = &Entry{FullPath: path, Attr: Attr{Mtime: now}}
// A local mutation holding the path lock; a replay started behind it
// sees the recorded commit and skips.
held := f.EntryLockTable.AcquireLock("test local write", path, util.ExclusiveLock)
f.notePathChange(path, now.UnixNano(), false)
done := make(chan error, 1)
go func() {
done <- f.Replay(overwriteEvent("/dir", "file", now.Add(-time.Minute).UnixNano(), "stale"))
}()
select {
case <-done:
t.Fatal("replay finished while the path lock was still held")
case <-time.After(50 * time.Millisecond):
}
f.EntryLockTable.ReleaseLock(path, held)
if err := <-done; err != nil {
t.Fatalf("Replay: %v", err)
}
entry, err := store.FindEntry(context.Background(), path)
if err != nil {
t.Fatalf("FindEntry: %v", err)
}
if entry.Content != nil {
t.Fatalf("content = %q, want local entry kept", entry.Content)
}
}
// A closed stop channel must abort the retry loop with
// errPeerSubscriptionStopped so the subscription tears down and the offset
// save keeps the last completed event instead of skipping this one.
func TestReplicateMetadataChangeAbortsOnStop(t *testing.T) {
store := &flakyReplayStore{
stubFilerStore: newStubFilerStore(),
failUntil: 1 << 30,
err: errors.New("i/o timeout talking to store"),
}
peer := pb.ServerAddress("peer-stopped:1")
f := newReplayTestFiler(store)
stopChan := make(chan struct{})
close(stopChan)
err := replicateMetadataChange(f, peer, quotaChangeEvent("stuck-bucket", 1<<20), stopChan)
if !errors.Is(err, errPeerSubscriptionStopped) {
t.Fatalf("err = %v, want errPeerSubscriptionStopped", err)
}
if got := store.attemptCount(); got != 0 {
t.Fatalf("attempts = %d, want 0: a stopped subscription must not replay", got)
}
}
// Commit-record keys must stay small: SQL KV stores keep the key suffix in a
// VARCHAR(766) column, so a raw path over ~578 bytes silently drops the
// record (the insert fails and the fallback update touches no row).
func TestMetaPathKeyFixedSizeForLongPaths(t *testing.T) {
path := util.NewFullPath("/", string(make([]rune, 600)))
for _, p := range []util.FullPath{"/a", path} {
if got := len(metaPathKey(metaPathTsPrefix, p)); got != len(metaPathTsPrefix)+16 {
t.Fatalf("metaPathKey(%d-byte path) = %d bytes, want %d", len(p), got, len(metaPathTsPrefix)+16)
}
}
}
// Exact-path locking alone lets a replayed child create slip between a local
// ancestor delete's tombstone check and the sweep: replay locks every
// ancestor too, so it waits for the delete's lock and then sees the
// tombstone.
func TestReplaySerializesWithAncestorLock(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
now := time.Now()
child := util.NewFullPath("/dir", "file")
held := f.EntryLockTable.AcquireLock("test local delete", "/dir", util.ExclusiveLock)
f.notePathChange("/dir", now.UnixNano(), true)
createEvent := &filer_pb.SubscribeMetadataResponse{
Directory: "/dir",
EventNotification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "file", Content: []byte("stale resurrect")},
},
TsNs: now.Add(-time.Minute).UnixNano(),
}
done := make(chan error, 1)
go func() {
done <- f.Replay(createEvent)
}()
select {
case <-done:
t.Fatal("replay finished while the ancestor lock was still held")
case <-time.After(50 * time.Millisecond):
}
f.EntryLockTable.ReleaseLock("/dir", held)
if err := <-done; err != nil {
t.Fatalf("Replay: %v", err)
}
if _, err := store.FindEntry(context.Background(), child); err == nil {
t.Fatal("stale replayed create resurrected a child of a locally deleted directory")
}
}
// Bootstrap ordering uses the sender's commit timestamp: a local commit
// newer than the traversed entry keeps its state even when the peer entry's
// mtime is far in the future.
func TestBootstrapSkipsPeerEntryBehindLocalDelete(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
now := time.Now()
path := util.NewFullPath("/dir", "file")
f.notePathChange(path, now.UnixNano(), true)
entry := &Entry{
FullPath: path,
Attr: Attr{Mtime: now.Add(time.Hour)}, // client-settable, must not win
Content: []byte("stale peer entry"),
}
if err := f.applyTraversedEntry(context.Background(), entry, now.Add(-time.Minute).UnixNano()); err != nil {
t.Fatalf("applyTraversedEntry: %v", err)
}
if _, err := store.FindEntry(context.Background(), path); err == nil {
t.Fatal("bootstrap resurrected a path newer local delete covered")
}
}
// The bug this guards: a peer entry committed recently but stamped with an
// old client mtime must still apply over an absent-or-older local commit.
func TestBootstrapAppliesPeerCommitNewerThanLocal(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
now := time.Now()
path := util.NewFullPath("/dir", "file")
entry := &Entry{
FullPath: path,
Attr: Attr{Mtime: now.Add(-time.Hour)}, // old client-set mtime
Content: []byte("fresh peer write"),
}
if err := f.applyTraversedEntry(context.Background(), entry, now.UnixNano()); err != nil {
t.Fatalf("applyTraversedEntry: %v", err)
}
got, err := store.FindEntry(context.Background(), path)
if err != nil {
t.Fatalf("peer entry with newer commit must bootstrap: %v", err)
}
if string(got.Content) != "fresh peer write" {
t.Fatalf("content = %q, want peer entry applied", got.Content)
}
}
// A bootstrap-applied entry records the sender's commit timestamp, so a
// replayed event older than it — but newer than the entry's mtime — is
// still skipped.
func TestBootstrapAppliedCommitBlocksStaleReplay(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
now := time.Now()
path := util.NewFullPath("/dir", "file")
entry := &Entry{
FullPath: path,
Attr: Attr{Mtime: now.Add(-time.Hour)},
Content: []byte("fresh peer write"),
}
if err := f.applyTraversedEntry(context.Background(), entry, now.UnixNano()); err != nil {
t.Fatalf("applyTraversedEntry: %v", err)
}
if err := f.Replay(overwriteEvent("/dir", "file", now.Add(-time.Minute).UnixNano(), "stale replay")); err != nil {
t.Fatalf("Replay: %v", err)
}
got, _ := store.FindEntry(context.Background(), path)
if string(got.Content) != "fresh peer write" {
t.Fatalf("content = %q, stale replay must not overwrite the bootstrap entry", got.Content)
}
}
// An existing local entry is replaced only when the peer commit is newer —
// not when its mtime happens to be.
func TestBootstrapUpdatesExistingByCommitTime(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
now := time.Now()
path := util.NewFullPath("/dir", "file")
store.entries[string(path)] = &Entry{
FullPath: path,
Attr: Attr{Mtime: now.Add(time.Hour)}, // future mtime, e.g. client-set
Content: []byte("old local write"),
}
f.notePathChange(path, now.Add(-time.Minute).UnixNano(), false)
entry := &Entry{
FullPath: path,
Attr: Attr{Mtime: now.Add(-time.Hour)},
Content: []byte("newer peer commit"),
}
if err := f.applyTraversedEntry(context.Background(), entry, now.UnixNano()); err != nil {
t.Fatalf("applyTraversedEntry: %v", err)
}
got, _ := store.FindEntry(context.Background(), path)
if string(got.Content) != "newer peer commit" {
t.Fatalf("content = %q, newer peer commit must update despite the future local mtime", got.Content)
}
}
// A delete whose commit record failed to persist still blocks a stale
// replayed create: the pending record stands in until the retry lands it.
func TestPendingDeleteRecordBlocksStaleReplay(t *testing.T) {
store := newStubFilerStore()
store.setKvPutErr(errors.New("kv store unreachable"))
f := newReplayTestFiler(store)
f.flushCtx, f.flushCancel = context.WithCancel(context.Background())
defer f.flushCancel()
now := time.Now()
path := util.NewFullPath("/dir", "file")
// The delete already committed; only its ordering record failed.
f.notePathChange(path, now.UnixNano(), true)
createEvent := &filer_pb.SubscribeMetadataResponse{
Directory: "/dir",
EventNotification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "file", Content: []byte("stale resurrect")},
},
TsNs: now.Add(-time.Minute).UnixNano(),
}
if err := f.Replay(createEvent); err != nil {
t.Fatalf("Replay: %v", err)
}
if _, err := store.FindEntry(context.Background(), path); err == nil {
t.Fatal("stale create resurrected a path whose delete record is still pending")
}
}
// The failed record write retries until it lands, then leaves the pending set.
func TestPendingCommitRecordRepairs(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
f.flushCtx, f.flushCancel = context.WithCancel(context.Background())
defer f.flushCancel()
path := util.NewFullPath("/dir", "file")
store.setKvPutErr(errors.New("kv store unreachable"))
f.notePathChange(path, 100, true)
if _, loaded := f.metaTsPending.Load(string(path)); !loaded {
t.Fatal("failed record write must stay pending")
}
store.setKvPutErr(nil)
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if _, loaded := f.metaTsPending.Load(string(path)); !loaded {
ts, found, err := f.kvReadTsNs(context.Background(), metaPathKey(metaPathDelPrefix, path))
if err != nil || !found || ts != 100 {
t.Fatalf("delete record after repair: ts=%v found=%v err=%v", ts, found, err)
}
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatal("pending record was not repaired")
}
// CommitTsNs reports the recorded commit time over the entry mtime, and a
// still-pending record over both.
func TestCommitTsNsPrefersRecordAndPending(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
path := util.NewFullPath("/dir", "file")
if got, err := f.CommitTsNs(context.Background(), path, 7); err != nil || got != 7 {
t.Fatalf("no record: CommitTsNs = %d, %v; want fallback 7", got, err)
}
f.notePathChange(path, 50, false)
if got, err := f.CommitTsNs(context.Background(), path, 7); err != nil || got != 50 {
t.Fatalf("recorded: CommitTsNs = %d, %v; want 50", got, err)
}
pending := &metaTsPendingRec{}
pending.changeTsNs.Store(80)
f.metaTsPending.Store(string(path), pending)
if got, err := f.CommitTsNs(context.Background(), path, 7); err != nil || got != 80 {
t.Fatalf("pending: CommitTsNs = %d, %v; want 80", got, err)
}
}
// A successful write discharges what the pending record owed before it, so a
// repair pass can never put an older timestamp back over it.
func TestPendingCommitRecordDischargedByNewerWrite(t *testing.T) {
store := newStubFilerStore()
f := newReplayTestFiler(store)
f.flushCtx, f.flushCancel = context.WithCancel(context.Background())
defer f.flushCancel()
path := util.NewFullPath("/dir", "file")
store.setKvPutErr(errors.New("kv store unreachable"))
f.notePathChange(path, 100, false)
store.setKvPutErr(nil)
f.notePathChange(path, 200, false)
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if _, loaded := f.metaTsPending.Load(string(path)); !loaded {
break
}
time.Sleep(10 * time.Millisecond)
}
ts, found, err := f.kvReadTsNs(context.Background(), metaPathKey(metaPathTsPrefix, path))
if err != nil || !found || ts != 200 {
t.Fatalf("MpTs after repair = %v, %v, %v; want 200", ts, found, err)
}
}
// A failed delete record keeps its own time even when a later failed change
// merges into the pending record: the repair must not stamp the delete at
// the newer change's time.
func TestPendingDeleteRecordKeepsItsTime(t *testing.T) {
store := newStubFilerStore()
store.setKvPutErr(errors.New("kv store unreachable"))
f := newReplayTestFiler(store)
f.flushCtx, f.flushCancel = context.WithCancel(context.Background())
defer f.flushCancel()
path := util.NewFullPath("/dir", "file")
f.notePathChange(path, 100, true)
f.notePathChange(path, 200, false)
store.setKvPutErr(nil)
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if _, loaded := f.metaTsPending.Load(string(path)); !loaded {
break
}
time.Sleep(10 * time.Millisecond)
}
if ts, found, err := f.kvReadTsNs(context.Background(), metaPathKey(metaPathTsPrefix, path)); err != nil || !found || ts != 200 {
t.Fatalf("MpTs = %v, %v, %v; want 200", ts, found, err)
}
if ts, found, err := f.kvReadTsNs(context.Background(), metaPathKey(metaPathDelPrefix, path)); err != nil || !found || ts != 100 {
t.Fatalf("MpDl = %v, %v, %v; want 100", ts, found, err)
}
}
weed/filer/meta_aggregator_test.go
+52
View File
@@ -34,3 +34,55 @@ func TestOnPeerUpdateRepeatedAdd(t *testing.T) {
t.Fatal("expecting the subscription to be removed")
}
}
// Shutdown must close every peer's stopChan and wait for the loops, which is
// what lets each loop's deferred offset save run before the store closes.
func TestMetaAggregatorShutdownStopsPeerLoops(t *testing.T) {
ma := NewMetaAggregator(nil, pb.ServerAddress("127.0.0.1:2"), nil)
stopChan := make(chan struct{})
peer := pb.ServerAddress("127.0.0.1:1")
ma.peerChans[peer] = stopChan
ma.peerWg.Add(1)
exited := make(chan struct{})
go func() {
defer ma.peerWg.Done()
<-stopChan
close(exited)
}()
done := make(chan struct{})
go func() {
ma.Shutdown()
close(done)
}()
select {
case <-done:
case <-time.After(3 * time.Second):
t.Fatal("Shutdown did not return")
}
select {
case <-exited:
default:
t.Fatal("peer loop was not stopped")
}
}
// After Shutdown begins, a peer-add update arriving on the still-connected
// master watch must not start a subscription: it would have no stop signal
// and could run past the store close.
func TestOnPeerUpdateAfterShutdownRejected(t *testing.T) {
ma := NewMetaAggregator(nil, pb.ServerAddress("127.0.0.1:2"), nil)
ma.Shutdown()
peer := pb.ServerAddress("127.0.0.1:1")
ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(peer), IsAdd: true}, time.Now())
ma.peerChansLock.Lock()
_, found := ma.peerChans[peer]
ma.peerChansLock.Unlock()
if found {
t.Fatal("peer subscription started after shutdown")
}
}
weed/filer/meta_path_ts.go
+324
View File
@@ -0,0 +1,324 @@
package filer
import (
"context"
"crypto/sha256"
"errors"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
const (
// metaPathTsPrefix keys per-path commit timestamps in the filer store KV:
// the newest event timestamp this filer committed or applied for the
// path. Written for every emitted mutation event and kept after the entry
// is deleted, so a replayed peer event can still be ordered against a
// change that left no entry behind.
metaPathTsPrefix = "MpTs"
// metaPathDelPrefix keys delete-only timestamps. A replayed event checks
// the path's ancestors too, so a directory delete also covers children
// swept without per-child events (a dropped bucket's whole-store delete,
// or a walk aborted partway).
metaPathDelPrefix = "MpDl"
)
// metaPathKey hashes the path into a fixed-size key. SQL-backed KV stores
// split the key at byte 8 and store the remainder base64-encoded in a
// VARCHAR(766) name column: a raw path over ~578 bytes overflows it, the
// KvPut insert fails, and the fallback update touches no row yet returns no
// error — the commit record would never land and stale replay could restore
// the path.
func metaPathKey(prefix string, path util.FullPath) []byte {
sum := sha256.Sum256([]byte(path))
key := make([]byte, len(prefix)+16)
copy(key, prefix)
copy(key[len(prefix):], sum[:16])
return key
}
// kvReadTsNs returns the timestamp recorded for key, or found=false when the
// key is absent or malformed. Any other store read failure is an error: a
// caller deciding whether a replayed change is stale must not guess.
func (f *Filer) kvReadTsNs(ctx context.Context, key []byte) (tsNs int64, found bool, err error) {
value, err := f.Store.KvGet(ctx, key)
if err != nil {
if errors.Is(err, ErrKvNotFound) {
return 0, false, nil
}
return 0, false, err
}
if len(value) < 8 {
return 0, false, nil
}
return int64(util.BytesToUint64(value)), true, nil
}
// notePathChange persists tsNs as path's newest committed change. Local event
// timestamps are stamped monotonically (metaLogInflight.stamp), so a plain
// put cannot regress an earlier local record; a higher peer record it may
// overwrite was already newer under last-writer-wins ordering anyway. A
// delete additionally writes a delete record the ancestor lookup consults.
//
// The write uses a fresh context: callers notify after CommitTransaction,
// and SQL stores keep the already-committed *sql.Tx in the request context —
// a record written through it would fail and the path would lose ordering.
//
// A failed write is kept in metaTsPending and retried in the background: the
// mutation the record orders has already committed, so dropping it would let
// a stale replayed change overwrite or resurrect the path while the store is
// unreachable. Ordering reads consult the pending record until the retry
// lands it.
func (f *Filer) notePathChange(path util.FullPath, tsNs int64, deleted bool) {
if f.Store == nil {
return
}
ctx := context.Background()
key := string(path)
rec := f.pendingRec(key)
defer rec.mu.Unlock()
value := make([]byte, 8)
util.Uint64toBytes(value, uint64(tsNs))
if err := f.Store.KvPut(ctx, metaPathKey(metaPathTsPrefix, path), value); err != nil {
glog.WarningfCtx(ctx, "record meta change time %s: %v", path, err)
if tsNs > rec.changeTsNs.Load() {
rec.changeTsNs.Store(tsNs)
}
} else if rec.changeTsNs.Load() <= tsNs {
// This write covers every change record owed so far.
rec.changeTsNs.Store(0)
}
if deleted {
if err := f.Store.KvPut(ctx, metaPathKey(metaPathDelPrefix, path), value); err != nil {
glog.WarningfCtx(ctx, "record meta delete time %s: %v", path, err)
if tsNs > rec.deleteTsNs.Load() {
rec.deleteTsNs.Store(tsNs)
}
} else if rec.deleteTsNs.Load() <= tsNs {
rec.deleteTsNs.Store(0)
}
}
if rec.changeTsNs.Load() == 0 && rec.deleteTsNs.Load() == 0 {
f.metaTsPending.CompareAndDelete(key, rec)
return
}
if _, running := f.metaTsRepairing.LoadOrStore(key, struct{}{}); !running {
go f.repairMetaPathTs(path)
}
}
// metaTsPendingRec is a commit record still on its way to the store, holding
// only work not yet written: changeTsNs is the newest committed change the
// MpTs record lacks and deleteTsNs the newest delete the MpDl record lacks —
// the two keep their own times so a later update cannot masquerade as the
// delete. Store writes and the bookkeeping they discharge share mu, so a
// repair pass can never write an older record over a newer one that already
// landed; readers take the atomic fields directly.
type metaTsPendingRec struct {
mu sync.Mutex
changeTsNs atomic.Int64
deleteTsNs atomic.Int64
}
// pendingRec returns the live pending record for key, locked for mutation,
// recreating it when a repair already dropped the previous one.
func (f *Filer) pendingRec(key string) *metaTsPendingRec {
for {
cur, _ := f.metaTsPending.LoadOrStore(key, &metaTsPendingRec{})
rec := cur.(*metaTsPendingRec)
rec.mu.Lock()
if live, ok := f.metaTsPending.Load(key); ok && live == cur {
return rec
}
rec.mu.Unlock()
}
}
// repairMetaPathTs retries a pending commit record until it lands. Each pass
// writes only what the record still owes, so a merge by a later failure is
// picked up without another goroutine and a landed write is never rewritten.
// The retry is unbounded — the mutation is already committed and only the
// record preserves its ordering.
func (f *Filer) repairMetaPathTs(path util.FullPath) {
key := string(path)
ctx := f.flushCtx
if ctx == nil {
ctx = context.Background()
}
defer func() {
f.metaTsRepairing.Delete(key)
// A failure landing between the last pass and the flag release
// still needs a repair pass — unless the retry context is done.
if ctx.Err() != nil {
return
}
if _, ok := f.metaTsPending.Load(key); ok {
if _, running := f.metaTsRepairing.LoadOrStore(key, struct{}{}); !running {
go f.repairMetaPathTs(path)
}
}
}()
for {
if _, ok := f.metaTsPending.Load(key); !ok {
return
}
rec := f.pendingRec(key)
var tsErr, delErr error
if tsNs := rec.changeTsNs.Load(); tsNs > 0 {
value := make([]byte, 8)
util.Uint64toBytes(value, uint64(tsNs))
if tsErr = f.Store.KvPut(ctx, metaPathKey(metaPathTsPrefix, path), value); tsErr == nil {
rec.changeTsNs.Store(0)
}
}
if tsNs := rec.deleteTsNs.Load(); tsNs > 0 {
value := make([]byte, 8)
util.Uint64toBytes(value, uint64(tsNs))
if delErr = f.Store.KvPut(ctx, metaPathKey(metaPathDelPrefix, path), value); delErr == nil {
rec.deleteTsNs.Store(0)
}
}
done := rec.changeTsNs.Load() == 0 && rec.deleteTsNs.Load() == 0
if done {
f.metaTsPending.CompareAndDelete(key, rec)
}
rec.mu.Unlock()
if !done {
glog.V(0).InfofCtx(ctx, "retry record meta change time %s: %v %v", path, tsErr, delErr)
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
}
}
}
}
// pendingPathTsNs returns the newest pending commit timestamp for path and
// the newest pending delete timestamp.
func (f *Filer) pendingPathTsNs(path util.FullPath) (changeTsNs, deleteTsNs int64) {
cur, ok := f.metaTsPending.Load(string(path))
if !ok {
return 0, 0
}
rec := cur.(*metaTsPendingRec)
return rec.changeTsNs.Load(), rec.deleteTsNs.Load()
}
// CommitTsNs returns the newest commit timestamp known for path — the
// recorded event time, or a still-pending one — and fallbackTsNs when none
// exists. Peers stamp traversal entries with it so a bootstrap subscriber
// orders each entry on the sender's commit clock rather than the entry's
// client-settable mtime. A store read failure is an error, not a fallback:
// the mtime it would mint could outdate real commits.
func (f *Filer) CommitTsNs(ctx context.Context, path util.FullPath, fallbackTsNs int64) (int64, error) {
if f.Store == nil {
return fallbackTsNs, nil
}
tsNs := fallbackTsNs
recTs, found, err := f.kvReadTsNs(ctx, metaPathKey(metaPathTsPrefix, path))
if err != nil {
return 0, err
}
if found {
tsNs = recTs
}
if pendTs, _ := f.pendingPathTsNs(path); pendTs > tsNs {
tsNs = pendTs
}
return tsNs, nil
}
// noteCommittedEvent records the event's commit timestamp for every path it
// touches: the source of a delete or rename, and the target of a create,
// update, or rename. It runs after the local mutation commits while the
// caller still holds the path's entry lock (for filer-server mutations), so
// a peer replay ordered by the same lock sees either the whole local change
// or none of it.
func (f *Filer) noteCommittedEvent(event *filer_pb.SubscribeMetadataResponse) {
message := event.EventNotification
if message == nil {
return
}
var source, target util.FullPath
if message.OldEntry != nil {
source = util.FullPath(filer_pb.MetadataEventSourceFullPath(event))
}
if message.NewEntry != nil {
target = util.FullPath(filer_pb.MetadataEventTargetFullPath(event))
}
// A delete or a rename removes the source path; an update reports the
// same path on both sides and counts once.
if source != "" && source != target {
f.notePathChange(source, event.TsNs, true)
}
if target != "" {
f.notePathChange(target, event.TsNs, false)
}
}
// deletedAfterTsNs returns the newest recorded delete timestamp among path
// and its ancestors.
func (f *Filer) deletedAfterTsNs(ctx context.Context, path util.FullPath) (int64, error) {
var tsNs int64
for p := string(path); p != "" && p != "/"; {
ts, found, err := f.kvReadTsNs(ctx, metaPathKey(metaPathDelPrefix, util.FullPath(p)))
if err != nil {
return 0, err
}
if found && ts > tsNs {
tsNs = ts
}
if _, pendDel := f.pendingPathTsNs(util.FullPath(p)); pendDel > tsNs {
tsNs = pendDel
}
i := strings.LastIndexByte(p, '/')
if i <= 0 {
break
}
p = p[:i]
}
return tsNs, nil
}
// localCommitTsNs returns the newest timestamp this filer is known to have
// committed for path: the recorded event time when one exists — it is not
// client-controllable the way an entry's mtime is — else the entry's mtime
// for entries written before recording existed, plus any recorded delete of
// the path or an ancestor.
func (f *Filer) localCommitTsNs(ctx context.Context, path util.FullPath) (int64, error) {
if f.Store == nil {
return 0, nil
}
var tsNs int64
recTs, found, err := f.kvReadTsNs(ctx, metaPathKey(metaPathTsPrefix, path))
if err != nil {
return 0, err
}
if found {
tsNs = recTs
} else if entry, findErr := f.Store.FindEntry(ctx, path); findErr != nil {
if !errors.Is(findErr, filer_pb.ErrNotFound) {
return 0, findErr
}
} else if entry != nil {
tsNs = entry.Mtime.UnixNano()
}
if pendTs, _ := f.pendingPathTsNs(path); pendTs > tsNs {
tsNs = pendTs
}
delTs, err := f.deletedAfterTsNs(ctx, path)
if err != nil {
return 0, err
}
if delTs > tsNs {
tsNs = delTs
}
return tsNs, nil
}
weed/filer/meta_replay.go
+151 -14
View File
@@ -2,6 +2,8 @@ package filer
import (
"context"
"slices"
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -9,31 +11,166 @@ import (
)
// Replay applies the delete and the insert as separate, non-transactional
// store calls, so retrying it after a partial failure is not atomic.
func Replay(filerStore FilerStore, resp *filer_pb.SubscribeMetadataResponse) error {
// store calls, so retrying it after a partial failure is not atomic; both
// operations are idempotent and the timestamp check tolerates re-delivery.
//
// A replayed event can be older than local state: a peer resuming from a
// stale offset replays everything since the offset, and two writes through
// different filers race within the replication lag. Apply each path's change
// only when no newer local change is recorded, so a late event cannot revert
// or resurrect entries.
//
// Ordering uses the per-path commit timestamp recorded for every emitted
// mutation (meta_path_ts.go), which survives deletes as a tombstone and is
// not client-controllable the way entry mtime is — a mounted client can set
// mtime into the future, and a rename keeps the source file's mtime on the
// destination entry. Entries written before recording existed fall back to
// their mtime.
//
// The check and the mutation run under the same per-path entry lock the
// filer server's local writes hold, so they cannot be split by a concurrent
// local update or by another peer's replay.
func (f *Filer) Replay(resp *filer_pb.SubscribeMetadataResponse) error {
message := resp.EventNotification
var oldPath util.FullPath
var newEntry *Entry
dir := resp.Directory
if message.NewParentPath != "" {
dir = message.NewParentPath
}
var oldPath, newPath util.FullPath
if message.OldEntry != nil {
oldPath = util.NewFullPath(resp.Directory, message.OldEntry.Name)
glog.V(4).Infof("deleting %v", oldPath)
if err := filerStore.DeleteEntry(context.Background(), oldPath); err != nil {
}
if message.NewEntry != nil {
newPath = util.NewFullPath(dir, message.NewEntry.Name)
}
ctx := context.Background()
unlock := f.lockReplayPaths(oldPath, newPath)
defer unlock()
samePath := oldPath != "" && oldPath == newPath
if samePath {
newer, err := f.replayBlockedByLocal(ctx, oldPath, resp.TsNs)
if err != nil || newer {
return err
}
glog.V(4).Infof("deleting %v", oldPath)
if err := f.Store.DeleteEntry(ctx, oldPath); err != nil {
return err
}
// Record the commit before the insert too: if the insert fails and
// the event is retried later, the path sits deleted in between —
// without the record a stale peer create could resurrect it in the
// gap. The entry still exists afterward, so no delete record.
f.notePathChange(oldPath, resp.TsNs, false)
glog.V(4).Infof("creating %v", newPath)
newEntry := FromPbEntry(dir, message.NewEntry)
if err := f.Store.InsertEntry(ctx, newEntry); err != nil {
return err
}
f.notePathChange(oldPath, resp.TsNs, false)
return nil
}
if message.OldEntry != nil {
newer, err := f.replayBlockedByLocal(ctx, oldPath, resp.TsNs)
if err != nil {
return err
}
if !newer {
glog.V(4).Infof("deleting %v", oldPath)
if err := f.Store.DeleteEntry(ctx, oldPath); err != nil {
return err
}
f.notePathChange(oldPath, resp.TsNs, true)
}
}
if message.NewEntry != nil {
dir := resp.Directory
if message.NewParentPath != "" {
dir = message.NewParentPath
}
key := util.NewFullPath(dir, message.NewEntry.Name)
glog.V(4).Infof("creating %v", key)
newEntry = FromPbEntry(dir, message.NewEntry)
if err := filerStore.InsertEntry(context.Background(), newEntry); err != nil {
newer, err := f.replayBlockedByLocal(ctx, newPath, resp.TsNs)
if err != nil {
return err
}
if !newer {
glog.V(4).Infof("creating %v", newPath)
newEntry := FromPbEntry(dir, message.NewEntry)
if err := f.Store.InsertEntry(ctx, newEntry); err != nil {
return err
}
f.notePathChange(newPath, resp.TsNs, false)
}
}
return nil
}
// replayBlockedByLocal reports whether the local store already holds a
// version of path committed after the event's timestamp. Lookup failures are
// propagated: a replayed change must not be applied on the guess that the
// path was simply absent, or a transient read error could let a stale event
// overwrite a newer entry past the point of no return.
func (f *Filer) replayBlockedByLocal(ctx context.Context, path util.FullPath, tsNs int64) (bool, error) {
if tsNs <= 0 {
return false, nil
}
localTsNs, err := f.localCommitTsNs(ctx, path)
if err != nil {
return false, err
}
return localTsNs > tsNs, nil
}
// lockReplayPaths holds the shared entry locks for every path the event
// touches, in the same sorted order the rename handler takes them.
// lockReplayPaths takes the per-path entry locks for the event's paths and
// every ancestor of them, in lexicographic order — parents sort before
// children, the same ordering the local rename handler's two-path lock uses,
// so lock acquisition is deadlock-free across operations. Ancestors are
// included because the replay check consults ancestor delete records: a
// local recursive delete or directory rename holds only the ancestor's lock,
// and without contending on it a replay could reinsert a stale child after
// its check. Ancestors are locked shared — they only need to exclude an
// exclusive holder, not each other, or every replay under a busy parent
// would serialize — while a path the event actually changes stays exclusive
// even when it is also the other path's ancestor. The root is never locked:
// it cannot be deleted, and a shared root lock would still be excluded by
// every top-level delete.
func (f *Filer) lockReplayPaths(oldPath, newPath util.FullPath) func() {
lt := f.EntryLockTable
if lt == nil {
return func() {}
}
var paths []util.FullPath
addAncestors := func(p util.FullPath) {
for i := strings.LastIndexByte(string(p), '/'); i > 0; i = strings.LastIndexByte(string(p), '/') {
p = p[:i]
paths = append(paths, p)
}
}
addAncestors(oldPath)
addAncestors(newPath)
if oldPath != "" {
paths = append(paths, oldPath)
}
if newPath != "" {
paths = append(paths, newPath)
}
if len(paths) == 0 {
return func() {}
}
slices.Sort(paths)
paths = slices.Compact(paths)
locks := make([]*util.ActiveLock, len(paths))
for i, p := range paths {
lockType := util.SharedLock
if p == oldPath || p == newPath {
lockType = util.ExclusiveLock
}
locks[i] = lt.AcquireLock("MetaReplay", p, lockType)
}
return func() {
for i := len(paths) - 1; i >= 0; i-- {
lt.ReleaseLock(paths[i], locks[i])
}
}
}
weed/filer/reader_at_prefetch_window_test.go
+74
View File
@@ -0,0 +1,74 @@
package filer
import (
"context"
"fmt"
"strconv"
"strings"
"sync/atomic"
"testing"
"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)
// A sequential reader that consumes a chunk in several small ReadAt calls, as
// the S3 GET path does with its 256 KiB copy buffer over 4 MiB chunks, must
// keep its read-ahead at prefetchCount chunks. Each ReadAt triggers a
// prefetch; if chunks already being fetched are not counted toward the
// window, every call starts prefetchCount more downloads and the read-ahead
// runs to the ReaderCache downloader limit, buffering most of the object.
func TestChunkReadAtSequentialPrefetchStaysWithinWindow(t *testing.T) {
const chunkSize = 64 << 10
const chunkCount = 64
const sliceSize = 16 << 10
var readerChunk atomic.Int64
var maxLead atomic.Int64
rc := NewReaderCache(256, (*chunk_cache.TieredChunkCache)(nil), func(context.Context, string) ([]string, error) {
return []string{"unused"}, nil
}, nil)
defer rc.destroy()
rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, fileId string, _ util_http.RefreshUrlsFunc) (int, error) {
idx, err := strconv.Atoi(strings.TrimPrefix(fileId, "chunk"))
if err != nil {
return 0, err
}
lead := int64(idx) - readerChunk.Load()
for {
cur := maxLead.Load()
if lead <= cur || maxLead.CompareAndSwap(cur, lead) {
break
}
}
return len(buffer), nil
}
views := NewIntervalList[*ChunkView]()
for i := 0; i < chunkCount; i++ {
views.AppendInterval(&Interval[*ChunkView]{
StartOffset: int64(i * chunkSize),
StopOffset: int64((i + 1) * chunkSize),
Value: &ChunkView{
FileId: fmt.Sprintf("chunk%d", i),
ViewSize: chunkSize,
ViewOffset: int64(i * chunkSize),
ChunkSize: chunkSize,
},
})
}
reader := NewChunkReaderAtFromClient(context.Background(), rc, views, chunkSize*chunkCount, DefaultPrefetchCount)
defer reader.ReleaseStream()
buf := make([]byte, sliceSize)
for off := int64(0); off < chunkSize*chunkCount; off += sliceSize {
readerChunk.Store(off / chunkSize)
if _, err := reader.ReadAt(buf, off); err != nil && off+sliceSize < chunkSize*chunkCount {
t.Fatalf("ReadAt(%d): %v", off, err)
}
}
if got := maxLead.Load(); got > DefaultPrefetchCount {
t.Fatalf("read-ahead reached %d chunks past the reader, want at most %d", got, DefaultPrefetchCount)
}
}
weed/filer/reader_cache.go
+4 -3
View File
@@ -92,8 +92,10 @@ func (rc *ReaderCache) MaybeCache(chunkViews *Interval[*ChunkView], count int) {
return
}
cached := 0
for x := chunkViews; x != nil && cached < count; x = x.Next {
// The window is the next count chunks, whether or not a download for them
// is already running: counting only newly started downloads would push the
// window count chunks further on every call.
for x, i := chunkViews, 0; x != nil && i < count; x, i = x.Next, i+1 {
chunkView := x.Value
if _, found := rc.downloaders[chunkView.FileId]; found {
continue
@@ -124,7 +126,6 @@ func (rc *ReaderCache) MaybeCache(chunkViews *Interval[*ChunkView], count int) {
rc.downloaders[chunkView.FileId] = cacher
go cacher.startCaching()
started = append(started, cacher)
cached++
}
return
weed/mount/dir_sink.go
+3 -1
View File
@@ -1,6 +1,8 @@
package mount
import (
"context"
"github.com/seaweedfs/go-fuse/v2/fuse"
)
@@ -39,5 +41,5 @@ func (l fuseDirEntryList) TakesLookupRef() bool { return true }
// ReadDirectoryInto runs a readdir against sink. ReadDir and ReadDirPlus are
// this with the kernel reply buffer as the sink.
func (wfs *WFS) ReadDirectoryInto(input *fuse.ReadIn, sink DirEntrySink, isPlusMode bool) fuse.Status {
return wfs.doReadDirectory(input, sink, isPlusMode)
return wfs.doReadDirectory(context.Background(), input, sink, isPlusMode)
}
weed/mount/filehandle_read.go
+7 -4
View File
@@ -43,7 +43,7 @@ func (fh *FileHandle) readFromChunksWithContext(ctx context.Context, buff []byte
entry.RUnlock()
if remoteOnly {
glog.V(4).Infof("download remote entry %s", fileFullPath)
err := fh.downloadRemoteEntry(entry)
err := fh.downloadRemoteEntry(ctx, entry)
if err != nil {
glog.V(1).Infof("download remote entry %s: %v", fileFullPath, err)
return 0, 0, err
@@ -111,12 +111,15 @@ func (fh *FileHandle) readFromChunksWithContext(ctx context.Context, buff []byte
return int64(totalRead), ts, err
}
func (fh *FileHandle) downloadRemoteEntry(entry *LockedEntry) error {
func (fh *FileHandle) downloadRemoteEntry(ctx context.Context, entry *LockedEntry) error {
fileFullPath := fh.FullPath()
dir, _ := fileFullPath.DirAndName()
err := fh.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
// callWait 0: this RPC streams a whole remote object filer-side, so a
// healthy download can legitimately outlast any metadata-call bound. The
// attempt stays bound to the caller's context instead.
err := fh.wfs.withFilerClient(ctx, false, 0, func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.CacheRemoteObjectToLocalClusterRequest{
Directory: string(dir),
@@ -124,7 +127,7 @@ func (fh *FileHandle) downloadRemoteEntry(entry *LockedEntry) error {
}
glog.V(4).Infof("download entry: %v", request)
resp, err := client.CacheRemoteObjectToLocalCluster(context.Background(), request)
resp, err := client.CacheRemoteObjectToLocalCluster(ctx, request)
if err != nil {
return fmt.Errorf("CacheRemoteObjectToLocalCluster file %s: %v", fileFullPath, err)
}
weed/mount/filer_conf.go
+2 -2
View File
@@ -20,8 +20,8 @@ func (wfs *WFS) subscribeFilerConfEvents() (*meta_cache.MetadataFollower, error)
confFullName := filepath.Join(filer.DirectoryEtcSeaweedFS, filer.FilerConfName)
// read current conf
err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
content, err := filer.ReadInsideFiler(context.Background(), client, confDir, confName)
err := wfs.withFilerClient(context.Background(), false, wfs.filerRPCWait(), func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
content, err := filer.ReadInsideFiler(ctx, client, confDir, confName)
if err != nil {
return err
}
weed/mount/meta_cache/meta_cache_build_test.go
+5 -5
View File
@@ -110,7 +110,7 @@ func TestEnsureVisitedReplaysBufferedEventsAfterSnapshot(t *testing.T) {
},
}
if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 0); err != nil {
if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/dir"), 0); err != nil {
t.Fatalf("ensure visited: %v", err)
}
if applyErr != nil {
@@ -504,7 +504,7 @@ func TestEnsureVisitedPreservesLocalOnlyEntry(t *testing.T) {
}},
}}
if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 0); err != nil {
if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/dir"), 0); err != nil {
t.Fatalf("ensure visited: %v", err)
}
if !mc.IsDirectoryCached(util.FullPath("/dir")) {
@@ -550,7 +550,7 @@ func TestEnsureVisitedDropsUnpinnedStaleEntry(t *testing.T) {
}},
}}
if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 0); err != nil {
if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/dir"), 0); err != nil {
t.Fatalf("ensure visited: %v", err)
}
if entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/stale.txt")); err != filer_pb.ErrNotFound || entry != nil {
@@ -597,7 +597,7 @@ func TestEnsureVisitedConfirmsTransientEmptyListing(t *testing.T) {
},
}}
if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 0); err != nil {
if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/dir"), 0); err != nil {
t.Fatalf("ensure visited: %v", err)
}
if !mc.IsDirectoryCached(util.FullPath("/dir")) {
@@ -619,7 +619,7 @@ func TestEnsureVisitedCachesGenuinelyEmptyDirectory(t *testing.T) {
}
accessor := &buildFilerAccessor{client: client}
if err := EnsureVisited(mc, accessor, util.FullPath("/empty"), 0); err != nil {
if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/empty"), 0); err != nil {
t.Fatalf("ensure visited: %v", err)
}
if !mc.IsDirectoryCached(util.FullPath("/empty")) {
weed/mount/meta_cache/meta_cache_init.go
+12 -5
View File
@@ -26,7 +26,7 @@ func (e *DirectoryTooLargeError) Error() string {
// maxCacheableEntries is the directory size above which a build gives up, or 0
// to cache everything.
func EnsureVisited(mc *MetaCache, client filer_pb.FilerClient, dirPath util.FullPath, maxCacheableEntries int) error {
func EnsureVisited(ctx context.Context, mc *MetaCache, client filer_pb.FilerClient, dirPath util.FullPath, maxCacheableEntries int) error {
// Collect all uncached paths from target directory up to root
var uncachedPaths []util.FullPath
currentPath := dirPath
@@ -61,7 +61,7 @@ func EnsureVisited(mc *MetaCache, client filer_pb.FilerClient, dirPath util.Full
// Fetch all uncached directories in parallel with context for cancellation
// If one fetch fails, cancel the others to avoid unnecessary work
g, ctx := errgroup.WithContext(context.Background())
g, ctx := errgroup.WithContext(ctx)
for _, p := range uncachedPaths {
path := p // capture for closure
g.Go(func() error {
@@ -91,8 +91,10 @@ const (
)
func doEnsureVisited(ctx context.Context, mc *MetaCache, client filer_pb.FilerClient, path util.FullPath, maxCacheableEntries int) error {
// Use singleflight to deduplicate concurrent requests for the same path
_, err, _ := mc.visitGroup.Do(string(path), func() (interface{}, error) {
// Use singleflight to deduplicate concurrent requests for the same path.
// DoChan so a canceled request stops waiting even when another request
// owns the in-flight build.
resultCh := mc.visitGroup.DoChan(string(path), func() (interface{}, error) {
// Check for cancellation before starting
if ctx.Err() != nil {
return nil, ctx.Err()
@@ -222,7 +224,12 @@ func doEnsureVisited(ctx context.Context, mc *MetaCache, client filer_pb.FilerCl
cleanupDone = true // Prevent deferred cleanup after successful publish
return nil, nil
})
return err
select {
case result := <-resultCh:
return result.Err
case <-ctx.Done():
return ctx.Err()
}
}
func IsHiddenSystemEntry(dir, name string) bool {
weed/mount/meta_cache/oversized_dir_test.go
+5 -5
View File
@@ -37,7 +37,7 @@ func TestEnsureVisitedRefusesOversizedDirectory(t *testing.T) {
accessor := &buildFilerAccessor{client: &buildListClient{responses: listResponses(10)}}
err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 5)
err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/dir"), 5)
var tooLarge *DirectoryTooLargeError
if !errors.As(err, &tooLarge) {
t.Fatalf("EnsureVisited = %v, want DirectoryTooLargeError", err)
@@ -61,7 +61,7 @@ func TestEnsureVisitedRefusesOversizedDirectory(t *testing.T) {
}
// A second visit fails fast without streaming to the limit again.
if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 5); !errors.As(err, &tooLarge) {
if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/dir"), 5); !errors.As(err, &tooLarge) {
t.Fatalf("second EnsureVisited = %v, want DirectoryTooLargeError", err)
}
}
@@ -74,7 +74,7 @@ func TestEnsureVisitedStepsOverOversizedAncestor(t *testing.T) {
mc.markOversized(util.FullPath("/huge"))
accessor := &buildFilerAccessor{client: &buildListClient{responses: listResponses(3)}}
if err := EnsureVisited(mc, accessor, util.FullPath("/huge/sub"), 5); err != nil {
if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/huge/sub"), 5); err != nil {
t.Fatalf("EnsureVisited under an oversized ancestor: %v", err)
}
if !mc.IsDirectoryCached(util.FullPath("/huge/sub")) {
@@ -92,7 +92,7 @@ func TestEnsureVisitedUnderTheLimitStillCaches(t *testing.T) {
defer mc.Shutdown()
accessor := &buildFilerAccessor{client: &buildListClient{responses: listResponses(5)}}
if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 5); err != nil {
if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/dir"), 5); err != nil {
t.Fatalf("EnsureVisited: %v", err)
}
if !mc.IsDirectoryCached(util.FullPath("/dir")) {
@@ -123,7 +123,7 @@ func TestEnsureVisitedAncestorFoundOversizedMidVisit(t *testing.T) {
"/huge/sub": listResponses(3),
}}}
if err := EnsureVisited(mc, accessor, util.FullPath("/huge/sub"), 5); err != nil {
if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/huge/sub"), 5); err != nil {
t.Fatalf("EnsureVisited: %v", err)
}
if !mc.IsDirectoryCached(util.FullPath("/huge/sub")) {
weed/mount/weedfs.go
+39 -13
View File
@@ -145,6 +145,7 @@ type WFS struct {
manifestCache *filer.ChunkManifestCache
writeBufferAccountant *page_writer.WriteBufferAccountant
signature int32
filerCallTimeout time.Duration // bounds one filer call; tests only, 0 = filerRPCTimeout
concurrentWriters *util.LimitedConcurrentExecutor
copyBufferPool sync.Pool
concurrentCopiersSem chan struct{}
@@ -501,12 +502,30 @@ func (wfs *WFS) Init(server *fuse.Server) {
wfs.fuseServer = server
}
// ctxFromCancel bridges a FUSE request's interrupt channel into a context so
// metadata lookups can abandon a stalled filer call when the kernel has given
// up on the operation.
func ctxFromCancel(cancel <-chan struct{}) (context.Context, context.CancelFunc) {
ctx, cancelFunc := context.WithCancel(context.Background())
if cancel == nil {
return ctx, cancelFunc
}
go func() {
select {
case <-cancel:
cancelFunc()
case <-ctx.Done():
}
}()
return ctx, cancelFunc
}
// maybeReadEntry resolves an inode to the entry metadata operations act on. An
// open handle answers ahead of the path: unlink drops the name while the
// descriptor stays valid, so an unlinked-but-open file returns its handle's
// entry with an empty path instead of ENOENT. A removed directory has no
// handle; its remembered entry answers the same way.
func (wfs *WFS) maybeReadEntry(inode uint64) (path util.FullPath, fh *FileHandle, entry *filer_pb.Entry, status fuse.Status) {
func (wfs *WFS) maybeReadEntry(ctx context.Context, inode uint64) (path util.FullPath, fh *FileHandle, entry *filer_pb.Entry, status fuse.Status) {
var found bool
if fh, found = wfs.fhMap.FindFileHandle(inode); found {
path, _ = wfs.inodeToPath.GetPath(inode)
@@ -524,7 +543,7 @@ func (wfs *WFS) maybeReadEntry(inode uint64) (path util.FullPath, fh *FileHandle
}
return
}
entry, _, status = wfs.maybeLoadEntry(path)
entry, _, status = wfs.maybeLoadEntry(ctx, path)
return
}
@@ -553,7 +572,7 @@ func (wfs *WFS) isLocalOnlyEntry(entry *filer.Entry) bool {
// maybeLoadEntry returns the entry and the log position it reflects, or a zero
// position when unknown.
func (wfs *WFS) maybeLoadEntry(fullpath util.FullPath) (*filer_pb.Entry, entryVersion, fuse.Status) {
func (wfs *WFS) maybeLoadEntry(ctx context.Context, fullpath util.FullPath) (*filer_pb.Entry, entryVersion, fuse.Status) {
// glog.V(3).Infof("read entry cache miss %s", fullpath)
_, name := fullpath.DirAndName()
@@ -572,7 +591,7 @@ func (wfs *WFS) maybeLoadEntry(fullpath util.FullPath) (*filer_pb.Entry, entryVe
}, entryVersion{}, fuse.OK
}
entry, version, status := wfs.lookupEntry(fullpath)
entry, version, status := wfs.lookupEntry(ctx, fullpath)
if status != fuse.OK {
return nil, entryVersion{}, status
}
@@ -591,7 +610,7 @@ const expiredDirRebuildCooldown = 30 * time.Second
// local entries do not leak back into lookup results.
// It also returns the log position the entry reflects: the entry's stored
// version, the lookup response's, or zero if unknown.
func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, entryVersion, fuse.Status) {
func (wfs *WFS) lookupEntry(ctx context.Context, fullpath util.FullPath) (*filer.Entry, entryVersion, fuse.Status) {
dir, _ := fullpath.DirAndName()
dirPath := util.FullPath(dir)
@@ -604,9 +623,11 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, entryVersion,
// flush of the target entry land first so the rebuilt cache does not
// capture pre-flush metadata and bypass the wait below.
if inode, found := wfs.inodeToPath.GetInode(fullpath); found {
wfs.waitForPendingAsyncFlush(inode)
if !wfs.waitForPendingAsyncFlush(ctx, inode) {
return nil, entryVersion{}, fuse.EINTR
}
}
if err := wfs.ensureDirectoryVisited(dirPath); err != nil {
if err := wfs.ensureDirectoryVisited(ctx, dirPath); err != nil {
// Record the attempt so the cooldown suppresses repeated rebuilds
// while the listing keeps failing; once it elapses a later lookup
// retries. Oversized dirs are already marked read-through.
@@ -618,7 +639,7 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, entryVersion,
}
if wfs.metaCache.IsDirectoryCached(dirPath) && wfs.metaCache.IsNameFresh(fullpath) {
cachedEntry, cachedVersionTsNs, cacheErr := wfs.metaCache.FindEntry(context.Background(), fullpath)
cachedEntry, cachedVersionTsNs, cacheErr := wfs.metaCache.FindEntry(ctx, fullpath)
if cacheErr != nil && cacheErr != filer_pb.ErrNotFound {
glog.Errorf("lookupEntry: cache lookup for %s failed: %v", fullpath, cacheErr)
return nil, entryVersion{}, fuse.EIO
@@ -649,7 +670,9 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, entryVersion,
// truncate's old size or a write's old chunks. The cache paths above are
// already consistent and must not pay this wait.
if inode, found := wfs.inodeToPath.GetInode(fullpath); found {
wfs.waitForPendingAsyncFlush(inode)
if !wfs.waitForPendingAsyncFlush(ctx, inode) {
return nil, entryVersion{}, fuse.EINTR
}
}
// Directory not cached - fetch directly from filer without caching the entire directory.
@@ -657,8 +680,8 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, entryVersion,
var entry *filer_pb.Entry
var lookupVersion entryVersion
lookupDir, lookupName := fullpath.DirAndName()
err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
resp, lookupErr := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{
err := wfs.withFilerClient(ctx, false, wfs.filerRPCWait(), func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
resp, lookupErr := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: lookupDir,
Name: lookupName,
})
@@ -670,6 +693,9 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, entryVersion,
return nil
})
if err != nil {
if ctx.Err() != nil {
return nil, entryVersion{}, fuse.EINTR
}
if err == filer_pb.ErrNotFound {
// The entry may exist in the local store from a deferred create
// (deferFilerCreate=true) that hasn't been flushed yet. Only trust
@@ -688,7 +714,7 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, entryVersion,
wfs.pendingAsyncFlushMu.Unlock()
if hasDirtyHandle || hasPendingFlush {
if localEntry, localVersionTsNs, localErr := wfs.metaCache.FindEntry(context.Background(), fullpath); localErr == nil && localEntry != nil {
if localEntry, localVersionTsNs, localErr := wfs.metaCache.FindEntry(ctx, fullpath); localErr == nil && localEntry != nil {
glog.V(4).Infof("lookupEntry found deferred entry in local cache %s", fullpath)
return localEntry, entryVersion{tsNs: localVersionTsNs}, fuse.OK
}
@@ -722,7 +748,7 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, entryVersion,
// up without -v=4 — and include layer-by-layer state so the
// next failed run pinpoints which layer dropped the entry.
localPresent := false
if localEntry, _, localErr := wfs.metaCache.FindEntry(context.Background(), fullpath); localErr == nil && localEntry != nil {
if localEntry, _, localErr := wfs.metaCache.FindEntry(ctx, fullpath); localErr == nil && localEntry != nil {
localPresent = true
}
glog.Warningf("lookupEntry: filer ErrNotFound for tracked path %s (inode=%d dirtyHandle=%v pendingFlush=%v localCache=%v dirCached=%v) — possible coherence bug",
weed/mount/weedfs_access.go
+3 -1
View File
@@ -82,7 +82,9 @@ func clearSupplementaryGroupCache() {
* This method is not called under Linux kernel versions 2.4.x
*/
func (wfs *WFS) Access(cancel <-chan struct{}, input *fuse.AccessIn) (code fuse.Status) {
_, _, entry, code := wfs.maybeReadEntry(input.NodeId)
ctx, cancelFunc := ctxFromCancel(cancel)
defer cancelFunc()
_, _, entry, code := wfs.maybeReadEntry(ctx, input.NodeId)
if code != fuse.OK {
return code
}
weed/mount/weedfs_attr.go
+7 -3
View File
@@ -26,7 +26,9 @@ func (wfs *WFS) GetAttr(cancel <-chan struct{}, input *fuse.GetAttrIn, out *fuse
}
inode := input.NodeId
path, fh, entry, status := wfs.maybeReadEntry(inode)
ctx, cancelFunc := ctxFromCancel(cancel)
defer cancelFunc()
path, fh, entry, status := wfs.maybeReadEntry(ctx, inode)
if status != fuse.OK {
return status
}
@@ -65,7 +67,9 @@ func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse
return fuse.Status(syscall.ENOSPC)
}
path, fh, entry, status := wfs.maybeReadEntry(input.NodeId)
ctx, cancelFunc := ctxFromCancel(cancel)
defer cancelFunc()
path, fh, entry, status := wfs.maybeReadEntry(ctx, input.NodeId)
if status != fuse.OK || entry == nil {
return status
}
@@ -336,7 +340,7 @@ func (wfs *WFS) touchDirMtimeCtimeBest(dirPath util.FullPath) {
// touchDirMtimeCtime updates a directory's mtime and ctime on the filer.
// POSIX requires this when entries are created or removed in the directory.
func (wfs *WFS) touchDirMtimeCtime(dirPath util.FullPath) {
dirEntry, _, code := wfs.maybeLoadEntry(dirPath)
dirEntry, _, code := wfs.maybeLoadEntry(context.Background(), dirPath)
if code != fuse.OK || dirEntry == nil || dirEntry.Attributes == nil {
return
}
weed/mount/weedfs_dir_lookup.go
+3 -1
View File
@@ -27,7 +27,9 @@ func (wfs *WFS) Lookup(cancel <-chan struct{}, header *fuse.InHeader, name strin
fullFilePath := dirPath.Child(name)
// Use shared lookup logic that checks cache first, then filer if needed
localEntry, _, status := wfs.lookupEntry(fullFilePath)
ctx, cancelFunc := ctxFromCancel(cancel)
defer cancelFunc()
localEntry, _, status := wfs.lookupEntry(ctx, fullFilePath)
if status != fuse.OK {
return status
}
weed/mount/weedfs_dir_lookup_cache_test.go
+8 -8
View File
@@ -158,7 +158,7 @@ func TestLookupRebuildsExpiredDirectoryCache(t *testing.T) {
}
startFakeFiler(t, wfs, fake)
if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dir, 0); err != nil {
if err := meta_cache.EnsureVisited(context.Background(), wfs.metaCache, wfs, dir, 0); err != nil {
t.Fatalf("EnsureVisited: %v", err)
}
if !wfs.inodeToPath.IsChildrenCached(dir) {
@@ -174,7 +174,7 @@ func TestLookupRebuildsExpiredDirectoryCache(t *testing.T) {
lookupCallsBefore := fake.lookupCalls.Load()
for _, name := range []string{"a", "b", "c"} {
entry, _, status := wfs.lookupEntry(dir.Child(name))
entry, _, status := wfs.lookupEntry(context.Background(), dir.Child(name))
if status != fuse.OK {
t.Fatalf("lookupEntry %s: %v", name, status)
}
@@ -211,7 +211,7 @@ func TestLookupRebuildDeduplicatesConcurrentRebuilds(t *testing.T) {
}
startFakeFiler(t, wfs, fake)
if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dir, 0); err != nil {
if err := meta_cache.EnsureVisited(context.Background(), wfs.metaCache, wfs, dir, 0); err != nil {
t.Fatalf("EnsureVisited: %v", err)
}
expireDirCache(t, wfs.inodeToPath, dir)
@@ -228,7 +228,7 @@ func TestLookupRebuildDeduplicatesConcurrentRebuilds(t *testing.T) {
wg.Add(1)
go func(i int, name string) {
defer wg.Done()
if _, _, status := wfs.lookupEntry(dir.Child(name)); status != fuse.OK {
if _, _, status := wfs.lookupEntry(context.Background(), dir.Child(name)); status != fuse.OK {
errs[i] = fmt.Errorf("lookupEntry %s: %v", name, status)
}
}(i, name)
@@ -271,7 +271,7 @@ func TestLookupRebuildCooldownAfterFailure(t *testing.T) {
}
startFakeFiler(t, wfs, fake)
if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dir, 0); err != nil {
if err := meta_cache.EnsureVisited(context.Background(), wfs.metaCache, wfs, dir, 0); err != nil {
t.Fatalf("EnsureVisited: %v", err)
}
expireDirCache(t, wfs.inodeToPath, dir)
@@ -280,7 +280,7 @@ func TestLookupRebuildCooldownAfterFailure(t *testing.T) {
fake.listErr = errors.New("persistent listing failure")
listCallsBefore := fake.listCalls.Load()
lookupCallsBefore := fake.lookupCalls.Load()
if entry, _, status := wfs.lookupEntry(dir.Child("a")); status != fuse.OK || entry == nil || entry.Name() != "a" {
if entry, _, status := wfs.lookupEntry(context.Background(), dir.Child("a")); status != fuse.OK || entry == nil || entry.Name() != "a" {
t.Fatalf("lookupEntry a after failed rebuild: status=%v entry=%v", status, entry)
}
if got := fake.lookupCalls.Load() - lookupCallsBefore; got != 1 {
@@ -293,7 +293,7 @@ func TestLookupRebuildCooldownAfterFailure(t *testing.T) {
// Immediate retry: cooldown suppresses the rebuild, falls through again.
listCallsBefore = fake.listCalls.Load()
lookupCallsBefore = fake.lookupCalls.Load()
if _, _, status := wfs.lookupEntry(dir.Child("a")); status != fuse.OK {
if _, _, status := wfs.lookupEntry(context.Background(), dir.Child("a")); status != fuse.OK {
t.Fatalf("second lookupEntry a: %v", status)
}
if got := fake.listCalls.Load() - listCallsBefore; got != 0 {
@@ -308,7 +308,7 @@ func TestLookupRebuildCooldownAfterFailure(t *testing.T) {
advanceRebuildAttempt(t, wfs.inodeToPath, dir, expiredDirRebuildCooldown+time.Second)
listCallsBefore = fake.listCalls.Load()
lookupCallsBefore = fake.lookupCalls.Load()
if _, _, status := wfs.lookupEntry(dir.Child("a")); status != fuse.OK {
if _, _, status := wfs.lookupEntry(context.Background(), dir.Child("a")); status != fuse.OK {
t.Fatalf("third lookupEntry a: %v", status)
}
if got := fake.listCalls.Load() - listCallsBefore; got != 1 {
weed/mount/weedfs_dir_mkrm.go
+9 -2
View File
@@ -156,13 +156,20 @@ func (wfs *WFS) Rmdir(cancel <-chan struct{}, header *fuse.InHeader, name string
}
entryFullPath := dirFullPath.Child(name)
targetEntry, _, targetCode := wfs.maybeLoadEntry(entryFullPath)
ctx, cancelFunc := ctxFromCancel(cancel)
defer cancelFunc()
targetEntry, _, targetCode := wfs.maybeLoadEntry(ctx, entryFullPath)
if targetCode == fuse.EINTR {
return targetCode
}
if targetCode != fuse.OK {
targetEntry = nil
}
// POSIX: enforce sticky bit on the parent directory.
if dirEntry, _, dirCode := wfs.maybeLoadEntry(dirFullPath); dirCode == fuse.OK && dirEntry != nil && dirEntry.Attributes != nil {
if dirEntry, _, dirCode := wfs.maybeLoadEntry(ctx, dirFullPath); dirCode != fuse.OK {
return dirCode
} else if dirEntry != nil && dirEntry.Attributes != nil {
targetUid := uint32(0)
if targetEntry != nil && targetEntry.Attributes != nil {
targetUid = targetEntry.Attributes.Uid
weed/mount/weedfs_dir_read.go
+44 -19
View File
@@ -176,14 +176,20 @@ func (wfs *WFS) FsyncDir(cancel <-chan struct{}, input *fuse.FsyncIn) (code fuse
* '1'.
*/
func (wfs *WFS) ReadDir(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
return wfs.doReadDirectory(input, fuseDirEntryList{out}, false)
ctx, cancelFunc := ctxFromCancel(cancel)
defer cancelFunc()
return wfs.doReadDirectory(ctx, input, fuseDirEntryList{out}, false)
}
func (wfs *WFS) ReadDirPlus(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
return wfs.doReadDirectory(input, fuseDirEntryList{out}, true)
ctx, cancelFunc := ctxFromCancel(cancel)
defer cancelFunc()
return wfs.doReadDirectory(ctx, input, fuseDirEntryList{out}, true)
}
func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out DirEntrySink, isPlusMode bool) fuse.Status {
func (wfs *WFS) doReadDirectory(ctx context.Context, input *fuse.ReadIn, out DirEntrySink, isPlusMode bool) fuse.Status {
listingCtx := filer_pb.WithChunksOmitted(ctx)
// Get the directory handle and lock it for the duration of this operation.
// This serializes concurrent readdir calls on the same handle, fixing the
// race condition that caused hangs with NFS-Ganesha.
@@ -282,7 +288,7 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out DirEntrySink, isPlusMode
dh.dropConsumed(input.Offset)
if wfs.inodeToPath.ShouldReadDirectoryDirect(dirPath) {
return wfs.readDirectoryDirect(input, out, dh, dirPath, processEachEntryFn)
return wfs.readDirectoryDirect(ctx, input, out, dh, dirPath, processEachEntryFn)
}
// Read from cache first, then load next batch if needed
@@ -292,32 +298,38 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out DirEntrySink, isPlusMode
if len(dh.entryStream) == 0 && input.Offset > dh.entryStreamOffset {
skipCount := int64(input.Offset - dh.entryStreamOffset)
if err := wfs.ensureDirectoryVisited(dirPath); err != nil {
if err := wfs.ensureDirectoryVisited(ctx, dirPath); err != nil {
var tooLarge *meta_cache.DirectoryTooLargeError
if errors.As(err, &tooLarge) {
return wfs.readDirectoryDirect(input, out, dh, dirPath, processEachEntryFn)
return wfs.readDirectoryDirect(ctx, input, out, dh, dirPath, processEachEntryFn)
}
if ctx.Err() != nil {
return fuse.EINTR
}
glog.Errorf("dir ReadDirAll %s: %v", dirPath, err)
return fuse.EIO
}
// Serve the maintained-but-unverified cache if the filer is unreachable.
if err := meta_cache.EnsureListingFresh(context.Background(), wfs.metaCache, wfs, dirPath, ""); err != nil {
if err := meta_cache.EnsureListingFresh(ctx, wfs.metaCache, wfs, dirPath, ""); err != nil {
if errors.Is(err, meta_cache.ErrRefreshRangeTooLarge) {
// re-tile with a full rebuild; serve this request direct
wfs.purgeDirectoryCache(dirPath)
return wfs.readDirectoryDirect(input, out, dh, dirPath, processEachEntryFn)
return wfs.readDirectoryDirect(ctx, input, out, dh, dirPath, processEachEntryFn)
}
glog.V(1).Infof("refresh %s sections: %v", dirPath, err)
}
// Load entries from beginning to fill cache up to the requested offset
storeLastName, loadErr := wfs.metaCache.ListDirectoryEntries(readdirContext, dirPath, "", false, skipCount+int64(batchSize), func(entry *filer.Entry) (bool, error) {
storeLastName, loadErr := wfs.metaCache.ListDirectoryEntries(listingCtx, dirPath, "", false, skipCount+int64(batchSize), func(entry *filer.Entry) (bool, error) {
dh.entryStream = append(dh.entryStream, entry)
return true, nil
})
dh.lastListedName = storeLastName
if loadErr != nil {
if ctx.Err() != nil {
return fuse.EINTR
}
glog.Errorf("list meta cache: %v", loadErr)
return fuse.EIO
}
@@ -349,12 +361,15 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out DirEntrySink, isPlusMode
}
// Cache exhausted, load next batch
if err := wfs.ensureDirectoryVisited(dirPath); err != nil {
if err := wfs.ensureDirectoryVisited(ctx, dirPath); err != nil {
var tooLarge *meta_cache.DirectoryTooLargeError
if errors.As(err, &tooLarge) {
// The direct path keeps the same pagination state on dh, so it
// carries on from wherever the cached walk reached.
return wfs.readDirectoryDirect(input, out, dh, dirPath, processEachEntryFn)
return wfs.readDirectoryDirect(ctx, input, out, dh, dirPath, processEachEntryFn)
}
if ctx.Err() != nil {
return fuse.EINTR
}
glog.Errorf("dir ReadDirAll %s: %v", dirPath, err)
return fuse.EIO
@@ -369,17 +384,17 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out DirEntrySink, isPlusMode
}
// Serve the maintained-but-unverified cache if the filer is unreachable.
if err := meta_cache.EnsureListingFresh(context.Background(), wfs.metaCache, wfs, dirPath, lastEntryName); err != nil {
if err := meta_cache.EnsureListingFresh(ctx, wfs.metaCache, wfs, dirPath, lastEntryName); err != nil {
if errors.Is(err, meta_cache.ErrRefreshRangeTooLarge) {
// re-tile with a full rebuild; serve this request direct
wfs.purgeDirectoryCache(dirPath)
return wfs.readDirectoryDirect(input, out, dh, dirPath, processEachEntryFn)
return wfs.readDirectoryDirect(ctx, input, out, dh, dirPath, processEachEntryFn)
}
glog.V(1).Infof("refresh %s sections: %v", dirPath, err)
}
bufferFull := false
storeLastName, loadErr := wfs.metaCache.ListDirectoryEntries(readdirContext, dirPath, lastEntryName, false, int64(batchSize), func(entry *filer.Entry) (bool, error) {
storeLastName, loadErr := wfs.metaCache.ListDirectoryEntries(listingCtx, dirPath, lastEntryName, false, int64(batchSize), func(entry *filer.Entry) (bool, error) {
currentIndex := int64(len(dh.entryStream))
dh.entryStream = append(dh.entryStream, entry)
if !processEachEntryFn(entry, currentIndex) {
@@ -389,6 +404,9 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out DirEntrySink, isPlusMode
return true, nil
})
if loadErr != nil {
if ctx.Err() != nil {
return fuse.EINTR
}
glog.Errorf("list meta cache: %v", loadErr)
return fuse.EIO
}
@@ -408,8 +426,8 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out DirEntrySink, isPlusMode
// ensureDirectoryVisited pulls the directory into the local cache, unless it is
// too large to cache: then the directory is marked read-through, so later
// listings go straight to the filer without re-asking.
func (wfs *WFS) ensureDirectoryVisited(dirPath util.FullPath) error {
err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath, wfs.option.CacheDirMaxEntries)
func (wfs *WFS) ensureDirectoryVisited(ctx context.Context, dirPath util.FullPath) error {
err := meta_cache.EnsureVisited(ctx, wfs.metaCache, wfs, dirPath, wfs.option.CacheDirMaxEntries)
var tooLarge *meta_cache.DirectoryTooLargeError
if errors.As(err, &tooLarge) {
wfs.inodeToPath.MarkDirectoryReadThrough(dirPath, time.Now())
@@ -417,14 +435,18 @@ func (wfs *WFS) ensureDirectoryVisited(dirPath util.FullPath) error {
return err
}
func (wfs *WFS) readDirectoryDirect(input *fuse.ReadIn, out DirEntrySink, dh *DirectoryHandle, dirPath util.FullPath, processEachEntryFn func(entry *filer.Entry, index int64) bool) fuse.Status {
func (wfs *WFS) readDirectoryDirect(ctx context.Context, input *fuse.ReadIn, out DirEntrySink, dh *DirectoryHandle, dirPath util.FullPath, processEachEntryFn func(entry *filer.Entry, index int64) bool) fuse.Status {
listingCtx := filer_pb.WithChunksOmitted(ctx)
var lastEntryName string
if input.Offset >= dh.entryStreamOffset {
if len(dh.entryStream) == 0 && input.Offset > dh.entryStreamOffset {
skipCount := uint32(input.Offset-dh.entryStreamOffset) + batchSize
entries, snapshotTs, err := loadDirectoryEntriesDirect(readdirContext, wfs, wfs.option.UidGidMapper, dirPath, "", false, skipCount, dh.snapshotTsNs, wfs.option.IncludeSystemEntries)
entries, snapshotTs, err := loadDirectoryEntriesDirect(listingCtx, wfs, wfs.option.UidGidMapper, dirPath, "", false, skipCount, dh.snapshotTsNs, wfs.option.IncludeSystemEntries)
if err != nil {
if ctx.Err() != nil {
return fuse.EINTR
}
glog.Errorf("list filer directory: %v", err)
return fuse.EIO
}
@@ -457,8 +479,11 @@ func (wfs *WFS) readDirectoryDirect(input *fuse.ReadIn, out DirEntrySink, dh *Di
}
}
entries, snapshotTs, err := loadDirectoryEntriesDirect(readdirContext, wfs, wfs.option.UidGidMapper, dirPath, lastEntryName, false, batchSize, dh.snapshotTsNs, wfs.option.IncludeSystemEntries)
entries, snapshotTs, err := loadDirectoryEntriesDirect(listingCtx, wfs, wfs.option.UidGidMapper, dirPath, lastEntryName, false, batchSize, dh.snapshotTsNs, wfs.option.IncludeSystemEntries)
if err != nil {
if ctx.Err() != nil {
return fuse.EINTR
}
glog.Errorf("list filer directory: %v", err)
return fuse.EIO
}
weed/mount/weedfs_dir_read_bench_test.go
+1 -1
View File
@@ -163,7 +163,7 @@ func walkOnce(tb testing.TB, wfs *WFS, dirInode uint64, sink *benchSink, forgets
offset := uint64(0)
for {
sink.reset()
status := wfs.doReadDirectory(&fuse.ReadIn{
status := wfs.doReadDirectory(context.Background(), &fuse.ReadIn{
InHeader: fuse.InHeader{NodeId: dirInode},
Fh: uint64(dhid),
Offset: offset,
weed/mount/weedfs_dir_read_pagination_test.go
+4 -4
View File
@@ -111,7 +111,7 @@ func TestReadDirResumePastShrunkDirectory(t *testing.T) {
defer wfs.ReleaseDirectoryHandle(dhid)
sink := &pagingSink{limit: 4096}
status := wfs.doReadDirectory(&fuse.ReadIn{
status := wfs.doReadDirectory(context.Background(), &fuse.ReadIn{
InHeader: fuse.InHeader{NodeId: dirInode},
Fh: uint64(dhid),
Offset: 152, // past the end of a directory that now holds 100
@@ -158,7 +158,7 @@ func TestReadDirWithExpiredEntries(t *testing.T) {
var offset uint64
for round := 0; round < 20; round++ {
sink.round = 0
status := wfs.doReadDirectory(&fuse.ReadIn{
status := wfs.doReadDirectory(context.Background(), &fuse.ReadIn{
InHeader: fuse.InHeader{NodeId: dirInode},
Fh: uint64(dhid),
Offset: offset,
@@ -209,7 +209,7 @@ func TestReadDirTrimsConsumedEntries(t *testing.T) {
for round := 0; round < 500; round++ {
sink.round = 0
before := len(sink.names)
status := wfs.doReadDirectory(&fuse.ReadIn{
status := wfs.doReadDirectory(context.Background(), &fuse.ReadIn{
InHeader: fuse.InHeader{NodeId: dirInode},
Fh: uint64(dhid),
Offset: offset,
@@ -301,7 +301,7 @@ func TestReadDirDirectTrimsConsumedEntries(t *testing.T) {
for round := 0; round < 500; round++ {
sink.round = 0
before := len(sink.names)
status := wfs.doReadDirectory(&fuse.ReadIn{
status := wfs.doReadDirectory(context.Background(), &fuse.ReadIn{
InHeader: fuse.InHeader{NodeId: dirInode},
Fh: uint64(dhid),
Offset: offset,
weed/mount/weedfs_file_io.go
+3 -1
View File
@@ -65,7 +65,9 @@ import (
*/
func (wfs *WFS) Open(cancel <-chan struct{}, in *fuse.OpenIn, out *fuse.OpenOut) (status fuse.Status) {
var fileHandle *FileHandle
fileHandle, status = wfs.AcquireHandle(in.NodeId, in.Flags, in.Uid, in.Gid)
ctx, cancelFunc := ctxFromCancel(cancel)
defer cancelFunc()
fileHandle, status = wfs.AcquireHandle(ctx, in.NodeId, in.Flags, in.Uid, in.Gid)
if status == fuse.OK {
out.Fh = uint64(fileHandle.fh)
out.OpenFlags = 0
weed/mount/weedfs_file_mkrm.go
+22 -14
View File
@@ -41,7 +41,9 @@ func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, o
entryFullPath := dirFullPath.Child(name)
var inode uint64
newEntry, _, code := wfs.maybeLoadEntry(entryFullPath)
ctx, cancelFunc := ctxFromCancel(cancel)
defer cancelFunc()
newEntry, _, code := wfs.maybeLoadEntry(ctx, entryFullPath)
if code == fuse.OK {
if newEntry == nil || newEntry.Attributes == nil {
return fuse.EIO
@@ -52,7 +54,7 @@ func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, o
return fuse.Status(syscall.EEXIST)
}
inode = wfs.inodeToPath.Lookup(entryFullPath, newEntry.Attributes.Crtime, false, len(newEntry.HardLinkId) > 0, newEntry.Attributes.Inode, true)
fileHandle, status := wfs.AcquireHandle(inode, in.Flags, in.Uid, in.Gid)
fileHandle, status := wfs.AcquireHandle(ctx, inode, in.Flags, in.Uid, in.Gid)
if status != fuse.OK {
return status
}
@@ -86,14 +88,14 @@ func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, o
}
}
inode, newEntry, code = wfs.createRegularFile(dirFullPath, name, in.Mode, in.Uid, in.Gid, 0, !wfs.option.EagerFilerCreate, true)
inode, newEntry, code = wfs.createRegularFile(ctx, dirFullPath, name, in.Mode, in.Uid, in.Gid, 0, !wfs.option.EagerFilerCreate, true)
if code == fuse.Status(syscall.EEXIST) && in.Flags&syscall.O_EXCL == 0 {
// Race: another process created the file between our check and create.
// Reopen the winner's entry; AcquireHandle takes its own lock.
if dlmLock != nil {
dlmLock.Stop()
}
newEntry, _, code = wfs.maybeLoadEntry(entryFullPath)
newEntry, _, code = wfs.maybeLoadEntry(ctx, entryFullPath)
if code != fuse.OK {
return code
}
@@ -101,7 +103,7 @@ func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, o
return fuse.EIO
}
inode = wfs.inodeToPath.Lookup(entryFullPath, newEntry.Attributes.Crtime, false, len(newEntry.HardLinkId) > 0, newEntry.Attributes.Inode, true)
fileHandle, status := wfs.AcquireHandle(inode, in.Flags, in.Uid, in.Gid)
fileHandle, status := wfs.AcquireHandle(ctx, inode, in.Flags, in.Uid, in.Gid)
if status != fuse.OK {
return status
}
@@ -177,7 +179,9 @@ func (wfs *WFS) Mknod(cancel <-chan struct{}, in *fuse.MknodIn, name string, out
return
}
inode, newEntry, code := wfs.createRegularFile(dirFullPath, name, in.Mode, in.Uid, in.Gid, in.Rdev, false, false)
ctx, cancelFunc := ctxFromCancel(cancel)
defer cancelFunc()
inode, newEntry, code := wfs.createRegularFile(ctx, dirFullPath, name, in.Mode, in.Uid, in.Gid, in.Rdev, false, false)
if code != fuse.OK {
return code
}
@@ -207,7 +211,9 @@ func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name strin
}
entryFullPath := dirFullPath.Child(name)
entry, _, code := wfs.maybeLoadEntry(entryFullPath)
ctx, cancelFunc := ctxFromCancel(cancel)
defer cancelFunc()
entry, _, code := wfs.maybeLoadEntry(ctx, entryFullPath)
if code != fuse.OK {
if code == fuse.ENOENT {
return fuse.OK
@@ -236,7 +242,7 @@ func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name strin
// maybeLoadEntry path above. Do not fall back to the stale
// pre-lock snapshot: proceeding with its HardLinkCounter would
// reintroduce the stale-base update the lock is meant to prevent.
fresh, _, freshCode := wfs.maybeLoadEntry(entryFullPath)
fresh, _, freshCode := wfs.maybeLoadEntry(ctx, entryFullPath)
if freshCode == fuse.ENOENT {
return fuse.OK
}
@@ -247,7 +253,9 @@ func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name strin
}
// POSIX: enforce sticky bit on the parent directory.
if dirEntry, _, dirCode := wfs.maybeLoadEntry(dirFullPath); dirCode == fuse.OK && dirEntry != nil && dirEntry.Attributes != nil {
if dirEntry, _, dirCode := wfs.maybeLoadEntry(ctx, dirFullPath); dirCode != fuse.OK {
return dirCode
} else if dirEntry != nil && dirEntry.Attributes != nil {
targetUid := uint32(0)
if entry != nil && entry.Attributes != nil {
targetUid = entry.Attributes.Uid
@@ -266,11 +274,11 @@ func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name strin
// will remove it again.
if inode, found := wfs.inodeToPath.GetInode(entryFullPath); found {
wfs.markHandleDeleted(inode)
wfs.waitForPendingAsyncFlush(inode)
wfs.waitForPendingAsyncFlush(context.Background(), inode)
} else if entry != nil && entry.Attributes != nil && entry.Attributes.Inode != 0 {
inodeFromEntry := entry.Attributes.Inode
wfs.markHandleDeleted(inodeFromEntry)
wfs.waitForPendingAsyncFlush(inodeFromEntry)
wfs.waitForPendingAsyncFlush(context.Background(), inodeFromEntry)
}
// first, ensure the filer store can correctly delete
@@ -337,7 +345,7 @@ func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name strin
}
func (wfs *WFS) createRegularFile(dirFullPath util.FullPath, name string, mode uint32, uid, gid, rdev uint32, deferFilerCreate bool, skipExistenceCheck bool) (inode uint64, newEntry *filer_pb.Entry, code fuse.Status) {
func (wfs *WFS) createRegularFile(ctx context.Context, dirFullPath util.FullPath, name string, mode uint32, uid, gid, rdev uint32, deferFilerCreate bool, skipExistenceCheck bool) (inode uint64, newEntry *filer_pb.Entry, code fuse.Status) {
if wfs.IsOverQuotaWithUncommitted() {
return 0, nil, fuse.Status(syscall.ENOSPC)
}
@@ -346,7 +354,7 @@ func (wfs *WFS) createRegularFile(dirFullPath util.FullPath, name string, mode u
// SkipCheckParentDirectory, so this is the only parent check). With
// default_permissions the kernel already verified write+search on it before
// Create/Mknod, so skip only the mode-bit check and its group lookup.
parentEntry, _, parentStatus := wfs.maybeLoadEntry(dirFullPath)
parentEntry, _, parentStatus := wfs.maybeLoadEntry(ctx, dirFullPath)
if parentStatus != fuse.OK {
return 0, nil, parentStatus
}
@@ -367,7 +375,7 @@ func (wfs *WFS) createRegularFile(dirFullPath util.FullPath, name string, mode u
entryFullPath := dirFullPath.Child(name)
if !skipExistenceCheck {
if _, _, status := wfs.maybeLoadEntry(entryFullPath); status == fuse.OK {
if _, _, status := wfs.maybeLoadEntry(ctx, entryFullPath); status == fuse.OK {
return 0, nil, fuse.Status(syscall.EEXIST)
} else if status != fuse.ENOENT {
return 0, nil, status
weed/mount/weedfs_file_mkrm_test.go
+2 -2
View File
@@ -528,7 +528,7 @@ func TestAcquireHandleHonorsDefaultPermissions(t *testing.T) {
}
inode := wfs.inodeToPath.Lookup(util.FullPath("/secret.txt"), entry.Attributes.Crtime, false, false, entry.Attributes.Inode, true)
fh, status := wfs.AcquireHandle(inode, syscall.O_RDONLY, 999, 999)
fh, status := wfs.AcquireHandle(context.Background(), inode, syscall.O_RDONLY, 999, 999)
if status != tc.want {
t.Fatalf("AcquireHandle status = %v, want %v", status, tc.want)
}
@@ -549,7 +549,7 @@ func TestCreateRegularFileValidatesParentUnderDefaultPermissions(t *testing.T) {
wfs, _ := newCreateTestWFS(t)
wfs.option.DefaultPermissions = true
if _, _, code := wfs.createRegularFile(util.FullPath("/ghost"), "f.txt", 0o644, 99, 100, 0, false, false); code == fuse.OK {
if _, _, code := wfs.createRegularFile(context.Background(), util.FullPath("/ghost"), "f.txt", 0o644, 99, 100, 0, false, false); code == fuse.OK {
t.Fatal("createRegularFile returned OK for a missing parent directory")
}
}
weed/mount/weedfs_file_read.go
+2 -11
View File
@@ -47,17 +47,8 @@ func (wfs *WFS) Read(cancel <-chan struct{}, in *fuse.ReadIn, buff []byte) (fuse
fhActiveLock := fh.wfs.fhLockTable.AcquireLock("Read", fh.fh, util.SharedLock)
defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
// Create a context that will be cancelled when the cancel channel receives a signal
ctx, cancelFunc := context.WithCancel(context.Background())
defer cancelFunc() // Ensure cleanup
go func() {
select {
case <-cancel:
cancelFunc()
case <-ctx.Done():
}
}()
ctx, cancelFunc := ctxFromCancel(cancel)
defer cancelFunc()
offset := int64(in.Offset)
totalRead, err := readDataByFileHandleWithContext(ctx, buff, fh, offset)
weed/mount/weedfs_filehandle.go
+16 -7
View File
@@ -1,6 +1,7 @@
package mount
import (
"context"
"fmt"
"syscall"
@@ -10,12 +11,14 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
func (wfs *WFS) AcquireHandle(inode uint64, flags, uid, gid uint32) (fileHandle *FileHandle, status fuse.Status) {
func (wfs *WFS) AcquireHandle(ctx context.Context, inode uint64, flags, uid, gid uint32) (fileHandle *FileHandle, status fuse.Status) {
// If there is an in-flight async flush for this inode, wait for it to
// complete before reopening. Otherwise the new handle would be built
// from pre-close filer metadata and its next flush could overwrite the
// data that was just written asynchronously.
wfs.waitForPendingAsyncFlush(inode)
if !wfs.waitForPendingAsyncFlush(ctx, inode) {
return nil, fuse.EINTR
}
path, pathStatus := wfs.inodeToPath.GetPath(inode)
if pathStatus != fuse.OK {
@@ -38,7 +41,7 @@ func (wfs *WFS) AcquireHandle(inode uint64, flags, uid, gid uint32) (fileHandle
})
entryVersionTsNs = entryVersion{tsNs: existingFh.entryVersionTsNs.Load(), signature: existingFh.entryVersionSignature.Load()}
} else {
entry, entryVersionTsNs, status = wfs.maybeLoadEntry(path)
entry, entryVersionTsNs, status = wfs.maybeLoadEntry(ctx, path)
if status != fuse.OK {
return nil, status
}
@@ -125,14 +128,20 @@ func (wfs *WFS) ReleaseHandle(handleId FileHandleId) {
// waitForPendingAsyncFlush blocks until any in-flight async flush for
// the given inode completes. Called from AcquireHandle before building
// new handle state, so the filer metadata reflects the flushed data.
func (wfs *WFS) waitForPendingAsyncFlush(inode uint64) {
func (wfs *WFS) waitForPendingAsyncFlush(ctx context.Context, inode uint64) bool {
wfs.pendingAsyncFlushMu.Lock()
done, found := wfs.pendingAsyncFlush[inode]
wfs.pendingAsyncFlushMu.Unlock()
if found {
glog.V(3).Infof("waitForPendingAsyncFlush: waiting for inode %d", inode)
<-done
if !found {
return true
}
glog.V(3).Infof("waitForPendingAsyncFlush: waiting for inode %d", inode)
select {
case <-done:
return true
case <-ctx.Done():
return false
}
}
weed/mount/weedfs_invalidate_open_handle_test.go
+19 -19
View File
@@ -499,7 +499,7 @@ func TestQueuedEventDoesNotRollBackHandleOpenedAfterEnqueue(t *testing.T) {
// Open now: the lookup reaches the filer, which serves the newer
// size-200 state versioned at log position 2000.
inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false)
fh, status := wfs.AcquireHandle(inode, 0, 0, 0)
fh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
if status != fuse.OK {
t.Fatalf("AcquireHandle status = %v, want OK", status)
}
@@ -563,7 +563,7 @@ func TestAbortedBuildEventDoesNotRollBackLaterOpen(t *testing.T) {
}
inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false)
fh, status := wfs.AcquireHandle(inode, 0, 0, 0)
fh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
if status != fuse.OK {
t.Fatalf("AcquireHandle status = %v, want OK", status)
}
@@ -626,7 +626,7 @@ func TestEventDuringOpenLookupIsFenced(t *testing.T) {
}
opened := make(chan openResult, 1)
go func() {
fh, status := wfs.AcquireHandle(inode, 0, 0, 0)
fh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
opened <- openResult{fh, status}
}()
@@ -672,7 +672,7 @@ func TestRemoteCacheResponseVersionFencesUndeliveredEvents(t *testing.T) {
Attributes: &filer_pb.FuseAttributes{FileSize: 88},
}, 0, 0)
if err := fh.downloadRemoteEntry(fh.GetEntry()); err != nil {
if err := fh.downloadRemoteEntry(context.Background(), fh.GetEntry()); err != nil {
t.Fatalf("downloadRemoteEntry: %v", err)
}
if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 200 {
@@ -755,7 +755,7 @@ func TestLocalAckDoesNotFenceUnrelatedDelayedEvents(t *testing.T) {
// Open the file: the cached-store read must not claim position 2000 —
// the local ack versioned only its own path.
inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false)
fh, status := wfs.AcquireHandle(inode, 0, 0, 0)
fh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
if status != fuse.OK {
t.Fatalf("AcquireHandle status = %v, want OK", status)
}
@@ -794,14 +794,14 @@ func TestSlowerConcurrentOpenDoesNotOverwriteNewerHandle(t *testing.T) {
}
slow := make(chan openResult, 1)
go func() {
fh, status := wfs.AcquireHandle(inode, 0, 0, 0)
fh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
slow <- openResult{fh, status}
}()
<-fake.lookupStarted
// The faster opener completes with newer state while the slow lookup is
// still in flight.
fastFh, status := wfs.AcquireHandle(inode, 0, 0, 0)
fastFh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
if status != fuse.OK {
t.Fatalf("fast AcquireHandle status = %v, want OK", status)
}
@@ -925,12 +925,12 @@ func TestSlowerOpenRejectedWhenHandleDirtyOrLookupUnversioned(t *testing.T) {
}
slow := make(chan openResult, 1)
go func() {
fh, status := wfs.AcquireHandle(inode, 0, 0, 0)
fh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
slow <- openResult{fh, status}
}()
<-fake.lookupStarted
fastFh, status := wfs.AcquireHandle(inode, 0, 0, 0)
fastFh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
if status != fuse.OK {
t.Fatalf("fast AcquireHandle status = %v, want OK", status)
}
@@ -969,12 +969,12 @@ func TestUnversionedSlowerOpenDoesNotOverwrite(t *testing.T) {
}
slow := make(chan openResult, 1)
go func() {
fh, status := wfs.AcquireHandle(inode, 0, 0, 0)
fh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
slow <- openResult{fh, status}
}()
<-fake.lookupStarted
fastFh, status := wfs.AcquireHandle(inode, 0, 0, 0)
fastFh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
if status != fuse.OK {
t.Fatalf("fast AcquireHandle status = %v, want OK", status)
}
@@ -1405,7 +1405,7 @@ func TestEmptyListingTrailerSnapshotSetsAbsenceFloor(t *testing.T) {
startFakeFiler(t, wfs, &fakeFilerServer{listSnapshotTrailerTsNs: 4000})
wfs.inodeToPath.Lookup(util.FullPath("/dir"), time.Now().Unix(), true, false, 0, false)
if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, util.FullPath("/dir"), 0); err != nil {
if err := meta_cache.EnsureVisited(context.Background(), wfs.metaCache, wfs, util.FullPath("/dir"), 0); err != nil {
t.Fatalf("EnsureVisited: %v", err)
}
@@ -1506,7 +1506,7 @@ func TestRemoteDownloadBaseMappedToLocal(t *testing.T) {
RemoteEntry: &filer_pb.RemoteEntry{RemoteSize: 200},
}, 0, 0)
if err := fh.downloadRemoteEntry(fh.GetEntry()); err != nil {
if err := fh.downloadRemoteEntry(context.Background(), fh.GetEntry()); err != nil {
t.Fatalf("downloadRemoteEntry: %v", err)
}
// The base — and the live entry — must be in local uid form.
@@ -1617,7 +1617,7 @@ func TestStaleRemoteDownloadDoesNotRollBack(t *testing.T) {
})
fh.advanceEntryVersion(3000, 0)
if err := fh.downloadRemoteEntry(fh.GetEntry()); err != nil {
if err := fh.downloadRemoteEntry(context.Background(), fh.GetEntry()); err != nil {
t.Fatalf("downloadRemoteEntry: %v", err)
}
@@ -1644,7 +1644,7 @@ func TestRemoteOnlyHandleTakesUnversionedDownload(t *testing.T) {
}, 0, 0)
fh.advanceEntryVersion(3000, 0)
if err := fh.downloadRemoteEntry(fh.GetEntry()); err != nil {
if err := fh.downloadRemoteEntry(context.Background(), fh.GetEntry()); err != nil {
t.Fatalf("downloadRemoteEntry: %v", err)
}
@@ -1801,7 +1801,7 @@ func TestUnversionedRemoteDownloadDoesNotOverwriteVersioned(t *testing.T) {
})
fh.advanceEntryVersion(3000, 0)
if err := fh.downloadRemoteEntry(fh.GetEntry()); err != nil {
if err := fh.downloadRemoteEntry(context.Background(), fh.GetEntry()); err != nil {
t.Fatalf("downloadRemoteEntry: %v", err)
}
@@ -1820,7 +1820,7 @@ func TestEventFromAnotherFilerIsNotFencedByForeignClock(t *testing.T) {
startFakeFiler(t, wfs, &fakeFilerServer{lookupSize: 200, lookupLogTsNs: 5000, lookupSignature: 11})
inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false)
fh, status := wfs.AcquireHandle(inode, 0, 0, 0)
fh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
if status != fuse.OK {
t.Fatalf("AcquireHandle status = %v, want OK", status)
}
@@ -1931,7 +1931,7 @@ func TestRejectedDownloadDoesNotRollBackCache(t *testing.T) {
}, 0, 0)
fh.advanceEntryVersion(3000, 0)
if err := fh.downloadRemoteEntry(fh.GetEntry()); err != nil {
if err := fh.downloadRemoteEntry(context.Background(), fh.GetEntry()); err != nil {
t.Fatalf("downloadRemoteEntry: %v", err)
}
// The download publishes asynchronously; a synchronous apply behind it
@@ -1966,7 +1966,7 @@ func TestRemoteOnlyHandleRefusesKnownOlderDownload(t *testing.T) {
}, 0, 0)
fh.advanceEntryVersion(3000, 0)
if err := fh.downloadRemoteEntry(fh.GetEntry()); err != nil {
if err := fh.downloadRemoteEntry(context.Background(), fh.GetEntry()); err != nil {
t.Fatalf("downloadRemoteEntry: %v", err)
}
weed/mount/weedfs_link.go
+9 -8
View File
@@ -57,7 +57,9 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
// probe can move the two apart even for an entry that does carry one.
sourceInode := in.Oldnodeid
oldEntry, _, status := wfs.maybeLoadEntry(oldEntryPath)
ctx, cancelFunc := ctxFromCancel(cancel)
defer cancelFunc()
oldEntry, _, status := wfs.maybeLoadEntry(ctx, oldEntryPath)
if status != fuse.OK {
return status
}
@@ -89,7 +91,7 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
// Do not fall back to the pre-lock snapshot: if every alias
// was deleted while we waited, abort instead of deriving the
// next counter from a stale entry.
fresh, _, freshStatus := wfs.maybeLoadEntry(oldEntryPath)
fresh, _, freshStatus := wfs.maybeLoadEntry(ctx, oldEntryPath)
if freshStatus != fuse.OK {
return freshStatus
}
@@ -137,21 +139,20 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
// apply changes to the filer, and also apply to local metaCache
wfs.mapPbIdFromLocalToFiler(request.Entry)
ctx := context.Background()
updateResp, err := wfs.streamUpdateEntry(ctx, updateOldEntryRequest)
updateResp, err := wfs.streamUpdateEntry(context.Background(), updateOldEntryRequest)
if err == nil {
updateEvent := updateResp.GetMetadataEvent()
if updateEvent == nil {
updateEvent = metadataUpdateEvent(oldParentPath, updateOldEntryRequest.Entry)
}
if applyErr := wfs.applyLocalMetadataEvent(ctx, updateEvent); applyErr != nil {
if applyErr := wfs.applyLocalMetadataEvent(context.Background(), updateEvent); applyErr != nil {
glog.Warningf("link %s: best-effort metadata apply failed: %v", oldEntryPath, applyErr)
wfs.inodeToPath.InvalidateChildrenCache(util.FullPath(oldParentPath))
}
}
if err == nil {
var createResp *filer_pb.CreateEntryResponse
createResp, err = wfs.streamCreateEntry(ctx, request)
createResp, err = wfs.streamCreateEntry(context.Background(), request)
if err != nil {
// Rollback: restore original HardLinkId/Counter on the source entry
oldEntry.HardLinkId = origHardLinkId
@@ -161,7 +162,7 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
Entry: oldEntry,
Signatures: []int32{wfs.signature},
}
if _, rollbackErr := wfs.streamUpdateEntry(ctx, rollbackReq); rollbackErr != nil {
if _, rollbackErr := wfs.streamUpdateEntry(context.Background(), rollbackReq); rollbackErr != nil {
glog.Warningf("link rollback %s: %v", oldEntryPath, rollbackErr)
}
} else {
@@ -169,7 +170,7 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
if createEvent == nil {
createEvent = metadataCreateEvent(string(newParentPath), request.Entry)
}
if applyErr := wfs.applyLocalMetadataEvent(ctx, createEvent); applyErr != nil {
if applyErr := wfs.applyLocalMetadataEvent(context.Background(), createEvent); applyErr != nil {
glog.Warningf("link %s: best-effort metadata apply failed: %v", newParentPath.Child(name), applyErr)
wfs.inodeToPath.InvalidateChildrenCache(newParentPath)
}
weed/mount/weedfs_link_test.go
+2 -2
View File
@@ -106,7 +106,7 @@ func TestLinkReportsKernelNodeIdWhenEntryHasNoPersistedInode(t *testing.T) {
if _, status := wfs.inodeToPath.GetPath(sourceInode); status != fuse.OK {
t.Fatalf("GetPath(%d) = %v, want OK", sourceInode, status)
}
linked, _, status := wfs.maybeLoadEntry(sourcePath)
linked, _, status := wfs.maybeLoadEntry(context.Background(), sourcePath)
if status != fuse.OK {
t.Fatalf("reload source: %v", status)
}
@@ -229,7 +229,7 @@ func TestLinkSyncsHardLinkSiblingsOnThirdLink(t *testing.T) {
// any sibling returns the attributes of the last write to any of them,
// synced or not. Feeding the loop an authoritative counter that no other
// write has stored is what makes its work visible.
authoritative, _, status := wfs.maybeLoadEntry(sourcePath)
authoritative, _, status := wfs.maybeLoadEntry(context.Background(), sourcePath)
if status != fuse.OK {
t.Fatalf("reload source: %v", status)
}
weed/mount/weedfs_posix_lock_routed.go
+3 -3
View File
@@ -94,7 +94,7 @@ func (wfs *WFS) posixLockKeyForInode(inode uint64) (string, bool) {
if status != fuse.OK {
return "", false
}
if entry, _, st := wfs.maybeLoadEntry(path); st == fuse.OK && entry != nil && len(entry.HardLinkId) > 0 {
if entry, _, st := wfs.maybeLoadEntry(context.Background(), path); st == fuse.OK && entry != nil && len(entry.HardLinkId) > 0 {
return posixLockKeyPrefix + "hl:" + hex.EncodeToString(entry.HardLinkId), true
}
return posixLockKeyPrefix + string(path), true
@@ -153,7 +153,7 @@ func posixLockContext(cancel <-chan struct{}) (context.Context, context.CancelFu
func (wfs *WFS) callPosixLock(ctx context.Context, key string, op filer_pb.PosixLockOp, lk posixlock.Range) (*filer_pb.PosixLockResponse, error) {
var resp *filer_pb.PosixLockResponse
err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
err := wfs.withFilerClient(ctx, false, wfs.filerRPCWait(), func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
var e error
resp, e = client.PosixLock(ctx, &filer_pb.PosixLockRequest{
Key: key,
@@ -377,7 +377,7 @@ func (wfs *WFS) callPosixReassert(ctx context.Context, key string, locks []posix
Sid: l.Sid, Owner: l.Owner, Pid: l.Pid, IsFlock: l.IsFlock,
})
}
return wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return wfs.withFilerClient(ctx, false, wfs.filerRPCWait(), func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
_, e := client.PosixLock(ctx, &filer_pb.PosixLockRequest{
Key: key,
Op: filer_pb.PosixLockOp_KEEP_ALIVE,
weed/mount/weedfs_quota.go
+2 -2
View File
@@ -110,7 +110,7 @@ func (wfs *WFS) getQuotaCheckInterval() time.Duration {
}
func (wfs *WFS) checkQuotaOnce() {
err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
err := wfs.withFilerClient(context.Background(), false, wfs.filerRPCWait(), func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.StatisticsRequest{
Collection: wfs.option.Collection,
@@ -119,7 +119,7 @@ func (wfs *WFS) checkQuotaOnce() {
DiskType: string(wfs.option.DiskType),
}
resp, err := client.Statistics(context.Background(), request)
resp, err := client.Statistics(ctx, request)
if err != nil {
glog.V(0).Infof("reading quota usage %v: %v", request, err)
return err
weed/mount/weedfs_rename.go
+50 -11
View File
@@ -6,7 +6,9 @@ import (
"fmt"
"io"
"strings"
"sync/atomic"
"syscall"
"time"
"github.com/seaweedfs/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/cluster"
@@ -28,7 +30,28 @@ func (wfs *WFS) doRename(ctx context.Context, request *filer_pb.StreamRenameEntr
glog.V(1).Infof("Rename %s => %s: stream failed, falling back to unary: %v", oldPath, newPath, err)
}
return wfs.WithFilerClient(true, func(client filer_pb.SeaweedFilerClient) error {
stream, streamErr := client.StreamRenameEntry(ctx, request)
// The stream is dedicated to this attempt, so cancelling its context
// abandons just this rename. Bound the silence between events the way
// the mutation mux does: a filer that accepts the stream then stalls
// must not hold the failover walk hostage, but a long rename that
// keeps sending events may take as long as it needs. Local event
// work is not wire silence — a directory rename can spend longer
// than the bound migrating a child's DLM lock — so the timer defers
// while the handler runs instead of cancelling a healthy stream.
streamCtx, cancel := context.WithCancel(ctx)
defer cancel()
var inHandler atomic.Bool
var silence *time.Timer
silence = time.AfterFunc(wfs.filerRPCWait(), func() {
if inHandler.Load() {
silence.Reset(wfs.filerRPCWait())
return
}
cancel()
})
defer silence.Stop()
stream, streamErr := client.StreamRenameEntry(streamCtx, request)
if streamErr != nil {
return fmt.Errorf("dir AtomicRenameEntry %s => %s : %v", oldPath, newPath, streamErr)
}
@@ -38,11 +61,20 @@ func (wfs *WFS) doRename(ctx context.Context, request *filer_pb.StreamRenameEntr
if recvErr == io.EOF {
break
}
if ctx.Err() == nil && streamCtx.Err() != nil {
return fmt.Errorf("dir Rename %s => %s: no event for %s", oldPath, newPath, wfs.filerRPCWait())
}
return fmt.Errorf("dir Rename %s => %s receive: %v", oldPath, newPath, recvErr)
}
if err := wfs.handleRenameResponse(ctx, resp, newPath, newPathLock); err != nil {
inHandler.Store(true)
err := wfs.handleRenameResponse(ctx, resp, newPath, newPathLock)
inHandler.Store(false)
if err != nil {
return err
}
// The event just proved the filer alive; the bound covers the
// wait for the next one, not the whole rename.
silence.Reset(wfs.filerRPCWait())
}
return nil
})
@@ -199,13 +231,17 @@ func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string
}
newPath := newDir.Child(newName)
oldEntry, _, status := wfs.maybeLoadEntry(oldPath)
ctx, cancelFunc := ctxFromCancel(cancel)
defer cancelFunc()
oldEntry, _, status := wfs.maybeLoadEntry(ctx, oldPath)
if status != fuse.OK {
return status
}
// POSIX: enforce sticky bit on the source directory.
if oldDirEntry, _, dirCode := wfs.maybeLoadEntry(oldDir); dirCode == fuse.OK && oldDirEntry != nil && oldDirEntry.Attributes != nil {
if oldDirEntry, _, dirCode := wfs.maybeLoadEntry(ctx, oldDir); dirCode != fuse.OK {
return dirCode
} else if oldDirEntry != nil && oldDirEntry.Attributes != nil {
targetUid := uint32(0)
if oldEntry != nil && oldEntry.Attributes != nil {
targetUid = oldEntry.Attributes.Uid
@@ -217,14 +253,18 @@ func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string
var newEntry *filer_pb.Entry
if in.Flags != RenameNoReplace {
if loaded, _, newStatus := wfs.maybeLoadEntry(newPath); newStatus == fuse.OK {
if loaded, _, newStatus := wfs.maybeLoadEntry(ctx, newPath); newStatus == fuse.OK {
newEntry = loaded
} else if newStatus != fuse.ENOENT {
return newStatus
}
}
// POSIX: enforce sticky bit on the destination directory when replacing an existing entry.
if newEntry != nil {
if newDirEntry, _, dirCode := wfs.maybeLoadEntry(newDir); dirCode == fuse.OK && newDirEntry != nil && newDirEntry.Attributes != nil {
if newDirEntry, _, dirCode := wfs.maybeLoadEntry(ctx, newDir); dirCode != fuse.OK {
return dirCode
} else if newDirEntry != nil && newDirEntry.Attributes != nil {
targetUid := uint32(0)
if newEntry.Attributes != nil {
targetUid = newEntry.Attributes.Uid
@@ -359,13 +399,13 @@ func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string
// skips old-path metadata creation (which would re-insert the
// renamed entry into the meta cache after rename events clean it up).
wfs.fhMap.MarkInodeRenamed(sourceInode)
wfs.waitForPendingAsyncFlush(sourceInode)
wfs.waitForPendingAsyncFlush(context.Background(), sourceInode)
} else if oldEntry != nil && oldEntry.Attributes != nil && oldEntry.Attributes.Inode != 0 {
// GetInode failed (Forget already removed the mapping), but the
// entry's stored inode can still identify pending async flushes.
sourceInode = oldEntry.Attributes.Inode
wfs.fhMap.MarkInodeRenamed(sourceInode)
wfs.waitForPendingAsyncFlush(sourceInode)
wfs.waitForPendingAsyncFlush(context.Background(), sourceInode)
}
// Replacing the destination deletes what it held, so a handle still open
@@ -385,7 +425,7 @@ func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string
// belongs to the destination as it stands, and one that runs
// marked is skipped and takes its handle with it, past the reach
// of the restore below.
wfs.waitForPendingAsyncFlush(targetInode)
wfs.waitForPendingAsyncFlush(context.Background(), targetInode)
epoch := wfs.markHandleDeleted(targetInode)
// A rename that does not happen leaves the destination where it
// was, so its handle has to go back to writing that name.
@@ -406,8 +446,7 @@ func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string
Signatures: []int32{wfs.signature},
}
ctx := context.Background()
err := wfs.doRename(ctx, request, oldPath, newPath, &newPathLock)
err := wfs.doRename(context.Background(), request, oldPath, newPath, &newPathLock)
if err != nil {
glog.V(0).Infof("Rename %s => %s: %v", oldPath, newPath, err)
// Map error strings to FUSE status codes. String matching is used
weed/mount/weedfs_stats.go
+2 -2
View File
@@ -37,7 +37,7 @@ func (wfs *WFS) StatFs(cancel <-chan struct{}, in *fuse.InHeader, out *fuse.Stat
if wfs.stats.lastChecked < time.Now().Unix()-20 {
err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
err := wfs.withFilerClient(context.Background(), false, wfs.filerRPCWait(), func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.StatisticsRequest{
Collection: wfs.option.Collection,
@@ -47,7 +47,7 @@ func (wfs *WFS) StatFs(cancel <-chan struct{}, in *fuse.InHeader, out *fuse.Stat
}
glog.V(4).Infof("reading filer stats: %+v", request)
resp, err := client.Statistics(context.Background(), request)
resp, err := client.Statistics(ctx, request)
if err != nil {
glog.V(0).Infof("reading filer stats %v: %v", request, err)
return err
weed/mount/weedfs_stream_helpers.go
+3 -3
View File
@@ -20,7 +20,7 @@ func (wfs *WFS) streamCreateEntry(ctx context.Context, req *filer_pb.CreateEntry
glog.V(1).Infof("streamCreateEntry %s/%s: stream failed, falling back to unary: %v", req.Directory, req.Entry.Name, err)
}
var resp *filer_pb.CreateEntryResponse
err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
err := wfs.withFilerClient(ctx, false, wfs.filerRPCWait(), func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
var err error
resp, err = filer_pb.CreateEntryWithResponse(ctx, client, req)
return err
@@ -39,7 +39,7 @@ func (wfs *WFS) streamUpdateEntry(ctx context.Context, req *filer_pb.UpdateEntry
glog.V(1).Infof("streamUpdateEntry %s/%s: stream failed, falling back to unary: %v", req.Directory, req.Entry.Name, err)
}
var resp *filer_pb.UpdateEntryResponse
err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
err := wfs.withFilerClient(ctx, false, wfs.filerRPCWait(), func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
var err error
resp, err = client.UpdateEntry(ctx, req)
return err
@@ -58,7 +58,7 @@ func (wfs *WFS) streamDeleteEntry(ctx context.Context, req *filer_pb.DeleteEntry
glog.V(1).Infof("streamDeleteEntry %s/%s: stream failed, falling back to unary: %v", req.Directory, req.Name, err)
}
var resp *filer_pb.DeleteEntryResponse
err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
err := wfs.withFilerClient(ctx, false, wfs.filerRPCWait(), func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
var err error
resp, err = client.DeleteEntry(ctx, req)
if err != nil {
weed/mount/weedfs_stream_mutate.go
+51 -1
View File
@@ -8,6 +8,7 @@ import (
"sync"
"sync/atomic"
"syscall"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -53,6 +54,7 @@ type streamMutateMux struct {
stream filer_pb.SeaweedFiler_StreamMutateEntryClient
cancel context.CancelFunc
grpcConn *grpc.ClientConn // dedicated connection, closed on stream teardown
filerIdx int32 // FilerAddresses index the current stream dialed
closed bool
disabled bool // permanently disabled if filer doesn't support the RPC
stopSend chan struct{} // closed to signal the current sendLoop to exit
@@ -182,22 +184,41 @@ func (m *streamMutateMux) Rename(ctx context.Context, req *filer_pb.StreamRename
case <-ctx.Done():
return ctx.Err()
}
sendTimer := time.NewTimer(m.wfs.filerRPCWait())
select {
case err := <-sendReq.errCh:
sendTimer.Stop()
if err != nil {
return fmt.Errorf("rename send: %w: %v", ErrStreamTransport, err)
}
case <-sendTimer.C:
m.teardownStream(gen)
return fmt.Errorf("rename send: %w: timed out", ErrStreamTransport)
case <-ctx.Done():
sendTimer.Stop()
return ctx.Err()
}
// Collect rename events until is_last=true.
// Collect rename events until is_last=true. The timer bounds silence
// between events, not the whole rename.
respTimer := time.NewTimer(m.wfs.filerRPCWait())
defer respTimer.Stop()
for {
select {
case resp, ok := <-ch:
if !ok {
return fmt.Errorf("rename recv: %w: stream closed", ErrStreamTransport)
}
// Applying an event can legitimately outlast the silence bound —
// a directory rename migrates an open child's DLM lock under
// another writer. The timer bounds wire silence only, so stop it
// for the local work and re-arm it before the next wait.
if !respTimer.Stop() {
select {
case <-respTimer.C:
default:
}
}
if r, ok := resp.Response.(*filer_pb.StreamMutateEntryResponse_RenameResponse); ok {
if r.RenameResponse != nil && r.RenameResponse.EventNotification != nil {
if err := onEvent(r.RenameResponse); err != nil {
@@ -214,6 +235,10 @@ func (m *streamMutateMux) Rename(ctx context.Context, req *filer_pb.StreamRename
}
return nil
}
respTimer.Reset(m.wfs.filerRPCWait())
case <-respTimer.C:
m.teardownStream(gen)
return fmt.Errorf("rename recv: %w: timed out", ErrStreamTransport)
case <-ctx.Done():
return ctx.Err()
}
@@ -243,17 +268,29 @@ func (m *streamMutateMux) doUnary(ctx context.Context, req *filer_pb.StreamMutat
case <-ctx.Done():
return nil, ctx.Err()
}
// Most callers pass context.Background(), so a stream blocked on Send or
// Recv over a dead transport would never surface an error here. Bound the
// wait and tear the stream down so the call can fall back to a unary RPC
// on another filer.
sendTimer := time.NewTimer(m.wfs.filerRPCWait())
select {
case err := <-sendReq.errCh:
sendTimer.Stop()
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrStreamTransport, err)
}
case <-sendTimer.C:
m.teardownStream(gen)
return nil, fmt.Errorf("%w: send timed out", ErrStreamTransport)
case <-ctx.Done():
sendTimer.Stop()
return nil, ctx.Err()
}
respTimer := time.NewTimer(m.wfs.filerRPCWait())
select {
case resp, ok := <-ch:
respTimer.Stop()
if !ok {
return nil, fmt.Errorf("%w: stream closed", ErrStreamTransport)
}
@@ -268,7 +305,11 @@ func (m *streamMutateMux) doUnary(ctx context.Context, req *filer_pb.StreamMutat
}
}
return resp, nil
case <-respTimer.C:
m.teardownStream(gen)
return nil, fmt.Errorf("%w: response timed out", ErrStreamTransport)
case <-ctx.Done():
respTimer.Stop()
return nil, ctx.Err()
}
}
@@ -364,6 +405,7 @@ func (m *streamMutateMux) openStream(out *filer_pb.SeaweedFiler_StreamMutateEntr
}
atomic.StoreInt32(&m.wfs.option.filerIndex, idx)
m.filerIdx = idx
m.cancel = cancel
m.grpcConn = grpcConn
*out = stream
@@ -428,6 +470,7 @@ func (m *streamMutateMux) teardownStream(gen uint64) {
m.mu.Unlock()
return
}
filerIdx := m.filerIdx
m.stream = nil
if m.stopSend != nil {
close(m.stopSend)
@@ -439,6 +482,13 @@ func (m *streamMutateMux) teardownStream(gen uint64) {
}
conn := m.grpcConn
m.grpcConn = nil
// Rotate while still holding mu: a new generation cannot open until
// teardown releases the lock, so the index advanced here always belongs
// to the stream that just died — never to a successor ensureStream may
// have opened in between.
if n := int32(len(m.wfs.option.FilerAddresses)); n > 0 {
atomic.CompareAndSwapInt32(&m.wfs.option.filerIndex, filerIdx, (filerIdx+1)%n)
}
m.mu.Unlock()
// Do NOT call failAllPending here — recvLoop is the sole owner of
weed/mount/weedfs_symlink.go
+3 -1
View File
@@ -91,7 +91,9 @@ func (wfs *WFS) Readlink(cancel <-chan struct{}, header *fuse.InHeader) (out []b
return
}
entry, _, status := wfs.maybeLoadEntry(entryFullPath)
ctx, cancelFunc := ctxFromCancel(cancel)
defer cancelFunc()
entry, _, status := wfs.maybeLoadEntry(ctx, entryFullPath)
if status != fuse.OK {
return nil, status
}
weed/mount/weedfs_xattr.go
+12 -4
View File
@@ -38,7 +38,9 @@ func (wfs *WFS) GetXAttr(cancel <-chan struct{}, header *fuse.InHeader, attr str
return 0, fuse.EINVAL
}
_, _, entry, status := wfs.maybeReadEntry(header.NodeId)
ctx, cancelFunc := ctxFromCancel(cancel)
defer cancelFunc()
_, _, entry, status := wfs.maybeReadEntry(ctx, header.NodeId)
if status != fuse.OK {
return 0, status
}
@@ -105,7 +107,9 @@ func (wfs *WFS) SetXAttr(cancel <-chan struct{}, input *fuse.SetXAttrIn, attr st
}
}
path, fh, entry, status := wfs.maybeReadEntry(input.NodeId)
ctx, cancelFunc := ctxFromCancel(cancel)
defer cancelFunc()
path, fh, entry, status := wfs.maybeReadEntry(ctx, input.NodeId)
if status != fuse.OK {
return status
}
@@ -160,7 +164,9 @@ func (wfs *WFS) ListXAttr(cancel <-chan struct{}, header *fuse.InHeader, dest []
return 0, fuse.Status(syscall.ENOTSUP)
}
_, _, entry, status := wfs.maybeReadEntry(header.NodeId)
ctx, cancelFunc := ctxFromCancel(cancel)
defer cancelFunc()
_, _, entry, status := wfs.maybeReadEntry(ctx, header.NodeId)
if status != fuse.OK {
return 0, status
}
@@ -197,7 +203,9 @@ func (wfs *WFS) RemoveXAttr(cancel <-chan struct{}, header *fuse.InHeader, attr
if len(attr) == 0 {
return fuse.EINVAL
}
path, fh, entry, status := wfs.maybeReadEntry(header.NodeId)
ctx, cancelFunc := ctxFromCancel(cancel)
defer cancelFunc()
path, fh, entry, status := wfs.maybeReadEntry(ctx, header.NodeId)
if status != fuse.OK {
return status
}
weed/mount/weedfs_xattr_cancel_test.go
+96
View File
@@ -0,0 +1,96 @@
package mount
import (
"context"
"testing"
"time"
"github.com/seaweedfs/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// stalledLookupFiler parks every LookupDirectoryEntry until the caller's
// context ends, the way a wedged filer holds an RPC forever.
type stalledLookupFiler struct {
filer_pb.UnimplementedSeaweedFilerServer
lookupStarted chan struct{}
}
func (s *stalledLookupFiler) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
select {
case <-s.lookupStarted:
default:
close(s.lookupStarted)
}
<-ctx.Done()
return nil, status.Error(codes.Canceled, ctx.Err().Error())
}
// A FUSE interrupt must unwind a metadata lookup stuck on the filer; without
// it the op waits on the RPC forever and the caller — Samba issuing
// listxattr — never sees an answer.
func TestListXAttrCanceledWhileFilerStalled(t *testing.T) {
wfs := newInvalidateTestWFS(t)
fake := &stalledLookupFiler{lookupStarted: make(chan struct{})}
startFakeFiler(t, wfs, fake)
wfs.inodeToPath.EnsurePath(util.FullPath("/dir"), true)
filePath := util.FullPath("/dir/f")
inode := wfs.inodeToPath.AllocateInode(filePath, time.Now().Unix())
wfs.inodeToPath.AddPath(inode, filePath)
cancel := make(chan struct{})
done := make(chan fuse.Status, 1)
go func() {
_, code := wfs.ListXAttr(cancel, &fuse.InHeader{NodeId: inode}, nil)
done <- code
}()
select {
case <-fake.lookupStarted:
case <-time.After(10 * time.Second):
t.Fatal("lookup never reached the filer")
}
close(cancel)
select {
case code := <-done:
if code != fuse.EINTR {
t.Fatalf("interrupted lookup returned %v, want EINTR", code)
}
case <-time.After(10 * time.Second):
t.Fatal("ListXAttr still blocked after the FUSE request was interrupted")
}
}
// Even without an interrupt the filer lookup has a deadline, so a hung RPC
// returns a filesystem error instead of parking the caller indefinitely.
func TestListXAttrTimesOutWhileFilerStalled(t *testing.T) {
wfs := newInvalidateTestWFS(t)
wfs.filerCallTimeout = 50 * time.Millisecond
fake := &stalledLookupFiler{lookupStarted: make(chan struct{})}
startFakeFiler(t, wfs, fake)
wfs.inodeToPath.EnsurePath(util.FullPath("/dir"), true)
filePath := util.FullPath("/dir/f")
inode := wfs.inodeToPath.AllocateInode(filePath, time.Now().Unix())
wfs.inodeToPath.AddPath(inode, filePath)
done := make(chan fuse.Status, 1)
go func() {
_, code := wfs.ListXAttr(nil, &fuse.InHeader{NodeId: inode}, nil)
done <- code
}()
select {
case code := <-done:
if code != fuse.EIO {
t.Fatalf("stalled filer lookup returned %v, want EIO", code)
}
case <-time.After(30 * time.Second):
t.Fatal("ListXAttr still blocked after the filer lookup deadline")
}
}
weed/mount/wfs_filer_client.go
+87 -4
View File
@@ -2,9 +2,13 @@ package mount
import (
"context"
"fmt"
"sync/atomic"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -14,6 +18,20 @@ import (
var _ = filer_pb.FilerClient(&WFS{})
func (wfs *WFS) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) (err error) {
return wfs.withFilerClient(context.Background(), streamingMode, wfs.filerRPCWait(), func(_ context.Context, client filer_pb.SeaweedFilerClient) error {
return fn(client)
})
}
// withFilerClient walks the filer addresses until one answers fn. callWait
// bounds one attempt; callWait <= 0 leaves the attempt to ctx alone, for
// calls whose duration the metadata bound cannot anticipate (a whole-file
// remote download). Streaming calls ignore callWait — their callbacks bound
// themselves (the rename fallback arms its own silence timer). The callback
// must run its RPCs under the supplied context: that is what lets a
// timed-out attempt be cancelled without disturbing the shared cached
// channel's other callers.
func (wfs *WFS) withFilerClient(ctx context.Context, streamingMode bool, callWait time.Duration, fn func(ctx context.Context, client filer_pb.SeaweedFilerClient) error) (err error) {
return util.Retry("filer grpc", func() error {
@@ -22,10 +40,7 @@ func (wfs *WFS) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFile
for x := 0; x < n; x++ {
filerGrpcAddress := wfs.option.FilerAddresses[i].ToGrpcAddress()
err = pb.WithGrpcClient(context.Background(), streamingMode, wfs.signature, func(grpcConnection *grpc.ClientConn) error {
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
return fn(client)
}, filerGrpcAddress, false, wfs.option.GrpcDialOption)
err = wfs.callFiler(ctx, streamingMode, callWait, filerGrpcAddress, fn)
if err != nil {
// glog.V(0).Infof("WithFilerClient %d %v: %v", x, filerGrpcAddress, err)
@@ -45,6 +60,74 @@ func (wfs *WFS) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFile
}
// filerRPCWait bounds one filer call when the callers' RPCs carry no deadline
// of their own. filerCallTimeout is only settable on the WFS for tests.
func (wfs *WFS) filerRPCWait() time.Duration {
if wfs.filerCallTimeout > 0 {
return wfs.filerCallTimeout
}
return filerRPCTimeout
}
// callFiler runs fn against one filer under a per-attempt context derived
// from ctx, so a filer whose transport silently stalls (stopped peer,
// black-holed connection) cannot block forever and keep the walk from
// reaching the remaining filer addresses. Cancelling the attempt aborts its
// RPCs on the shared cached channel without tearing that channel down.
//
// On timeout the call waits for the attempt's callback to unwind before
// returning: a slow attempt must not overlap the next filer's run of the
// same closure and clobber the caller's shared result variables. A callback
// that ignored the attempt context is forced out by dropping the channel.
func (wfs *WFS) callFiler(ctx context.Context, streamingMode bool, callWait time.Duration, filerGrpcAddress string, fn func(ctx context.Context, client filer_pb.SeaweedFilerClient) error) error {
if streamingMode {
return pb.WithGrpcClient(ctx, streamingMode, wfs.signature, func(grpcConnection *grpc.ClientConn) error {
return fn(ctx, filer_pb.NewSeaweedFilerClient(grpcConnection))
}, filerGrpcAddress, false, wfs.option.GrpcDialOption)
}
attemptCtx, cancel := context.WithCancel(ctx)
if callWait > 0 {
attemptCtx, cancel = context.WithTimeout(ctx, callWait)
}
defer cancel()
done := make(chan error, 1)
go func() {
done <- pb.WithGrpcClient(attemptCtx, streamingMode, wfs.signature, func(grpcConnection *grpc.ClientConn) error {
return fn(attemptCtx, filer_pb.NewSeaweedFilerClient(grpcConnection))
}, filerGrpcAddress, false, wfs.option.GrpcDialOption)
}()
select {
case err := <-done:
if err != nil && ctx.Err() == nil && attemptCtx.Err() == context.DeadlineExceeded {
// fn surfaced the attempt deadline itself; report it like the
// timer path so the failure reads as a filer timeout, not as a
// caller cancellation (which IsTransientError would not retry).
return status.Error(codes.Unavailable, fmt.Sprintf("filer %s: call timed out", filerGrpcAddress))
}
return err
case <-attemptCtx.Done():
// Give the callback one RPC's grace to unwind on its own; only if it
// ignored the attempt context does the cached channel get dropped —
// the collateral is borne by calls to the same unresponsive filer,
// which the walk is abandoning anyway.
drain := time.NewTimer(wfs.filerRPCWait())
defer drain.Stop()
select {
case <-done:
case <-drain.C:
pb.InvalidateGrpcConnection(filerGrpcAddress)
<-done
}
if ctx.Err() != nil {
return ctx.Err()
}
return status.Error(codes.Unavailable, fmt.Sprintf("filer %s: call timed out", filerGrpcAddress))
}
}
func (wfs *WFS) AdjustedUrl(location *filer_pb.Location) string {
if wfs.option.VolumeServerAccess == "publicUrl" {
return location.PublicUrl
weed/mount/wfs_filer_failover_test.go
+212
View File
@@ -0,0 +1,212 @@
package mount
import (
"context"
"errors"
"net"
"sync/atomic"
"testing"
"time"
"google.golang.org/grpc"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
// wedgedFiler accepts connections but never answers: LookupDirectoryEntry and
// StreamMutateEntry block until the transport dies. It stands in for a filer
// whose TCP connection black-holes instead of refusing.
type wedgedFiler struct {
filer_pb.UnimplementedSeaweedFilerServer
}
func (s *wedgedFiler) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
<-ctx.Done()
return nil, ctx.Err()
}
func (s *wedgedFiler) StreamMutateEntry(stream filer_pb.SeaweedFiler_StreamMutateEntryServer) error {
<-stream.Context().Done()
return stream.Context().Err()
}
func (s *wedgedFiler) StreamRenameEntry(req *filer_pb.StreamRenameEntryRequest, stream filer_pb.SeaweedFiler_StreamRenameEntryServer) error {
<-stream.Context().Done()
return stream.Context().Err()
}
// StreamRenameEntry ends cleanly with no events: enough for a failover test
// to confirm the walk reached this filer.
func (s *fakeFilerServer) StreamRenameEntry(req *filer_pb.StreamRenameEntryRequest, stream filer_pb.SeaweedFiler_StreamRenameEntryServer) error {
return nil
}
func listenFiler(t *testing.T, fake filer_pb.SeaweedFilerServer) pb.ServerAddress {
t.Helper()
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
t.Cleanup(func() { _ = listener.Close() })
server := grpc.NewServer()
filer_pb.RegisterSeaweedFilerServer(server, fake)
go server.Serve(listener)
t.Cleanup(server.Stop)
return pb.NewServerAddressWithGrpcPort("127.0.0.1:1", listener.Addr().(*net.TCPAddr).Port)
}
// A wedged filer must not hold the walk hostage: the call is bounded and the
// walk reaches the next address.
func TestWithFilerClientFailsOverPastWedgedFiler(t *testing.T) {
wfs := newInvalidateTestWFS(t)
wfs.filerCallTimeout = 200 * time.Millisecond
wedged := listenFiler(t, &wedgedFiler{})
live := listenFiler(t, &fakeFilerServer{lookupSize: 42, lookupLogTsNs: 1000})
wfs.option.FilerAddresses = []pb.ServerAddress{wedged, live}
atomic.StoreInt32(&wfs.option.filerIndex, 0)
var entry *filer_pb.Entry
err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
resp, err := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{
Directory: "/dir",
Name: "file",
})
if err != nil {
return err
}
entry = resp.Entry
return nil
})
if err != nil {
t.Fatalf("WithFilerClient: %v", err)
}
if entry == nil || entry.Attributes.FileSize != 42 {
t.Fatalf("entry = %+v, want lookup served by the live filer", entry)
}
if got := atomic.LoadInt32(&wfs.option.filerIndex); got != 1 {
t.Fatalf("filerIndex = %d, want 1 (the live filer)", got)
}
}
// A mutation stream whose peer never answers must surface as a transport
// error instead of hanging, and the sticky index must move off the dead filer
// so the next stream opens elsewhere.
func TestStreamMutateTimesOutAndRotatesFiler(t *testing.T) {
wfs := newInvalidateTestWFS(t)
wfs.filerCallTimeout = 200 * time.Millisecond
wedged := listenFiler(t, &wedgedFiler{})
wfs.option.FilerAddresses = []pb.ServerAddress{
wedged,
pb.NewServerAddressWithGrpcPort("127.0.0.1:1", 1),
}
atomic.StoreInt32(&wfs.option.filerIndex, 0)
mux := newStreamMutateMux(wfs)
defer mux.Close()
_, err := mux.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{
Directory: "/dir",
Entry: &filer_pb.Entry{Name: "file"},
})
if !errors.Is(err, ErrStreamTransport) {
t.Fatalf("err = %v, want ErrStreamTransport", err)
}
if got := atomic.LoadInt32(&wfs.option.filerIndex); got != 1 {
t.Fatalf("filerIndex = %d, want 1 after teardown of the wedged stream", got)
}
}
// A timed-out attempt must unwind before the walk runs the next filer's
// attempt: every attempt runs the same closure, so two in flight at once
// race on the caller's captured result variables.
func TestWithFilerClientDoesNotOverlapAttempts(t *testing.T) {
wfs := newInvalidateTestWFS(t)
wfs.filerCallTimeout = 200 * time.Millisecond
wedged := listenFiler(t, &wedgedFiler{})
live := listenFiler(t, &fakeFilerServer{lookupSize: 42})
wfs.option.FilerAddresses = []pb.ServerAddress{wedged, live}
atomic.StoreInt32(&wfs.option.filerIndex, 0)
var attempts, inside, overlap atomic.Int32
release := make(chan struct{})
done := make(chan error, 1)
go func() {
done <- wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
n := attempts.Add(1)
if !inside.CompareAndSwap(0, 1) {
overlap.Add(1)
}
defer inside.Store(0)
if n == 1 {
// Outlive the attempt deadline on purpose: the walk must
// wait for this callback instead of racing ahead.
<-release
return errors.New("first attempt released")
}
return nil
})
}()
deadline := time.Now().Add(10 * wfs.filerCallTimeout)
for attempts.Load() == 0 && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
if attempts.Load() == 0 {
t.Fatal("first attempt never started")
}
// Past the deadline plus its unwind grace, the blocked callback must
// still be the only attempt: no second run may start alongside it.
time.Sleep(3 * wfs.filerCallTimeout)
if got := attempts.Load(); got != 1 {
t.Fatalf("attempts = %d, want 1 while the timed-out attempt is still unwinding", got)
}
close(release)
if err := <-done; err != nil {
t.Fatalf("WithFilerClient: %v", err)
}
if got := overlap.Load(); got != 0 {
t.Fatalf("attempts overlapped %d times", got)
}
if got := attempts.Load(); got != 2 {
t.Fatalf("attempts = %d, want 2", got)
}
}
// The unary fallback for a streamed rename must bound silence too: a filer
// that accepts StreamRenameEntry then never answers cannot hold the walk.
func TestDoRenameFailsOverPastWedgedFiler(t *testing.T) {
wfs := newInvalidateTestWFS(t)
wfs.filerCallTimeout = 200 * time.Millisecond
wedged := listenFiler(t, &wedgedFiler{})
live := listenFiler(t, &fakeFilerServer{})
wfs.option.FilerAddresses = []pb.ServerAddress{wedged, live}
atomic.StoreInt32(&wfs.option.filerIndex, 0)
var newPathLock *cluster.LiveLock
start := time.Now()
err := wfs.doRename(context.Background(), &filer_pb.StreamRenameEntryRequest{
OldDirectory: "/a",
OldName: "f",
NewDirectory: "/b",
NewName: "f",
}, util.FullPath("/a/f"), util.FullPath("/b/f"), &newPathLock)
if err != nil {
t.Fatalf("doRename: %v", err)
}
if elapsed := time.Since(start); elapsed > 10*wfs.filerCallTimeout {
t.Fatalf("doRename took %v, wedged fallback stream was not bounded", elapsed)
}
if got := atomic.LoadInt32(&wfs.option.filerIndex); got != 1 {
t.Fatalf("filerIndex = %d, want 1 (the live filer)", got)
}
}
weed/mq/broker/broker_server.go
+4 -1
View File
@@ -125,7 +125,10 @@ func NewMessageBroker(option *MessageQueueBrokerOption, grpcDialOption grpc.Dial
)
glog.V(0).Info("Started idle partition cleanup task (check: 1m, timeout: 5m)")
existingNodes := cluster.ListExistingPeerUpdates(mqBroker.MasterClient.GetMaster(context.Background()), grpcDialOption, option.FilerGroup, cluster.FilerType)
existingNodes, listErr := cluster.ListExistingPeerUpdates(context.Background(), mqBroker.MasterClient.GetMaster(context.Background()), grpcDialOption, option.FilerGroup, cluster.FilerType)
if listErr != nil {
glog.Warningf("broker failed to list existing peers: %v", listErr)
}
for _, newNode := range existingNodes {
mqBroker.OnBrokerUpdate(newNode, time.Now())
}
weed/pb/filer.proto
+4
View File
@@ -714,6 +714,10 @@ message TraverseBfsMetadataRequest {
message TraverseBfsMetadataResponse {
string directory = 1;
Entry entry = 2;
// The sender's per-path commit timestamp (see meta_path_ts.go): the newest
// event time this filer committed for the path, 0 when unrecorded so
// subscribers fall back to the entry's mtime.
int64 entry_ts_ns = 3;
}
message LogEntry {
weed/pb/filer_pb/filer.pb.go
+17 -5
View File
@@ -4777,9 +4777,13 @@ func (x *TraverseBfsMetadataRequest) GetExcludedPrefixes() []string {
}
type TraverseBfsMetadataResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
Directory string `protobuf:"bytes,1,opt,name=directory,proto3" json:"directory,omitempty"`
Entry *Entry `protobuf:"bytes,2,opt,name=entry,proto3" json:"entry,omitempty"`
state protoimpl.MessageState `protogen:"open.v1"`
Directory string `protobuf:"bytes,1,opt,name=directory,proto3" json:"directory,omitempty"`
Entry *Entry `protobuf:"bytes,2,opt,name=entry,proto3" json:"entry,omitempty"`
// The sender's per-path commit timestamp (see meta_path_ts.go): the newest
// event time this filer committed for the path, 0 when unrecorded so
// subscribers fall back to the entry's mtime.
EntryTsNs int64 `protobuf:"varint,3,opt,name=entry_ts_ns,json=entryTsNs,proto3" json:"entry_ts_ns,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -4828,6 +4832,13 @@ func (x *TraverseBfsMetadataResponse) GetEntry() *Entry {
return nil
}
func (x *TraverseBfsMetadataResponse) GetEntryTsNs() int64 {
if x != nil {
return x.EntryTsNs
}
return 0
}
type LogEntry struct {
state protoimpl.MessageState `protogen:"open.v1"`
TsNs int64 `protobuf:"varint,1,opt,name=ts_ns,json=tsNs,proto3" json:"ts_ns,omitempty"`
@@ -7688,10 +7699,11 @@ const file_filer_proto_rawDesc = "" +
"\bfiler_id\x18\x03 \x01(\tR\afilerId\"g\n" +
"\x1aTraverseBfsMetadataRequest\x12\x1c\n" +
"\tdirectory\x18\x01 \x01(\tR\tdirectory\x12+\n" +
"\x11excluded_prefixes\x18\x02 \x03(\tR\x10excludedPrefixes\"b\n" +
"\x11excluded_prefixes\x18\x02 \x03(\tR\x10excludedPrefixes\"\x82\x01\n" +
"\x1bTraverseBfsMetadataResponse\x12\x1c\n" +
"\tdirectory\x18\x01 \x01(\tR\tdirectory\x12%\n" +
"\x05entry\x18\x02 \x01(\v2\x0f.filer_pb.EntryR\x05entry\"\x8b\x01\n" +
"\x05entry\x18\x02 \x01(\v2\x0f.filer_pb.EntryR\x05entry\x12\x1e\n" +
"\ventry_ts_ns\x18\x03 \x01(\x03R\tentryTsNs\"\x8b\x01\n" +
"\bLogEntry\x12\x13\n" +
"\x05ts_ns\x18\x01 \x01(\x03R\x04tsNs\x12,\n" +
"\x12partition_key_hash\x18\x02 \x01(\x05R\x10partitionKeyHash\x12\x12\n" +
weed/pb/filer_pb/filer_vtproto.pb.go
+27
View File
@@ -4379,6 +4379,11 @@ func (m *TraverseBfsMetadataResponse) MarshalToSizedBufferVT(dAtA []byte) (int,
i -= len(m.unknownFields)
copy(dAtA[i:], m.unknownFields)
}
if m.EntryTsNs != 0 {
i = protohelpers.EncodeVarint(dAtA, i, uint64(m.EntryTsNs))
i--
dAtA[i] = 0x18
}
if m.Entry != nil {
size, err := m.Entry.MarshalToSizedBufferVT(dAtA[:i])
if err != nil {
@@ -8267,6 +8272,9 @@ func (m *TraverseBfsMetadataResponse) SizeVT() (n int) {
l = m.Entry.SizeVT()
n += 1 + l + protohelpers.SizeOfVarint(uint64(l))
}
if m.EntryTsNs != 0 {
n += 1 + protohelpers.SizeOfVarint(uint64(m.EntryTsNs))
}
n += len(m.unknownFields)
return n
}
@@ -20865,6 +20873,25 @@ func (m *TraverseBfsMetadataResponse) UnmarshalVT(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field EntryTsNs", wireType)
}
m.EntryTsNs = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return protohelpers.ErrIntOverflow
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.EntryTsNs |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := protohelpers.Skip(dAtA[iNdEx:])
weed/pb/server_discovery.go
+19 -2
View File
@@ -24,8 +24,19 @@ func NewServiceDiscoveryFromMap(m map[string]ServerAddress) (sd *ServerDiscovery
// RefreshBySrvIfAvailable performs a DNS SRV lookup and updates list with the results
// of the lookup
func (sd *ServerDiscovery) RefreshBySrvIfAvailable() {
if newList := sd.LookupSrvInstances(); newList != nil {
sd.SetInstances(newList)
}
}
// LookupSrvInstances resolves the SRV record without touching the stored
// list. It returns nil when there is no SRV record, the lookup fails, or the
// result has no well-formed names. Running it without holding a lock keeps a
// slow resolver from blocking GetInstances callers; callers serialize the
// SetInstances that saves the result.
func (sd *ServerDiscovery) LookupSrvInstances() []ServerAddress {
if sd.srvRecord == nil {
return
return nil
}
newList, err := sd.srvRecord.LookUp()
if err != nil {
@@ -33,8 +44,14 @@ func (sd *ServerDiscovery) RefreshBySrvIfAvailable() {
}
if newList == nil || len(newList) == 0 {
glog.V(0).Infof("looked up SRV for %s, but found no well-formed names", *sd.srvRecord)
return
return nil
}
return newList
}
// SetInstances saves a resolved address list. Callers racing a refresh must
// serialize this with their GetInstances reads.
func (sd *ServerDiscovery) SetInstances(newList []ServerAddress) {
if !reflect.DeepEqual(sd.list, newList) {
sd.list = newList
}
weed/plugin/worker/worker.go
+9
View File
@@ -14,7 +14,10 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
@@ -204,6 +207,12 @@ func (w *Worker) runOnce(ctx context.Context, adminAddress string) error {
connCtx, cancelConn := context.WithCancel(ctx)
defer cancelConn()
if md := security.GrpcAdminPeerCredentials(
security.SigningKey(util.GetViper().GetString("jwt.filer_signing.key")),
util.GetViper().GetInt("jwt.filer_signing.expires_after_seconds")); len(md) > 0 {
connCtx = metadata.NewOutgoingContext(connCtx, md)
}
stream, err := client.WorkerStream(connCtx)
if err != nil {
return fmt.Errorf("open worker stream: %w", err)
weed/s3api/filer_multipart.go
+1
View File
@@ -110,6 +110,7 @@ func (s3a *S3ApiServer) createMultipartUpload(r *http.Request, input *s3.CreateM
if input.ContentType != nil {
entry.Attributes.Mime = *input.ContentType
}
applyPutObjectACL(r, entry)
// Prepare and apply encryption configuration within directory creation
// This ensures encryption resources are only allocated if directory creation succeeds
weed/s3api/filer_multipart_initiate_acl_test.go
+126
View File
@@ -0,0 +1,126 @@
package s3api
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/stretchr/testify/require"
)
func TestParseS3MetadataIgnoresInternalAclHeaders(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "http://s3/b/o?uploads", nil)
req.Header.Set(s3_constants.ExtAmzAclKey, `[{"Grantee":{"Type":"Group","URI":"http://acs.amazonaws.com/groups/global/AllUsers"},"Permission":"READ"}]`)
req.Header.Set(s3_constants.ExtAmzOwnerKey, "forged-owner")
req.Header.Set("X-Amz-Meta-Color", "red")
metadata, code := ParseS3Metadata(req, nil, false)
require.Equal(t, s3err.ErrNone, code)
require.NotContains(t, metadata, s3_constants.ExtAmzAclKey)
require.NotContains(t, metadata, s3_constants.ExtAmzOwnerKey)
require.Equal(t, "red", string(metadata["X-Amz-Meta-Color"]))
}
// Multipart initiation previously trusted the internal Seaweed-X-Amz-*
// headers, so a caller could forge ownership or a public ACL onto the
// completed object without s3:PutObjectAcl. Initiate must run the same ACL
// validation as PUT and store only the validated grants.
func TestNewMultipartUploadInitiateACL(t *testing.T) {
const bucket, object, writer, bucketOwner = "mpu-bucket", "dir/blob.bin", "mpu-writer", "bucket-owner"
forgedACL := `[{"Grantee":{"Type":"Group","URI":"http://acs.amazonaws.com/groups/global/AllUsers"},"Permission":"READ"}]`
setup := func(t *testing.T, writeAcp bool) (*S3ApiServer, *ambiguousPutFiler) {
volume := startFakeVolumeServer(t)
f := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
s3a := newPutTestServer(t, startFakeFiler(t, f))
s3a.iam = NewIdentityAccessManagementWithStore(s3a.option, nil, "memory")
t.Cleanup(s3a.iam.Shutdown)
s3a.iam.isAuthEnabled = true
account := &Account{Id: writer, DisplayName: writer}
identity := &Identity{Name: "mpu-acl-test", Account: account, IsStatic: true,
Actions: []Action{"Write:" + bucket + "/*"},
Credentials: []*Credential{{AccessKey: routingTestAccessKey, SecretKey: routingTestSecretKey}}}
if writeAcp {
identity.Actions = append(identity.Actions, Action("WriteAcp:"+bucket+"/*"))
}
s3a.iam.accessKeyIdent[routingTestAccessKey] = identity
s3a.iam.nameToIdentity[identity.Name] = identity
s3a.iam.accounts[writer] = account
s3a.iam.accounts[bucketOwner] = &Account{Id: bucketOwner, DisplayName: bucketOwner}
bucketEntry := &filer_pb.Entry{Name: bucket, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{},
Extended: map[string][]byte{s3_constants.ExtAmzOwnerKey: []byte(bucketOwner)}}
bucketEntry.Extended[s3_constants.ExtOwnershipKey] = []byte(s3_constants.OwnershipObjectWriter)
f.entries["/buckets/"+bucket] = bucketEntry
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
s3a.bucketConfigCache.Set(bucket, &BucketConfig{Name: bucket, Owner: bucketOwner, Ownership: s3_constants.OwnershipObjectWriter})
return s3a, f
}
initiate := func(t *testing.T, s3a *S3ApiServer, headers map[string]string) *httptest.ResponseRecorder {
req := httptest.NewRequest(http.MethodPost, "http://s3/"+bucket+"/"+object+"?uploads", nil)
req = mux.SetURLVars(req, map[string]string{"bucket": bucket, "object": object})
for k, v := range headers {
req.Header.Set(k, v)
}
signRoutingTestRequest(t, req, "", "s3")
rr := httptest.NewRecorder()
s3a.iam.Auth(s3a.NewMultipartUploadHandler, s3_constants.ACTION_WRITE)(rr, req)
return rr
}
uploadEntry := func(t *testing.T, f *ambiguousPutFiler) *filer_pb.Entry {
t.Helper()
f.mu.Lock()
defer f.mu.Unlock()
prefix := "/buckets/" + bucket + "/" + s3_constants.MultipartUploadsFolder + "/"
for key, entry := range f.entries {
if strings.HasPrefix(key, prefix) {
return entry
}
}
t.Fatalf("no upload entry recorded under %s", prefix)
return nil
}
t.Run("forged internal headers are ignored", func(t *testing.T) {
s3a, f := setup(t, false)
rr := initiate(t, s3a, map[string]string{
s3_constants.ExtAmzAclKey: forgedACL,
s3_constants.ExtAmzOwnerKey: "forged-owner",
})
require.Equal(t, http.StatusOK, rr.Code, rr.Body.String())
entry := uploadEntry(t, f)
require.Equal(t, writer, string(entry.Extended[s3_constants.ExtAmzOwnerKey]))
require.NotEqual(t, forgedACL, string(entry.Extended[s3_constants.ExtAmzAclKey]))
for _, grant := range GetAcpGrants(entry.Extended) {
require.Nil(t, grant.Grantee.URI)
}
})
t.Run("explicit acl requires write acp", func(t *testing.T) {
s3a, f := setup(t, false)
rr := initiate(t, s3a, map[string]string{s3_constants.AmzCannedAcl: "public-read"})
require.Equal(t, http.StatusForbidden, rr.Code, rr.Body.String())
require.Empty(t, f.entries["/buckets/"+bucket+"/"+s3_constants.MultipartUploadsFolder])
})
t.Run("explicit acl lands on upload entry", func(t *testing.T) {
s3a, f := setup(t, true)
rr := initiate(t, s3a, map[string]string{s3_constants.AmzCannedAcl: "public-read"})
require.Equal(t, http.StatusOK, rr.Code, rr.Body.String())
grants := GetAcpGrants(uploadEntry(t, f).Extended)
var publicRead bool
for _, grant := range grants {
if grant.Grantee.URI != nil && *grant.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && grant.Permission != nil && *grant.Permission == "READ" {
publicRead = true
}
}
require.True(t, publicRead, "public-read grants should persist on the upload entry: %s", string(uploadEntry(t, f).Extended[s3_constants.ExtAmzAclKey]))
})
}
weed/s3api/s3_metadata_util.go
-12
View File
@@ -94,18 +94,6 @@ func ParseS3Metadata(r *http.Request, existing map[string][]byte, isReplace bool
metadata[s3_constants.AmzServerSideEncryptionCustomerKeyMD5] = []byte(keyMD5)
}
// ACL owner
acpOwner := r.Header.Get(s3_constants.ExtAmzOwnerKey)
if len(acpOwner) > 0 {
metadata[s3_constants.ExtAmzOwnerKey] = []byte(acpOwner)
}
// ACL grants
acpGrants := r.Header.Get(s3_constants.ExtAmzAclKey)
if len(acpGrants) > 0 {
metadata[s3_constants.ExtAmzAclKey] = []byte(acpGrants)
}
return metadata, s3err.ErrNone
}
weed/s3api/s3api_object_handlers.go
+4 -2
View File
@@ -2430,8 +2430,10 @@ func (s3a *S3ApiServer) setResponseHeaders(w http.ResponseWriter, r *http.Reques
// Apply S3 passthrough headers from query parameters
// AWS S3 supports overriding response headers via query parameters like:
// ?response-cache-control=no-cache&response-content-type=application/json
// This allows presigned URLs to control how browsers handle the downloaded content
if r != nil {
// This allows presigned URLs to control how browsers handle the downloaded content.
// AWS requires the request to be signed; anonymous requests must not influence
// how the stored object is served.
if r != nil && getRequestAuthType(r) != authTypeAnonymous {
for queryParam, headerValue := range r.URL.Query() {
if normalizedHeader, ok := s3_constants.PassThroughHeaders[strings.ToLower(queryParam)]; ok && len(headerValue) > 0 && headerValue[0] != "" {
w.Header().Set(normalizedHeader, headerValue[0])
weed/s3api/s3api_object_handlers_conditional_read_test.go
+27
View File
@@ -238,3 +238,30 @@ func TestRemoteObjectETagFallback(t *testing.T) {
t.Errorf("local object ETag = %q", got)
}
}
func TestSetResponseHeadersIgnoresOverridesForAnonymous(t *testing.T) {
s3a := &S3ApiServer{}
entry := &filer_pb.Entry{
Name: "object",
Attributes: &filer_pb.FuseAttributes{FileSize: 8, Mtime: 100},
}
for name, url := range map[string]string{
"anonymous": "/bucket/object?response-content-type=text%2Fhtml&response-content-disposition=inline",
"signed": "/bucket/object?response-content-type=text%2Fhtml&response-content-disposition=inline",
"presigned": "/bucket/object?X-Amz-Credential=k&response-content-type=text%2Fhtml",
} {
r := httptest.NewRequest(http.MethodGet, url, nil)
if name == "signed" {
r.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=k")
}
w := httptest.NewRecorder()
s3a.setResponseHeaders(w, r, entry, 8)
if name == "anonymous" {
if got := w.Header().Get("Content-Disposition"); got != "" {
t.Errorf("anonymous request must not override response headers, got Content-Disposition %q", got)
}
} else if got := w.Header().Get("Content-Type"); got != "text/html" {
t.Errorf("%s request should apply response overrides, got Content-Type %q", name, got)
}
}
}
weed/s3api/s3api_object_handlers_multipart.go
+9
View File
@@ -97,6 +97,15 @@ func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http
}
}
// An explicit ACL on initiate applies to the completed object and requires
// s3:PutObjectAcl, same as PUT. The validated grants ride on the upload
// entry so CompleteMultipartUpload cannot be handed forged internal headers.
r, aclCode := s3a.preparePutObjectACL(r, bucket)
if aclCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, aclCode)
return
}
createMultipartUploadInput := &s3.CreateMultipartUploadInput{
Bucket: aws.String(bucket),
Key: objectKey(aws.String(object)),
weed/s3api/s3api_server_grpc.go
+1 -42
View File
@@ -2,7 +2,6 @@ package s3api
import (
"context"
"net"
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -10,7 +9,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/security"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/peer"
"google.golang.org/grpc/status"
)
@@ -56,46 +54,7 @@ func (s3a *S3ApiServer) checkAdminAuth(ctx context.Context) error {
}
func checkLocalPeer(ctx context.Context) error {
if ctx == nil {
return status.Error(codes.Unauthenticated, "admin gRPC calls require jwt.filer_signing.key or a local client")
}
pr, ok := peer.FromContext(ctx)
if !ok {
return status.Error(codes.Unauthenticated, "admin gRPC calls require jwt.filer_signing.key or a local client")
}
if _, isUnix := pr.Addr.(*net.UnixAddr); isUnix {
return nil
}
var ip net.IP
if tcpAddr, ok := pr.Addr.(*net.TCPAddr); ok {
ip = tcpAddr.IP
} else if h, _, err := net.SplitHostPort(pr.Addr.String()); err == nil {
ip = net.ParseIP(h)
} else {
ip = net.ParseIP(pr.Addr.String())
}
if ip != nil && (ip.IsLoopback() || isLocalAddress(ip)) {
return nil
}
glog.V(1).Infof("rejected unauthenticated admin gRPC call from %s: no jwt.filer_signing.key configured", pr.Addr)
return status.Error(codes.Unauthenticated, "admin gRPC calls require jwt.filer_signing.key or a local client")
}
// isLocalAddress enumerates interfaces per call: only reachable for
// non-loopback TCP peers on the no-key admin path, which is low-volume —
// and a cached set would keep trusting an address after it is removed from
// the host and reassigned.
func isLocalAddress(ip net.IP) bool {
addrs, err := net.InterfaceAddrs()
if err != nil {
return false
}
for _, a := range addrs {
if ipNet, ok := a.(*net.IPNet); ok && ipNet.IP.Equal(ip) {
return true
}
}
return false
return security.CheckLocalGrpcPeer(ctx)
}
func (s3a *S3ApiServer) PutIdentity(ctx context.Context, req *iam_pb.PutIdentityRequest) (*iam_pb.PutIdentityResponse, error) {
weed/s3api/s3tables/handler_delete_decouple_test.go
+48
View File
@@ -3,6 +3,7 @@ package s3tables
import (
"context"
"encoding/json"
"path"
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
@@ -120,3 +121,50 @@ func TestDeleteTableRefusesLiveSiblingDataPath(t *testing.T) {
assert.NotNil(t, fs.Get(GetTablePath(renameTestBucket, "ns", "t"), "data"),
"the targeted table's data must survive")
}
// A location inside another table's directory carries no catalog marker of its
// own — the marker lives on the name path. Dropping a table whose
// MetadataLocation resolves into nsa/vtab/data must be refused by the ancestor
// check, or the victim's marker-less data subdirectory is purged wholesale.
func TestDeleteTableRefusesNestedSiblingDataPath(t *testing.T) {
fs, m := startRenameManager(t)
nsaMeta, _ := json.Marshal(namespaceMetadata{Namespace: []string{"nsa"}, OwnerAccountID: DefaultAccountID})
fs.Put(GetTableBucketPath(renameTestBucket), "nsa", map[string][]byte{ExtendedKeyMetadata: nsaMeta})
victimMeta, _ := json.Marshal(tableMetadataInternal{
Name: "vtab",
Namespace: "nsa",
Format: "ICEBERG",
OwnerAccountID: DefaultAccountID,
MetadataLocation: "s3://" + renameTestBucket + "/nsa/vtab/metadata/v1.metadata.json",
})
victimTablePath := GetTablePath(renameTestBucket, "nsa", "vtab")
fs.Put(GetNamespacePath(renameTestBucket, "nsa"), "vtab", map[string][]byte{
ExtendedKeyMetadata: victimMeta,
ExtendedKeyEntryType: []byte(EntryTypeTable),
})
fs.Put(victimTablePath, "data", nil)
fs.Put(path.Join(victimTablePath, "data"), "important-0001.parquet", nil)
evilMeta, _ := json.Marshal(tableMetadataInternal{
Name: "evil",
Namespace: "attackerns2",
Format: "ICEBERG",
OwnerAccountID: DefaultAccountID,
MetadataLocation: "s3://" + renameTestBucket + "/nsa/vtab/data/metadata/v1.metadata.json",
})
nsMeta, _ := json.Marshal(namespaceMetadata{Namespace: []string{"attackerns2"}, OwnerAccountID: DefaultAccountID})
fs.Put(GetTableBucketPath(renameTestBucket), "attackerns2", map[string][]byte{ExtendedKeyMetadata: nsMeta})
fs.Put(GetNamespacePath(renameTestBucket, "attackerns2"), "evil", map[string][]byte{
ExtendedKeyMetadata: evilMeta,
ExtendedKeyEntryType: []byte(EntryTypeTable),
})
require.Error(t, runDeleteTable(t, m, fs, "attackerns2", "evil"))
assert.NotNil(t, fs.Get(path.Join(victimTablePath, "data"), "important-0001.parquet"),
"data inside another table's directory must survive")
assert.NotNil(t, fs.Get(GetNamespacePath(renameTestBucket, "nsa"), "vtab"),
"the victim table's catalog entry must survive")
}
weed/s3api/s3tables/handler_table.go
+24 -3
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"io"
"net/http"
"path"
"strconv"
"strings"
"time"
@@ -100,7 +101,7 @@ func (h *S3TablesHandler) handleCreateTable(w http.ResponseWriter, r *http.Reque
return err
}
if err := ValidateMetadataLocation(req.MetadataLocation, bucketName); err != nil {
if err := ValidateMetadataLocation(req.MetadataLocation, bucketName, namespaceName); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
@@ -233,7 +234,7 @@ func (h *S3TablesHandler) handleRegisterTable(w http.ResponseWriter, r *http.Req
}
bucketName, namespaceName, tableName := target.bucketName, target.namespaceName, target.tableName
if err := ValidateMetadataLocation(req.MetadataLocation, bucketName); err != nil {
if err := ValidateMetadataLocation(req.MetadataLocation, bucketName, namespaceName); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
@@ -996,6 +997,26 @@ func (h *S3TablesHandler) handleDeleteTable(w http.ResponseWriter, r *http.Reque
// still a live catalog entry. Only this table's authorization was
// checked; a decoupled location has had its catalog attributes
// stripped, so a surviving marker means the path belongs elsewhere.
// The marker lives on the entry's name path, so ancestors of the
// data path are checked as well: a location inside another table's
// directory resolves above any marker the data path itself would
// carry.
for dir := path.Dir(dataPath); dir != GetTableBucketPath(bucketName) && strings.HasPrefix(dir, GetTableBucketPath(bucketName)+"/"); dir = path.Dir(dir) {
ancestor, lookupErr := h.lookupEntry(r.Context(), client, dir)
if lookupErr != nil {
if errors.Is(lookupErr, filer_pb.ErrNotFound) {
continue
}
return fmt.Errorf("refusing to delete table %s: cannot read data path %q: %w", tableName, dataPath, lookupErr)
}
if _, ok := ancestor.Extended[ExtendedKeyEntryType]; ok {
return fmt.Errorf("refusing to delete table %s: data path %q is inside another catalog entry", tableName, dataPath)
}
var ownerMeta tableMetadataInternal
if data := ancestor.Extended[ExtendedKeyMetadata]; json.Unmarshal(data, &ownerMeta) == nil && ownerMeta.Name != "" {
return fmt.Errorf("refusing to delete table %s: data path %q is inside another catalog entry", tableName, dataPath)
}
}
switch _, markerErr := h.getExtendedAttribute(r.Context(), client, dataPath, ExtendedKeyMetadata); {
case markerErr == nil:
return fmt.Errorf("refusing to delete table %s: data path %q is another catalog entry", tableName, dataPath)
@@ -1494,7 +1515,7 @@ func (h *S3TablesHandler) handleUpdateTable(w http.ResponseWriter, r *http.Reque
return err
}
if err := ValidateMetadataLocation(req.MetadataLocation, bucketName); err != nil {
if err := ValidateMetadataLocation(req.MetadataLocation, bucketName, namespaceName); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
weed/s3api/s3tables/handler_view.go
+2 -2
View File
@@ -121,7 +121,7 @@ func (h *S3TablesHandler) handleCreateView(w http.ResponseWriter, r *http.Reques
return err
}
if err := ValidateMetadataLocation(req.MetadataLocation, bucketName); err != nil {
if err := ValidateMetadataLocation(req.MetadataLocation, bucketName, namespaceName); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
@@ -410,7 +410,7 @@ func (h *S3TablesHandler) handleUpdateView(w http.ResponseWriter, r *http.Reques
return err
}
if err := ValidateMetadataLocation(req.MetadataLocation, bucketName); err != nil {
if err := ValidateMetadataLocation(req.MetadataLocation, bucketName, namespaceName); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
weed/s3api/s3tables/utils.go
+10 -5
View File
@@ -127,11 +127,13 @@ func TableDataDirFromMetadataLocation(metadataLocation string) string {
}
// ValidateMetadataLocation checks that an s3:// metadata location stays within
// the authorized table bucket and rejects traversal segments that path.Join
// would collapse to escape the bucket directory. Empty locations are allowed
// (the catalog derives one). A non-empty location must include a table path so
// its metadata directory is table-specific, not shared at the bucket level.
func ValidateMetadataLocation(metadataLocation, bucketName string) error {
// the authorized table bucket and inside the caller's own namespace subtree,
// the boundary per-tenant policies are delegated on. Without the namespace
// check a caller could register a location inside another tenant's data
// directory and DeleteTable would purge it. Empty locations are allowed (the
// catalog derives one). A non-empty location must include a table path so its
// metadata directory is table-specific, not shared at the bucket level.
func ValidateMetadataLocation(metadataLocation, bucketName, namespace string) error {
if metadataLocation == "" {
return nil
}
@@ -155,6 +157,9 @@ func ValidateMetadataLocation(metadataLocation, bucketName string) error {
if !hasSegment {
return fmt.Errorf("metadata location must include a table path")
}
if !strings.HasPrefix(tablePath, namespace+"/") {
return fmt.Errorf("metadata location must be within namespace %s", namespace)
}
return nil
}
weed/s3api/s3tables/utils_location_test.go
+25 -18
View File
@@ -4,29 +4,36 @@ import "testing"
func TestValidateMetadataLocation(t *testing.T) {
tests := []struct {
name string
loc string
bucket string
wantErr bool
name string
loc string
bucket string
namespace string
wantErr bool
}{
{"empty allowed", "", "bkt", false},
{"same bucket", "s3://bkt/ns/t/metadata/v1.metadata.json", "bkt", false},
{"cross bucket rejected", "s3://other/ns/t/metadata/v1.metadata.json", "bkt", true},
{"bucket only rejected", "s3://bkt", "bkt", true},
{"bucket with trailing slash rejected", "s3://bkt/", "bkt", true},
{"slash-only path rejected", "s3://bkt///", "bkt", true},
{"dotdot in path rejected", "s3://bkt/../victim/metadata/v1.metadata.json", "bkt", true},
{"dotdot mid path rejected", "s3://bkt/ns/../../victim/metadata/v1.metadata.json", "bkt", true},
{"dot segment rejected", "s3://bkt/ns/./t/metadata/v1.metadata.json", "bkt", true},
{"backslash segment rejected", "s3://bkt/ns/\\t/metadata/v1.metadata.json", "bkt", true},
{"non-s3 scheme rejected", "file:///bkt/ns/t", "bkt", true},
{"empty location string", "s3://", "bkt", true},
{"empty allowed", "", "bkt", "ns", false},
{"own namespace", "s3://bkt/ns/t/metadata/v1.metadata.json", "bkt", "ns", false},
{"own namespace arbitrary layout", "s3://bkt/ns/warehouse/t/metadata/v1.metadata.json", "bkt", "ns", false},
{"cross bucket rejected", "s3://other/ns/t/metadata/v1.metadata.json", "bkt", "ns", true},
{"cross namespace rejected", "s3://bkt/nsa/t/metadata/v1.metadata.json", "bkt", "ns", true},
{"cross namespace data dir rejected", "s3://bkt/nsa/vtab/data/metadata/v1.metadata.json", "bkt", "ns", true},
{"namespace prefix rejected", "s3://bkt/nsa/vtab/metadata/v1.metadata.json", "bkt", "ns", true},
{"outside namespace tree rejected", "s3://bkt/shared/t/metadata/v1.metadata.json", "bkt", "ns", true},
{"namespace dir without table path rejected", "s3://bkt/ns", "bkt", "ns", true},
{"bucket only rejected", "s3://bkt", "bkt", "ns", true},
{"bucket with trailing slash rejected", "s3://bkt/", "bkt", "ns", true},
{"slash-only path rejected", "s3://bkt///", "bkt", "ns", true},
{"dotdot in path rejected", "s3://bkt/../victim/metadata/v1.metadata.json", "bkt", "ns", true},
{"dotdot mid path rejected", "s3://bkt/ns/../../victim/metadata/v1.metadata.json", "bkt", "ns", true},
{"dot segment rejected", "s3://bkt/ns/./t/metadata/v1.metadata.json", "bkt", "ns", true},
{"backslash segment rejected", "s3://bkt/ns/\\t/metadata/v1.metadata.json", "bkt", "ns", true},
{"non-s3 scheme rejected", "file:///bkt/ns/t", "bkt", "ns", true},
{"empty location string", "s3://", "bkt", "ns", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := ValidateMetadataLocation(tt.loc, tt.bucket)
err := ValidateMetadataLocation(tt.loc, tt.bucket, tt.namespace)
if (err != nil) != tt.wantErr {
t.Fatalf("ValidateMetadataLocation(%q, %q) err = %v, wantErr = %v", tt.loc, tt.bucket, err, tt.wantErr)
t.Fatalf("ValidateMetadataLocation(%q, %q, %q) err = %v, wantErr = %v", tt.loc, tt.bucket, tt.namespace, err, tt.wantErr)
}
})
}
weed/security/grpc_peer.go
+113
View File
@@ -0,0 +1,113 @@
package security
import (
"context"
"net"
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/peer"
"google.golang.org/grpc/status"
)
// CheckGrpcAdminPeer authenticates the peer of an admin-plane gRPC handler:
// a client certificate verified by the server's TLS config, or a Bearer token
// signed by signingKey (jwt.filer_signing.key). With no key configured remote
// callers cannot be told apart, so only local clients (unix socket, loopback,
// or an address assigned to this host) are allowed.
func CheckGrpcAdminPeer(ctx context.Context, signingKey SigningKey) error {
if isMtlsVerifiedPeer(ctx) {
return nil
}
if len(signingKey) == 0 {
return CheckLocalGrpcPeer(ctx)
}
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return status.Error(codes.Unauthenticated, "missing metadata")
}
authHeaders := md.Get("authorization")
if len(authHeaders) == 0 {
return status.Error(codes.Unauthenticated, "missing authorization metadata")
}
raw := strings.TrimSpace(authHeaders[0])
parts := strings.Fields(raw)
if len(parts) != 2 || !strings.EqualFold(parts[0], "Bearer") || parts[1] == "" {
return status.Error(codes.Unauthenticated, "authorization header must use Bearer scheme")
}
parsed, err := DecodeJwt(signingKey, EncodedJwt(parts[1]), &SeaweedFilerAdminClaims{})
if err != nil || parsed == nil || !parsed.Valid {
return status.Error(codes.Unauthenticated, "invalid admin token")
}
if claims, ok := parsed.Claims.(*SeaweedFilerAdminClaims); !ok || claims.SessionId != "" {
return status.Error(codes.Unauthenticated, "invalid admin token")
}
return nil
}
// GrpcAdminPeerCredentials returns outgoing metadata carrying a Bearer token
// for admin-plane gRPC handlers, or nil when no signing key is configured.
func GrpcAdminPeerCredentials(signingKey SigningKey, expiresAfterSec int) metadata.MD {
token := GenJwtForFilerAdmin(signingKey, expiresAfterSec)
if token == "" {
return nil
}
return metadata.Pairs("authorization", "Bearer "+string(token))
}
func isMtlsVerifiedPeer(ctx context.Context) bool {
pr, ok := peer.FromContext(ctx)
if !ok {
return false
}
tlsInfo, ok := pr.AuthInfo.(credentials.TLSInfo)
return ok && len(tlsInfo.State.VerifiedChains) > 0
}
// CheckLocalGrpcPeer admits only clients on this host: unix sockets, loopback,
// or an address assigned to a local interface.
func CheckLocalGrpcPeer(ctx context.Context) error {
if ctx == nil {
return status.Error(codes.Unauthenticated, "admin gRPC calls require jwt.filer_signing.key or a local client")
}
pr, ok := peer.FromContext(ctx)
if !ok {
return status.Error(codes.Unauthenticated, "admin gRPC calls require jwt.filer_signing.key or a local client")
}
if _, isUnix := pr.Addr.(*net.UnixAddr); isUnix {
return nil
}
var ip net.IP
if tcpAddr, ok := pr.Addr.(*net.TCPAddr); ok {
ip = tcpAddr.IP
} else if h, _, err := net.SplitHostPort(pr.Addr.String()); err == nil {
ip = net.ParseIP(h)
} else {
ip = net.ParseIP(pr.Addr.String())
}
if ip != nil && (ip.IsLoopback() || isLocalAddress(ip)) {
return nil
}
glog.V(1).Infof("rejected unauthenticated admin gRPC call from %s: no jwt.filer_signing.key configured", pr.Addr)
return status.Error(codes.Unauthenticated, "admin gRPC calls require jwt.filer_signing.key or a local client")
}
// isLocalAddress enumerates interfaces per call: only reachable for
// non-loopback TCP peers on the no-key admin path, which is low-volume —
// and a cached set would keep trusting an address after it is removed from
// the host and reassigned.
func isLocalAddress(ip net.IP) bool {
addrs, err := net.InterfaceAddrs()
if err != nil {
return false
}
for _, a := range addrs {
if ipNet, ok := a.(*net.IPNet); ok && ipNet.IP.Equal(ip) {
return true
}
}
return false
}
weed/security/grpc_peer_test.go
+68
View File
@@ -0,0 +1,68 @@
package security
import (
"context"
"crypto/tls"
"crypto/x509"
"net"
"testing"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/peer"
"google.golang.org/grpc/status"
)
func peerCtx(addr net.Addr) context.Context {
return peer.NewContext(context.Background(), &peer.Peer{Addr: addr})
}
func bearerCtx(addr net.Addr, signingKey SigningKey) context.Context {
ctx := peerCtx(addr)
token := GenJwtForFilerAdmin(signingKey, 0)
return metadata.NewIncomingContext(ctx, metadata.Pairs("authorization", "Bearer "+string(token)))
}
func TestCheckGrpcAdminPeer(t *testing.T) {
key := SigningKey("test-signing-key-0123456789abcdef")
loopback := &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 5000}
remote := &net.TCPAddr{IP: net.ParseIP("203.0.113.9"), Port: 5000}
for name, tc := range map[string]struct {
ctx context.Context
signingKey SigningKey
wantCode codes.Code
}{
"no peer info, no key": {context.Background(), nil, codes.Unauthenticated},
"loopback, no key": {peerCtx(loopback), nil, codes.OK},
"remote, no key": {peerCtx(remote), nil, codes.Unauthenticated},
"remote + valid token": {bearerCtx(remote, key), key, codes.OK},
"remote + no metadata, key set": {peerCtx(remote), key, codes.Unauthenticated},
"remote + bad token": {metadata.NewIncomingContext(peerCtx(remote), metadata.Pairs("authorization", "Bearer wrong")), key, codes.Unauthenticated},
"unix socket, no key": {peerCtx(&net.UnixAddr{Name: "/tmp/s.sock"}), nil, codes.OK},
"mtls verified peer": {peer.NewContext(context.Background(), &peer.Peer{
Addr: remote,
AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{{}}}},
}), nil, codes.OK},
} {
err := CheckGrpcAdminPeer(tc.ctx, tc.signingKey)
got := codes.OK
if err != nil {
got = status.Code(err)
}
if got != tc.wantCode {
t.Errorf("%s: got %v want %v (err=%v)", name, got, tc.wantCode, err)
}
}
}
func TestGrpcAdminPeerCredentials(t *testing.T) {
if md := GrpcAdminPeerCredentials(nil, 0); len(md) != 0 {
t.Errorf("expected nil metadata without signing key, got %v", md)
}
md := GrpcAdminPeerCredentials(SigningKey("key"), 60)
if len(md.Get("authorization")) != 1 {
t.Fatalf("expected one authorization entry, got %v", md)
}
}
weed/server/filer_grpc_server_traverse_meta.go
+5
View File
@@ -42,9 +42,14 @@ func (fs *FilerServer) TraverseBfsMetadata(req *filer_pb.TraverseBfsMetadataRequ
// the filer on large trees (e.g. during a peer's first-time bootstrap).
sendEntry := func(entry *filer.Entry) error {
parent, _ := entry.FullPath.DirAndName()
entryTsNs, err := fs.filer.CommitTsNs(ctx, entry.FullPath, entry.Mtime.UnixNano())
if err != nil {
return fmt.Errorf("commit timestamp of %s: %w", entry.FullPath, err)
}
if err := stream.Send(&filer_pb.TraverseBfsMetadataResponse{
Directory: parent,
Entry: entry.ToProtoEntry(),
EntryTsNs: entryTsNs,
}); err != nil {
return fmt.Errorf("send traverse bfs metadata response: %w", err)
}
weed/server/filer_grpc_server_traverse_meta_test.go
+20 -3
View File
@@ -4,6 +4,7 @@ import (
"context"
"testing"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/stretchr/testify/assert"
@@ -37,8 +38,9 @@ func TestPtrie(t *testing.T) {
type captureTraverseStream struct {
grpc.ServerStream
ctx context.Context
visited []string
ctx context.Context
visited []string
entryTsNs map[string]int64
}
func (s *captureTraverseStream) Context() context.Context { return s.ctx }
@@ -48,7 +50,12 @@ func (s *captureTraverseStream) Send(resp *filer_pb.TraverseBfsMetadataResponse)
if dir != "/" {
dir += "/"
}
s.visited = append(s.visited, dir+resp.Entry.Name)
path := dir + resp.Entry.Name
s.visited = append(s.visited, path)
if s.entryTsNs == nil {
s.entryTsNs = make(map[string]int64)
}
s.entryTsNs[path] = resp.EntryTsNs
return nil
}
@@ -89,6 +96,16 @@ func TestTraverseBfsMetadata(t *testing.T) {
}
seen[path] = true
}
// Every response carries a commit timestamp for ordering; with no
// recorded commit it falls back to the entry's mtime.
for path, tsNs := range stream.entryTsNs {
if path == "/" {
assert.Equal(t, filer.Root.Attr.Mtime.UnixNano(), tsNs)
continue
}
assert.Equalf(t, store.entries[path].Attr.Mtime.UnixNano(), tsNs, "entry_ts_ns for %s", path)
}
}
var _ filer_pb.SeaweedFiler_TraverseBfsMetadataServer = (*captureTraverseStream)(nil)
weed/server/filer_server.go
+151 -2
View File
@@ -91,6 +91,10 @@ type FilerOption struct {
TusSessionExpiry time.Duration
S3ConfigFile string // optional path to static S3 identity config file
CredentialManager *credential.CredentialManager
// AnnounceCh, when set, must be closed before the filer registers on the
// master; it joins the filer list only once the gRPC port is serving and
// the metadata replay has caught up with the peers found at startup.
AnnounceCh chan struct{}
// AllowUntrustedRemoteEndpoints lets a read of a remote-only entry dial a
// mounted endpoint that resolves to a loopback / private / metadata host.
AllowUntrustedRemoteEndpoints bool
@@ -155,6 +159,16 @@ type FilerServer struct {
ringPeerIPs atomic.Pointer[ringPeerIPs]
ringResolveGroup singleflight.Group
// masterCtx drives KeepConnectedToMaster; cancelling it ends the stream
// so the master drops this filer from its list before gRPC stops.
masterCtx context.Context
masterCancel context.CancelFunc
// startupPeers are the filers discovered during NewFilerServer, before
// this filer announced itself; AnnounceOnceCaughtUp waits for their
// replayed backlogs before the filer joins the master's list.
startupPeers []pb.ServerAddress
// entryLockTable serializes mutations to the same entry path on this filer.
// CreateEntry takes it today; UpdateEntry and DeleteEntry are intended to take
// it too as their callers route a key's writes to this node, making it the
@@ -254,6 +268,9 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
maxFilenameLength := v.GetUint32("filer.options.max_file_name_length")
glog.V(0).Infof("max_file_name_length %d", maxFilenameLength)
fs.filer = filer.NewFiler(*option.Masters, fs.grpcDialOption, option.Host, option.FilerGroup, option.Collection, option.DefaultReplication, option.DataCenter, maxFilenameLength, nil)
// Peer metadata replay takes the same per-path locks, so a replayed change
// cannot split a local mutation's read-modify-write.
fs.entryLockTable = fs.filer.EntryLockTable
fs.filer.Cipher = option.Cipher
fs.filer.DefaultDiskType = option.DiskType
fs.filer.BuildGuardedRemoteClient = BuildGuardedRemoteStorageClient
@@ -272,7 +289,11 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
fs.checkWithMaster()
go stats.LoopPushingMetric("filer", string(fs.option.Host), fs.metricsAddress, fs.metricsIntervalSec)
go fs.filer.MasterClient.KeepConnectedToMaster(context.Background())
if option.AnnounceCh != nil {
fs.filer.MasterClient.SetAnnounceCh(option.AnnounceCh)
}
fs.masterCtx, fs.masterCancel = context.WithCancel(context.Background())
go fs.filer.MasterClient.KeepConnectedToMaster(fs.masterCtx)
fs.option.recursiveDelete = v.GetBool("filer.options.recursive_delete")
v.SetDefault("filer.options.buckets_folder", "/buckets")
@@ -321,7 +342,20 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
readonlyMux.HandleFunc("/", fs.filerGuard.WhiteList(requestIDMiddleware(fs.readonlyFilerHandler)))
}
existingNodes := fs.filer.ListExistingPeerUpdates(context.Background())
// A failed peer scan is not an empty cluster: a fresh filer that skipped
// bootstrap here would serve without the existing files, so retry until a
// master answers. This runs before the announce gate opens, so the filer
// is not yet advertised.
var existingNodes []*master_pb.ClusterNodeUpdate
for {
var listErr error
existingNodes, listErr = fs.filer.ListExistingPeerUpdates(context.Background())
if listErr == nil {
break
}
glog.Warningf("%s cannot list existing peers: %v; retrying", option.Host, listErr)
time.Sleep(2 * time.Second)
}
startFromTime := time.Now().Add(-filer.LogFlushInterval)
if isFresh {
glog.V(0).Infof("%s bootstrap from peers %+v", option.Host, existingNodes)
@@ -335,6 +369,15 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
}
fs.filer.AggregateFromPeers(option.Host, existingNodes, startFromTime)
// A peer replaying its backlog applies the old owner's events over this
// filer's fresh writes if it takes lock-ring ownership before the replay
// finishes, so the announce gate waits for these peers to catch up.
for _, node := range existingNodes {
if addr := pb.ServerAddress(node.Address); !addr.Equals(option.Host) {
fs.startupPeers = append(fs.startupPeers, addr)
}
}
fs.filer.LoadFilerConf()
fs.filer.LoadRemoteStorageConfAndMapping()
@@ -390,6 +433,12 @@ func (fs *FilerServer) Shutdown() {
fs.remoteCacheEvictCancel()
}
fs.filer.Shutdown()
// LeaveLockRing already ended the master stream in the normal path; this
// covers callers that skip it, and runs after the filer's own shutdown so
// the final metadata-log flush can still reach the master.
if fs.masterCancel != nil {
fs.masterCancel()
}
}
// priorOwnerWindowSkew covers peers and gateways that start their prior-owner
@@ -417,6 +466,106 @@ func (fs *FilerServer) leaveLockRingWithin(removalTimeout, budget time.Duration)
case <-time.After(budget):
glog.Warningf("LockRing: %s did not finish leaving within %v, shutting down anyway", fs.option.Host, budget)
}
// End the master stream so the master drops this filer from the cluster
// list before gRPC stops accepting connections.
if fs.masterCancel != nil {
fs.masterCancel()
}
fs.waitForClusterRemoval()
}
// announceCatchUpCap bounds how long startup holds the master registration
// while the metadata replay catches up with the peers found at startup. Past
// it the filer announces anyway, as it did when the gate opened on serve.
const announceCatchUpCap = time.Minute
// AnnounceOnceCaughtUp closes announce once every peer found at startup has
// replayed its backlog. Each peer's stream advances a watermark per event or
// idle heartbeat, so a watermark past waitStart means nothing older can still
// land on top of this filer's fresh writes. Called once gRPC is serving, so
// routed traffic is answerable the moment the filer is listed.
func (fs *FilerServer) AnnounceOnceCaughtUp(announce chan struct{}, waitStart time.Time) {
defer close(announce)
if len(fs.startupPeers) == 0 || fs.filer.MetaAggregator == nil {
return
}
deadline := time.Now().Add(announceCatchUpCap)
for {
// Take the channel before reading the watermarks so a rise in
// between still wakes this wait (see DeliveryWatermarkAdvancedChan).
advanced := fs.filer.MetaAggregator.DeliveryWatermarkAdvancedChan()
if fs.filer.MetaAggregator.PeersCaughtUpTo(fs.startupPeers, waitStart.UnixNano()) {
glog.V(0).Infof("%s caught up with %d startup peers, announcing to masters", fs.option.Host, len(fs.startupPeers))
return
}
remaining := time.Until(deadline)
if remaining <= 0 {
glog.Warningf("%s metadata replay not caught up after %v, announcing anyway", fs.option.Host, announceCatchUpCap)
return
}
select {
case <-advanced:
case <-time.After(min(remaining, time.Second)):
}
}
}
// masterRemovalWaitBudget bounds how long shutdown waits for masters to
// drop this filer before gRPC drains anyway.
const masterRemovalWaitBudget = 5 * time.Second
// waitForClusterRemoval polls the configured masters until none still lists
// this filer. The master removes membership when the KeepConnected handler
// unwinds, which is asynchronous to cancel, so new requests could otherwise
// route to this filer after its gRPC listener has already closed.
func (fs *FilerServer) waitForClusterRemoval() {
ctx, cancel := context.WithTimeout(context.Background(), masterRemovalWaitBudget)
defer cancel()
confirmedAbsent := make(map[pb.ServerAddress]struct{})
for ctx.Err() == nil {
pending := false
// The cached list keeps a slow SRV lookup from stretching the
// removal-wait budget.
for _, master := range fs.filer.MasterClient.ListCachedMasters() {
if _, ok := confirmedAbsent[master]; ok {
continue
}
if ctx.Err() != nil {
return
}
// The per-query timeout shares the overall deadline, so a stalled
// master cannot stretch the wait past the budget.
queryCtx, queryCancel := context.WithTimeout(ctx, 2*time.Second)
nodes, err := cluster.ListExistingPeerUpdates(queryCtx, master, fs.grpcDialOption, fs.filer.MasterClient.FilerGroup, cluster.FilerType)
queryCancel()
if err != nil {
// A failed query is not confirmed removal; retry until the
// master answers or the overall deadline expires.
pending = true
continue
}
listed := false
for _, node := range nodes {
if node.Address == string(fs.option.Host) {
listed = true
break
}
}
if listed {
pending = true
} else {
confirmedAbsent[master] = struct{}{}
}
}
if !pending {
return
}
select {
case <-ctx.Done():
return
case <-time.After(200 * time.Millisecond):
}
}
}
func (fs *FilerServer) leaveLockRing(removalTimeout time.Duration) {

Some files were not shown because too many files have changed in this diff Show More