Compare commits

..
5 Commits
Author SHA1 Message Date
Chris Lu 2f3b225052 Address Copilot review: remove redundant zero initialization 2025-12-30 17:14:52 -08:00
Chris Lu e2b602c486 Address Copilot review comments 2025-12-30 17:00:13 -08:00
Chris Lu 4081940720 Merge master and resolve conflicts 2025-12-30 16:43:37 -08:00
Lisandro Pin 9318a1265e Have volume.list account for EC shards when computing disk usage. 2025-12-22 14:41:34 +01:00
Lisandro Pin 5725d2c464 Fix reporting of EC shard sizes from nodes to masters.
SeaweedFS tracks EC shard sizes on topology data stuctures, but this information is never
relayed to master servers :( The end result is that commands reporting disk usage, such
as `volume.list` and `cluster.status`, yield incorrect figures when EC shards are present.

As an example for a simple 5-node test cluster, before...

```
> volume.list
Topology volumeSizeLimit:30000 MB hdd(volume:6/40 active:6 free:33 remote:0)
  DataCenter DefaultDataCenter hdd(volume:6/40 active:6 free:33 remote:0)
    Rack DefaultRack hdd(volume:6/40 active:6 free:33 remote:0)
      DataNode 192.168.10.111:9001 hdd(volume:1/8 active:1 free:7 remote:0)
        Disk hdd(volume:1/8 active:1 free:7 remote:0) id:0
          volume id:3  size:88967096  file_count:172  replica_placement:2  version:3  modified_at_second:1766349617
          ec volume id:1 collection: shards:[1 5]
        Disk hdd total size:88967096 file_count:172
      DataNode 192.168.10.111:9001 total size:88967096 file_count:172
  DataCenter DefaultDataCenter hdd(volume:6/40 active:6 free:33 remote:0)
    Rack DefaultRack hdd(volume:6/40 active:6 free:33 remote:0)
      DataNode 192.168.10.111:9002 hdd(volume:2/8 active:2 free:6 remote:0)
        Disk hdd(volume:2/8 active:2 free:6 remote:0) id:0
          volume id:2  size:77267536  file_count:166  replica_placement:2  version:3  modified_at_second:1766349617
          volume id:3  size:88967096  file_count:172  replica_placement:2  version:3  modified_at_second:1766349617
          ec volume id:1 collection: shards:[0 4]
        Disk hdd total size:166234632 file_count:338
      DataNode 192.168.10.111:9002 total size:166234632 file_count:338
  DataCenter DefaultDataCenter hdd(volume:6/40 active:6 free:33 remote:0)
    Rack DefaultRack hdd(volume:6/40 active:6 free:33 remote:0)
      DataNode 192.168.10.111:9003 hdd(volume:1/8 active:1 free:7 remote:0)
        Disk hdd(volume:1/8 active:1 free:7 remote:0) id:0
          volume id:2  size:77267536  file_count:166  replica_placement:2  version:3  modified_at_second:1766349617
          ec volume id:1 collection: shards:[2 6]
        Disk hdd total size:77267536 file_count:166
      DataNode 192.168.10.111:9003 total size:77267536 file_count:166
  DataCenter DefaultDataCenter hdd(volume:6/40 active:6 free:33 remote:0)
    Rack DefaultRack hdd(volume:6/40 active:6 free:33 remote:0)
      DataNode 192.168.10.111:9004 hdd(volume:2/8 active:2 free:6 remote:0)
        Disk hdd(volume:2/8 active:2 free:6 remote:0) id:0
          volume id:2  size:77267536  file_count:166  replica_placement:2  version:3  modified_at_second:1766349617
          volume id:3  size:88967096  file_count:172  replica_placement:2  version:3  modified_at_second:1766349617
          ec volume id:1 collection: shards:[3 7]
        Disk hdd total size:166234632 file_count:338
      DataNode 192.168.10.111:9004 total size:166234632 file_count:338
  DataCenter DefaultDataCenter hdd(volume:6/40 active:6 free:33 remote:0)
    Rack DefaultRack hdd(volume:6/40 active:6 free:33 remote:0)
      DataNode 192.168.10.111:9005 hdd(volume:0/8 active:0 free:8 remote:0)
        Disk hdd(volume:0/8 active:0 free:8 remote:0) id:0
          ec volume id:1 collection: shards:[8 9 10 11 12 13]
        Disk hdd total size:0 file_count:0
    Rack DefaultRack total size:498703896 file_count:1014
  DataCenter DefaultDataCenter total size:498703896 file_count:1014
total size:498703896 file_count:1014
```

...and after:

```
> volume.list
Topology volumeSizeLimit:30000 MB hdd(volume:6/40 active:6 free:33 remote:0)
  DataCenter DefaultDataCenter hdd(volume:6/40 active:6 free:33 remote:0)
    Rack DefaultRack hdd(volume:6/40 active:6 free:33 remote:0)
      DataNode 192.168.10.111:9001 hdd(volume:1/8 active:1 free:7 remote:0)
        Disk hdd(volume:1/8 active:1 free:7 remote:0) id:0
          volume id:2  size:81761800  file_count:161  replica_placement:2  version:3  modified_at_second:1766349495
          ec volume id:1 collection: shards:[1 5 9] sizes:[1:8.00 MiB 5:8.00 MiB 9:8.00 MiB] total:24.00 MiB
        Disk hdd total size:81761800 file_count:161
      DataNode 192.168.10.111:9001 total size:81761800 file_count:161
  DataCenter DefaultDataCenter hdd(volume:6/40 active:6 free:33 remote:0)
    Rack DefaultRack hdd(volume:6/40 active:6 free:33 remote:0)
      DataNode 192.168.10.111:9002 hdd(volume:1/8 active:1 free:7 remote:0)
        Disk hdd(volume:1/8 active:1 free:7 remote:0) id:0
          volume id:3  size:88678712  file_count:170  replica_placement:2  version:3  modified_at_second:1766349495
          ec volume id:1 collection: shards:[11 12 13] sizes:[11:8.00 MiB 12:8.00 MiB 13:8.00 MiB] total:24.00 MiB
        Disk hdd total size:88678712 file_count:170
      DataNode 192.168.10.111:9002 total size:88678712 file_count:170
  DataCenter DefaultDataCenter hdd(volume:6/40 active:6 free:33 remote:0)
    Rack DefaultRack hdd(volume:6/40 active:6 free:33 remote:0)
      DataNode 192.168.10.111:9003 hdd(volume:2/8 active:2 free:6 remote:0)
        Disk hdd(volume:2/8 active:2 free:6 remote:0) id:0
          volume id:2  size:81761800  file_count:161  replica_placement:2  version:3  modified_at_second:1766349495
          volume id:3  size:88678712  file_count:170  replica_placement:2  version:3  modified_at_second:1766349495
          ec volume id:1 collection: shards:[0 4 8] sizes:[0:8.00 MiB 4:8.00 MiB 8:8.00 MiB] total:24.00 MiB
        Disk hdd total size:170440512 file_count:331
      DataNode 192.168.10.111:9003 total size:170440512 file_count:331
  DataCenter DefaultDataCenter hdd(volume:6/40 active:6 free:33 remote:0)
    Rack DefaultRack hdd(volume:6/40 active:6 free:33 remote:0)
      DataNode 192.168.10.111:9004 hdd(volume:2/8 active:2 free:6 remote:0)
        Disk hdd(volume:2/8 active:2 free:6 remote:0) id:0
          volume id:2  size:81761800  file_count:161  replica_placement:2  version:3  modified_at_second:1766349495
          volume id:3  size:88678712  file_count:170  replica_placement:2  version:3  modified_at_second:1766349495
          ec volume id:1 collection: shards:[2 6 10] sizes:[2:8.00 MiB 6:8.00 MiB 10:8.00 MiB] total:24.00 MiB
        Disk hdd total size:170440512 file_count:331
      DataNode 192.168.10.111:9004 total size:170440512 file_count:331
  DataCenter DefaultDataCenter hdd(volume:6/40 active:6 free:33 remote:0)
    Rack DefaultRack hdd(volume:6/40 active:6 free:33 remote:0)
      DataNode 192.168.10.111:9005 hdd(volume:0/8 active:0 free:8 remote:0)
        Disk hdd(volume:0/8 active:0 free:8 remote:0) id:0
          ec volume id:1 collection: shards:[3 7] sizes:[3:8.00 MiB 7:8.00 MiB] total:16.00 MiB
        Disk hdd total size:0 file_count:0
    Rack DefaultRack total size:511321536 file_count:993
  DataCenter DefaultDataCenter total size:511321536 file_count:993
total size:511321536 file_count:993
```
2025-12-21 23:03:57 +01:00
444 changed files with 4062 additions and 10164 deletions
+1 -21
View File
@@ -33,29 +33,9 @@ jobs:
run: |
go test -v
- name: Collect server logs on failure
if: failure()
run: |
echo "Collecting server logs from temp directories..."
mkdir -p /tmp/ec-test-logs
# Find all temp directories created by the tests (they persist on failure with t.TempDir())
find /tmp -maxdepth 1 -type d \( -name "TestEC*" -o -name "TestDisk*" -o -name "TestCross*" -o -name "TestEvacuation*" \) 2>/dev/null | while read dir; do
if [ -d "$dir" ]; then
echo "Found test directory: $dir"
# Copy the entire directory structure to preserve organization
cp -r "$dir" /tmp/ec-test-logs/ 2>/dev/null || true
fi
done
# List what we collected
echo "Collected logs:"
find /tmp/ec-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v6
with:
name: ec-integration-test-logs
path: |
/tmp/ec-test-logs/
test/erasure_coding/
if-no-files-found: warn
path: test/erasure_coding
+5 -28
View File
@@ -54,9 +54,9 @@ Table of Contents
=================
* [Quick Start](#quick-start)
* [Quick Start with weed mini](#quick-start-with-weed-mini)
* [Quick Start for S3 API on Docker](#quick-start-for-s3-api-on-docker)
* [Quick Start with Single Binary](#quick-start-with-single-binary)
* [Quick Start SeaweedFS S3 on AWS](#quick-start-seaweedfs-s3-on-aws)
* [Introduction](#introduction)
* [Features](#features)
* [Additional Features](#additional-features)
@@ -78,32 +78,6 @@ Table of Contents
# Quick Start #
## Quick Start with weed mini ##
The easiest way to get started with SeaweedFS for development and testing:
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`.
Example:
```bash
# remove quarantine on macOS
# xattr -d com.apple.quarantine ./weed
export AWS_ACCESS_KEY_ID=admin
export AWS_SECRET_ACCESS_KEY=key
./weed mini -dir=/data
```
This single command starts a complete SeaweedFS setup with:
- **Master UI**: http://localhost:9333
- **Volume Server**: http://localhost:9340
- **Filer UI**: http://localhost:8888
- **S3 Endpoint**: http://localhost:8333
- **WebDAV**: http://localhost:7333
- **Admin UI**: http://localhost:23646
Perfect for development, testing, learning SeaweedFS, and single node deployments!
## Quick Start for S3 API on Docker ##
`docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
@@ -111,10 +85,13 @@ Perfect for development, testing, learning SeaweedFS, and single node deployment
## Quick Start with Single Binary ##
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`. Or run `go install github.com/seaweedfs/seaweedfs/weed@latest`.
* `export AWS_ACCESS_KEY_ID=admin ; export AWS_SECRET_ACCESS_KEY=key` as the admin credentials to access the object store.
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway. The difference with `weed mini` is that `weed mini` can auto configure based on the single host environment, while `weed server` requires manual configuration and are designed for production use.
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -master="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
## Quick Start SeaweedFS S3 on AWS ##
* Setup fast production-ready [SeaweedFS S3 on AWS with cloudformation](https://aws.amazon.com/marketplace/pp/prodview-nzelz5gprlrjc)
# Introduction #
SeaweedFS is a simple and highly scalable distributed file system. There are two objectives:
+169
View File
@@ -0,0 +1,169 @@
# Server-Side Encryption with Customer-Provided Keys (SSE-C) Implementation
This document describes the implementation of SSE-C support in SeaweedFS, addressing the feature request from [GitHub Discussion #5361](https://github.com/seaweedfs/seaweedfs/discussions/5361).
## Overview
SSE-C allows clients to provide their own encryption keys for server-side encryption of objects stored in SeaweedFS. The server encrypts the data using the customer-provided AES-256 key but does not store the key itself - only an MD5 hash of the key for validation purposes.
## Implementation Details
### Architecture
The SSE-C implementation follows a transparent encryption/decryption pattern:
1. **Upload (PUT/POST)**: Data is encrypted with the customer key before being stored
2. **Download (GET/HEAD)**: Encrypted data is decrypted on-the-fly using the customer key
3. **Metadata Storage**: Only the encryption algorithm and key MD5 are stored as metadata
### Key Components
#### 1. Constants and Headers (`weed/s3api/s3_constants/header.go`)
- Added AWS-compatible SSE-C header constants
- Support for both regular and copy-source SSE-C headers
#### 2. Core SSE-C Logic (`weed/s3api/s3_sse_c.go`)
- **SSECustomerKey**: Structure to hold customer encryption key and metadata
- **SSECEncryptedReader**: Streaming encryption with AES-256-CTR mode
- **SSECDecryptedReader**: Streaming decryption with IV extraction
- **validateAndParseSSECHeaders**: Shared validation logic (DRY principle)
- **ParseSSECHeaders**: Parse regular SSE-C headers
- **ParseSSECCopySourceHeaders**: Parse copy-source SSE-C headers
- Header validation and parsing functions
- Metadata extraction and response handling
#### 3. Error Handling (`weed/s3api/s3err/s3api_errors.go`)
- New error codes for SSE-C validation failures
- AWS-compatible error messages and HTTP status codes
#### 4. S3 API Integration
- **PUT Object Handler**: Encrypts data streams transparently
- **GET Object Handler**: Decrypts data streams transparently
- **HEAD Object Handler**: Validates keys and returns appropriate headers
- **Metadata Storage**: Integrates with existing `SaveAmzMetaData` function
### Encryption Scheme
- **Algorithm**: AES-256-CTR (Counter mode)
- **Key Size**: 256 bits (32 bytes)
- **IV Generation**: Random 16-byte IV per object
- **Storage Format**: `[IV][EncryptedData]` where IV is prepended to encrypted content
### Metadata Storage
SSE-C metadata is stored in the filer's extended attributes:
```
x-amz-server-side-encryption-customer-algorithm: "AES256"
x-amz-server-side-encryption-customer-key-md5: "<md5-hash-of-key>"
```
## API Compatibility
### Required Headers for Encryption (PUT/POST)
```
x-amz-server-side-encryption-customer-algorithm: AES256
x-amz-server-side-encryption-customer-key: <base64-encoded-256-bit-key>
x-amz-server-side-encryption-customer-key-md5: <md5-hash-of-key>
```
### Required Headers for Decryption (GET/HEAD)
Same headers as encryption - the server validates the key MD5 matches.
### Copy Operations
Support for copy-source SSE-C headers:
```
x-amz-copy-source-server-side-encryption-customer-algorithm
x-amz-copy-source-server-side-encryption-customer-key
x-amz-copy-source-server-side-encryption-customer-key-md5
```
## Error Handling
The implementation provides AWS-compatible error responses:
- **InvalidEncryptionAlgorithmError**: Non-AES256 algorithm specified
- **InvalidArgument**: Invalid key format, size, or MD5 mismatch
- **Missing customer key**: Object encrypted but no key provided
- **Unnecessary customer key**: Object not encrypted but key provided
## Security Considerations
1. **Key Management**: Customer keys are never stored - only MD5 hashes for validation
2. **IV Randomness**: Fresh random IV generated for each object
3. **Transparent Security**: Volume servers never see unencrypted data
4. **Key Validation**: Strict validation of key format, size, and MD5
## Testing
Comprehensive test suite covers:
- Header validation and parsing (regular and copy-source)
- Encryption/decryption round-trip
- Error condition handling
- Metadata extraction
- Code reuse validation (DRY principle)
- AWS S3 compatibility
Run tests with:
```bash
go test -v ./weed/s3api
## Usage Example
### Upload with SSE-C
```bash
# Generate a 256-bit key
KEY=$(openssl rand -base64 32)
KEY_MD5=$(echo -n "$KEY" | base64 -d | openssl dgst -md5 -binary | base64)
# Upload object with SSE-C
curl -X PUT "http://localhost:8333/bucket/object" \
-H "x-amz-server-side-encryption-customer-algorithm: AES256" \
-H "x-amz-server-side-encryption-customer-key: $KEY" \
-H "x-amz-server-side-encryption-customer-key-md5: $KEY_MD5" \
--data-binary @file.txt
```
### Download with SSE-C
```bash
# Download object with SSE-C (same key required)
curl "http://localhost:8333/bucket/object" \
-H "x-amz-server-side-encryption-customer-algorithm: AES256" \
-H "x-amz-server-side-encryption-customer-key: $KEY" \
-H "x-amz-server-side-encryption-customer-key-md5: $KEY_MD5"
```
## Integration Points
### Existing SeaweedFS Features
- **Filer Metadata**: Extends existing metadata storage
- **Volume Servers**: No changes required - store encrypted data transparently
- **S3 API**: Integrates seamlessly with existing handlers
- **Versioning**: Compatible with object versioning
- **Multipart Upload**: Ready for multipart upload integration
### Future Enhancements
- **SSE-S3**: Server-managed encryption keys
- **SSE-KMS**: External key management service integration
- **Performance Optimization**: Hardware acceleration for encryption
- **Compliance**: Enhanced audit logging for encrypted objects
## File Changes Summary
1. **`weed/s3api/s3_constants/header.go`** - Added SSE-C header constants
2. **`weed/s3api/s3_sse_c.go`** - Core SSE-C implementation (NEW)
3. **`weed/s3api/s3_sse_c_test.go`** - Comprehensive test suite (NEW)
4. **`weed/s3api/s3err/s3api_errors.go`** - Added SSE-C error codes
5. **`weed/s3api/s3api_object_handlers.go`** - GET/HEAD with SSE-C support
6. **`weed/s3api/s3api_object_handlers_put.go`** - PUT with SSE-C support
7. **`weed/server/filer_server_handlers_write_autochunk.go`** - Metadata storage
## Compliance
This implementation follows the [AWS S3 SSE-C specification](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html) for maximum compatibility with existing S3 clients and tools.
## Performance Impact
- **Encryption Overhead**: Minimal CPU impact with efficient AES-CTR streaming
- **Memory Usage**: Constant memory usage via streaming encryption/decryption
- **Storage Overhead**: 16 bytes per object for IV storage
- **Network**: No additional network overhead
+10 -10
View File
@@ -93,12 +93,12 @@ require (
go.opencensus.io v0.24.0 // indirect
gocloud.dev v0.44.0
gocloud.dev/pubsub/natspubsub v0.44.0
gocloud.dev/pubsub/rabbitpubsub v0.44.0
gocloud.dev/pubsub/rabbitpubsub v0.43.0
golang.org/x/crypto v0.46.0
golang.org/x/exp v0.0.0-20250811191247-51f88131bc50
golang.org/x/image v0.34.0
golang.org/x/net v0.48.0
golang.org/x/oauth2 v0.34.0
golang.org/x/oauth2 v0.34.0 // indirect
golang.org/x/sys v0.39.0
golang.org/x/text v0.32.0 // indirect
golang.org/x/tools v0.39.0 // indirect
@@ -127,14 +127,15 @@ require (
github.com/aws/aws-sdk-go-v2 v1.41.0
github.com/aws/aws-sdk-go-v2/config v1.32.6
github.com/aws/aws-sdk-go-v2/credentials v1.19.6
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0
github.com/cognusion/imaging v1.0.2
github.com/fluent/fluent-logger-golang v1.10.1
github.com/getsentry/sentry-go v0.40.0
github.com/getsentry/sentry-go v0.38.0
github.com/gin-contrib/sessions v1.0.4
github.com/gin-gonic/gin v1.11.0
github.com/golang-jwt/jwt/v5 v5.3.0
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
github.com/hanwen/go-fuse/v2 v2.9.0
github.com/hashicorp/raft v1.7.3
github.com/hashicorp/raft-boltdb/v2 v2.3.1
github.com/hashicorp/vault/api v1.22.0
@@ -151,14 +152,13 @@ require (
github.com/rdleal/intervalst v1.5.0
github.com/redis/go-redis/v9 v9.17.2
github.com/schollz/progressbar/v3 v3.19.0
github.com/seaweedfs/go-fuse/v2 v2.9.1
github.com/shirou/gopsutil/v4 v4.25.11
github.com/tarantool/go-tarantool/v2 v2.4.1
github.com/tikv/client-go/v2 v2.0.7
github.com/xeipuuv/gojsonschema v1.2.0
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.1
go.etcd.io/etcd/client/pkg/v3 v3.6.7
github.com/ydb-platform/ydb-go-sdk/v3 v3.122.0
go.etcd.io/etcd/client/pkg/v3 v3.6.6
go.uber.org/atomic v1.11.0
golang.org/x/sync v0.19.0
golang.org/x/tools/godoc v0.1.0-deprecated
@@ -266,11 +266,11 @@ require (
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
+18 -18
View File
@@ -683,18 +683,18 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 h1:NLYTEyZmVZo0Qh183sC8nC+ydJXOOeIL/qI/sS3PdLY=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15/go.mod h1:Z803iB3B0bc8oJV8zH2PERLRfQUJ2n2BXISpsA4+O1M=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 h1:P1MU/SuhadGvg2jtviDXPEejU3jBNhoeeAlRadHzvHI=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6/go.mod h1:5KYaMG6wmVKMFBSfWoyG/zH8pWwzQFnKgpoSRlXHKdQ=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A=
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 h1:MIWra+MSq53CFaXXAywB2qg9YvVZifkk6vEGl/1Qor0=
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 h1:wsSQ4SVz5YE1crz0Ap7VBZrV4nNqZt4CIBBT8mnwoNc=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15/go.mod h1:I7sditnFGtYMIqPRU1QoHZAUrXkGp4SczmlLwrNPlD0=
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 h1:IrbE3B8O9pm3lsg96AXIN5MXX4pECEuExh/A0Du3AuI=
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0/go.mod h1:/sJLzHtiiZvs6C1RbxS/anSAFwZD6oC6M/kotQzOiLw=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU=
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 h1:OBuZE9Wt8h2imuRktu+WfjiTGrnYdCIJg8IX92aalHE=
@@ -927,8 +927,8 @@ github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBv
github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok=
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
github.com/getsentry/sentry-go v0.40.0 h1:VTJMN9zbTvqDqPwheRVLcp0qcUcM+8eFivvGocAaSbo=
github.com/getsentry/sentry-go v0.40.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
github.com/getsentry/sentry-go v0.38.0 h1:S8Xui7gLeAvXINVLMOaX94HnsDf1GexnfXGSNC4+KQs=
github.com/getsentry/sentry-go v0.38.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gin-contrib/sessions v1.0.4 h1:ha6CNdpYiTOK/hTp05miJLbpTSNfOnFg5Jm2kbcqy8U=
github.com/gin-contrib/sessions v1.0.4/go.mod h1:ccmkrb2z6iU2osiAHZG3x3J4suJK+OU27oqzlWOqQgs=
@@ -1187,6 +1187,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+u
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90=
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
github.com/hanwen/go-fuse/v2 v2.9.0 h1:0AOGUkHtbOVeyGLr0tXupiid1Vg7QB7M6YUcdmVdC58=
github.com/hanwen/go-fuse/v2 v2.9.0/go.mod h1:yE6D2PqWwm3CbYRxFXV9xUd8Md5d6NG0WBs5spCswmI=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -1619,8 +1621,6 @@ github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1Ivohy
github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138 h1:bX1vBF7GQjPeFQsCAZ8gCQGS/nJQnekL7gZ4Qg/pF4E=
github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
github.com/seaweedfs/go-fuse/v2 v2.9.1 h1:gnKmfrKreCRGJmekGz5WMnNZqXEf9s9+V2hdWQdvx88=
github.com/seaweedfs/go-fuse/v2 v2.9.1/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
github.com/seaweedfs/raft v1.1.6 h1:e83Xn0boscPnuSiBllUPeWRVS6JKrqJPYBozgFyBiC0=
@@ -1786,8 +1786,8 @@ github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1 h1:XaRxeVrOyl3y6v9CiYMWaF
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
github.com/ydb-platform/ydb-go-sdk/v3 v3.44.0/go.mod h1:oSLwnuilwIpaF5bJJMAofnGgzPJusoI3zWMNb8I+GnM=
github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.1 h1:YaqzRVbcncabB34YNjOl5ADomYUFva+6l74svIIIJUo=
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.1/go.mod h1:stS1mQYjbJvwwYaYzKyFY9eMiuVXWWXQA6T+SpOLg9c=
github.com/ydb-platform/ydb-go-sdk/v3 v3.122.0 h1:uvqheUfoEJz0CLLLaYct69wpvoaosM/2joWFcbwoqfw=
github.com/ydb-platform/ydb-go-sdk/v3 v3.122.0/go.mod h1:/LjMxb/rXmoGAAnImoqAFIlhO5ampHacbvDetQitCk4=
github.com/ydb-platform/ydb-go-yc v0.12.1 h1:qw3Fa+T81+Kpu5Io2vYHJOwcrYrVjgJlT6t/0dOXJrA=
github.com/ydb-platform/ydb-go-yc v0.12.1/go.mod h1:t/ZA4ECdgPWjAb4jyDe8AzQZB5dhpGbi3iCahFaNwBY=
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 h1:9E5q8Nsy2RiJMZDNVy0A3KUrIMBPakJ2VgloeWbcI84=
@@ -1822,8 +1822,8 @@ go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I=
go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM=
go.etcd.io/etcd/api/v3 v3.6.6 h1:mcaMp3+7JawWv69p6QShYWS8cIWUOl32bFLb6qf8pOQ=
go.etcd.io/etcd/api/v3 v3.6.6/go.mod h1:f/om26iXl2wSkcTA1zGQv8reJRSLVdoEBsi4JdfMrx4=
go.etcd.io/etcd/client/pkg/v3 v3.6.7 h1:vvzgyozz46q+TyeGBuFzVuI53/yd133CHceNb/AhBVs=
go.etcd.io/etcd/client/pkg/v3 v3.6.7/go.mod h1:2IVulJ3FZ/czIGl9T4lMF1uxzrhRahLqe+hSgy+Kh7Q=
go.etcd.io/etcd/client/pkg/v3 v3.6.6 h1:uoqgzSOv2H9KlIF5O1Lsd8sW+eMLuV6wzE3q5GJGQNs=
go.etcd.io/etcd/client/pkg/v3 v3.6.6/go.mod h1:YngfUVmvsvOJ2rRgStIyHsKtOt9SZI2aBJrZiWJhCbI=
go.etcd.io/etcd/client/v3 v3.6.6 h1:G5z1wMf5B9SNexoxOHUGBaULurOZPIgGPsW6CN492ec=
go.etcd.io/etcd/client/v3 v3.6.6/go.mod h1:36Qv6baQ07znPR3+n7t+Rk5VHEzVYPvFfGmfF4wBHV8=
go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss=
@@ -1895,8 +1895,8 @@ gocloud.dev v0.44.0 h1:iVyMAqFl2r6xUy7M4mfqwlN+21UpJoEtgHEcfiLMUXs=
gocloud.dev v0.44.0/go.mod h1:ZmjROXGdC/eKZLF1N+RujDlFRx3D+4Av2thREKDMVxY=
gocloud.dev/pubsub/natspubsub v0.44.0 h1:1Us76ckkdgtiE1p1rJZ+38b9TQP051bmjAiQlFQzYrM=
gocloud.dev/pubsub/natspubsub v0.44.0/go.mod h1:PvVAGIhL14PWGwWIXX/zAK42ixr2/PKP4Q4yMiAUraQ=
gocloud.dev/pubsub/rabbitpubsub v0.44.0 h1:MpRIO6XJ/JTqrlUWt3CxwDe1LvaiXUVu4sS5cv4f/AM=
gocloud.dev/pubsub/rabbitpubsub v0.44.0/go.mod h1:BB9+qT3r6g4M5+4asiXaEeqw4QAOzsWusO5krYaqkdA=
gocloud.dev/pubsub/rabbitpubsub v0.43.0 h1:6nNZFSlJ1dk2GujL8PFltfLz3vC6IbrpjGS4FTduo1s=
gocloud.dev/pubsub/rabbitpubsub v0.43.0/go.mod h1:sEaueAGat+OASRoB3QDkghCtibKttgg7X6zsPTm1pl0=
golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "4.06"
appVersion: "4.04"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.0.406
version: 4.0.404
@@ -25,6 +25,12 @@ spec:
port: {{ .Values.admin.grpcPort }}
targetPort: {{ .Values.admin.grpcPort }}
protocol: TCP
{{- if .Values.admin.metricsPort }}
- name: "metrics"
port: {{ .Values.admin.metricsPort }}
targetPort: {{ .Values.admin.metricsPort }}
protocol: TCP
{{- end }}
selector:
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
@@ -1,4 +1,5 @@
{{- if .Values.admin.enabled }}
{{- if .Values.admin.metricsPort }}
{{- if .Values.global.monitoring.enabled }}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
@@ -21,7 +22,7 @@ metadata:
spec:
endpoints:
- interval: 30s
port: http
port: metrics
scrapeTimeout: 5s
selector:
matchLabels:
@@ -29,3 +30,4 @@ spec:
app.kubernetes.io/component: admin
{{- end }}
{{- end }}
{{- end }}
@@ -119,7 +119,7 @@ spec:
{{- range $key, $value := .Values.admin.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
value: {{ $value | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
@@ -130,7 +130,7 @@ spec:
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
value: {{ $value | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
@@ -213,6 +213,10 @@ spec:
name: http
- containerPort: {{ .Values.admin.grpcPort }}
name: grpc
{{- if .Values.admin.metricsPort }}
- containerPort: {{ .Values.admin.metricsPort }}
name: metrics
{{- end }}
{{- if .Values.admin.readinessProbe.enabled }}
readinessProbe:
httpGet:
@@ -104,7 +104,7 @@ spec:
{{- if and (ne $key $clusterMasterKey) (ne $key $clusterFilerKey) }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
value: {{ $value | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 }}
@@ -117,7 +117,7 @@ spec:
{{- if and (ne $key $clusterMasterKey) (ne $key $clusterFilerKey) }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
value: {{ $value | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 }}
@@ -100,7 +100,7 @@ spec:
{{- range $key, $value := .Values.cosi.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
value: {{ $value | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
@@ -111,7 +111,7 @@ spec:
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
value: {{ $value | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
@@ -118,7 +118,7 @@ spec:
{{- range $key, $value := .Values.filer.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
value: {{ $value | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
@@ -129,7 +129,7 @@ spec:
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
value: {{ $value | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
@@ -229,7 +229,7 @@ spec:
- name: seaweedfs-filer-log-volume
mountPath: "/logs/"
{{- end }}
{{- if and .Values.filer.s3.enabled .Values.filer.s3.enableAuth }}
{{- if .Values.filer.s3.enableAuth }}
- name: config-users
mountPath: /etc/sw
readOnly: true
@@ -263,7 +263,7 @@ spec:
mountPath: /usr/local/share/ca-certificates/filer/
- name: client-cert
readOnly: true
mountPath: /usr/local/share/ca-certificates/client
mountPath: /usr/local/share/ca-certificates/client/
{{- end }}
{{ tpl .Values.filer.extraVolumeMounts . | nindent 12 | trim }}
ports:
@@ -281,18 +281,12 @@ spec:
name: swfs-s3-tls
{{- end }}
{{- end }}
{{- $isJwtEnabled := or .Values.global.securityConfig.jwtSigning.filerWrite .Values.global.securityConfig.jwtSigning.filerRead }}
{{- if .Values.filer.readinessProbe.enabled }}
readinessProbe:
{{- if or $isJwtEnabled .Values.filer.readinessProbe.tcpSocket }}
tcpSocket:
port: {{ if $isJwtEnabled }}{{ .Values.filer.port }}{{ else }}{{ .Values.filer.readinessProbe.tcpSocket.port }}{{ end }}
{{- else }}
httpGet:
path: {{ .Values.filer.readinessProbe.httpGet.path }}
port: {{ .Values.filer.port }}
scheme: {{ .Values.filer.readinessProbe.httpGet.scheme }}
{{- end }}
initialDelaySeconds: {{ .Values.filer.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.filer.readinessProbe.periodSeconds }}
successThreshold: {{ .Values.filer.readinessProbe.successThreshold }}
@@ -301,15 +295,10 @@ spec:
{{- end }}
{{- if .Values.filer.livenessProbe.enabled }}
livenessProbe:
{{- if or $isJwtEnabled .Values.filer.livenessProbe.tcpSocket }}
tcpSocket:
port: {{ if $isJwtEnabled }}{{ .Values.filer.port }}{{ else }}{{ .Values.filer.livenessProbe.tcpSocket.port }}{{ end }}
{{- else }}
httpGet:
path: {{ .Values.filer.livenessProbe.httpGet.path }}
port: {{ .Values.filer.port }}
scheme: {{ .Values.filer.livenessProbe.httpGet.scheme }}
{{- end }}
initialDelaySeconds: {{ .Values.filer.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.filer.livenessProbe.periodSeconds }}
successThreshold: {{ .Values.filer.livenessProbe.successThreshold }}
@@ -102,7 +102,7 @@ spec:
{{- range $key, $value := .Values.master.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
value: {{ $value | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
@@ -113,7 +113,7 @@ spec:
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
value: {{ $value | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
@@ -94,7 +94,7 @@ spec:
{{- range $key, $value := .Values.s3.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
value: {{ $value | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
@@ -105,7 +105,7 @@ spec:
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
value: {{ $value | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
@@ -4,13 +4,6 @@
{{- /* Determine service name based on deployment mode */}}
{{- $serviceName := ternary (printf "%s-all-in-one" (include "seaweedfs.name" .)) (printf "%s-s3" (include "seaweedfs.name" .)) .Values.allInOne.enabled }}
{{- $s3Port := .Values.allInOne.s3.port | default .Values.s3.port }}
{{- /* Build hosts list - support both legacy .host (string) and new .hosts (array) for backwards compatibility */}}
{{- $hosts := list }}
{{- if kindIs "slice" .Values.s3.ingress.host }}
{{- $hosts = .Values.s3.ingress.host }}
{{- else if .Values.s3.ingress.host }}
{{- $hosts = list .Values.s3.ingress.host }}
{{- end }}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
@@ -37,25 +30,6 @@ spec:
tls:
{{ .Values.s3.ingress.tls | default list | toYaml | nindent 6}}
rules:
{{- if $hosts }}
{{- range $hosts }}
- host: {{ . | quote }}
http:
paths:
- path: {{ $.Values.s3.ingress.path | quote }}
pathType: {{ $.Values.s3.ingress.pathType | quote }}
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ $serviceName }}
port:
number: {{ $s3Port }}
{{- else }}
serviceName: {{ $serviceName }}
servicePort: {{ $s3Port }}
{{- end }}
{{- end }}
{{- else }}
- http:
paths:
- path: {{ .Values.s3.ingress.path | quote }}
@@ -70,5 +44,7 @@ spec:
serviceName: {{ $serviceName }}
servicePort: {{ $s3Port }}
{{- end }}
{{- if .Values.s3.ingress.host }}
host: {{ .Values.s3.ingress.host | quote }}
{{- end }}
{{- end }}
@@ -94,7 +94,7 @@ spec:
{{- range $key, $value := .Values.sftp.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
value: {{ $value | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
@@ -105,7 +105,7 @@ spec:
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
value: {{ $value | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
@@ -122,7 +122,7 @@ spec:
{{- range $key, $value := $volume.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
value: {{ $value | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
@@ -133,7 +133,7 @@ spec:
{{- range $key, $value := $.Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
value: {{ $value | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
@@ -97,7 +97,7 @@ spec:
{{- range $key, $value := .Values.worker.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
value: {{ $value | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
@@ -108,7 +108,7 @@ spec:
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
value: {{ $value | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
@@ -134,7 +134,7 @@ spec:
{{- if .Values.worker.adminServer }}
-admin={{ .Values.worker.adminServer }} \
{{- else }}
-admin={{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}:{{ .Values.admin.port }}{{ if .Values.admin.grpcPort }}.{{ .Values.admin.grpcPort }}{{ end }} \
-admin={{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}:{{ .Values.admin.port }} \
{{- end }}
-capabilities={{ .Values.worker.capabilities }} \
-maxConcurrent={{ .Values.worker.maxConcurrent }} \
+27 -24
View File
@@ -39,8 +39,8 @@ global:
replicationPlacement: "001"
extraEnvironmentVars:
WEED_CLUSTER_DEFAULT: "sw"
WEED_CLUSTER_SW_MASTER: "{{ include \"seaweedfs.cluster.masterAddress\" . }}"
WEED_CLUSTER_SW_FILER: "{{ include \"seaweedfs.cluster.filerAddress\" . }}"
WEED_CLUSTER_SW_MASTER: "seaweedfs-master.seaweedfs:9333"
WEED_CLUSTER_SW_FILER: "seaweedfs-filer-client.seaweedfs:8888"
# WEED_JWT_SIGNING_KEY:
# secretKeyRef:
# name: seaweedfs-signing-key
@@ -261,10 +261,10 @@ master:
tls: []
extraEnvironmentVars:
WEED_MASTER_VOLUME_GROWTH_COPY_1: "7"
WEED_MASTER_VOLUME_GROWTH_COPY_2: "6"
WEED_MASTER_VOLUME_GROWTH_COPY_3: "3"
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: "1"
WEED_MASTER_VOLUME_GROWTH_COPY_1: '7'
WEED_MASTER_VOLUME_GROWTH_COPY_2: '6'
WEED_MASTER_VOLUME_GROWTH_COPY_3: '3'
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: '1'
# used to configure livenessProbe on master-server containers
#
@@ -348,16 +348,16 @@ volume:
# maxVolumes: 0 # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
dataDirs:
- name: data1
type: "hostPath"
hostPathPrefix: /ssd
maxVolumes: 0
- name: data1
type: "hostPath"
hostPathPrefix: /ssd
maxVolumes: 0
# - name: data2
# type: "persistentVolumeClaim"
# storageClass: "yourClassNameOfChoice"
# size: "800Gi"
# maxVolumes: 0
# - name: data2
# type: "persistentVolumeClaim"
# storageClass: "yourClassNameOfChoice"
# size: "800Gi"
# maxVolumes: 0
# This will automatically create a job for patching Kubernetes resources if the dataDirs type is 'persistentVolumeClaim' and the size has changed.
resizeHook:
@@ -397,6 +397,7 @@ volume:
# limit background compaction or copying speed in mega bytes per second
compactionMBps: "50"
# Volume server's rack name
rack: null
@@ -845,14 +846,14 @@ filer:
# secret env variables
secretExtraEnvironmentVars: {}
# WEED_POSTGRES_USERNAME:
# secretKeyRef:
# name: postgres-credentials
# key: username
# WEED_POSTGRES_PASSWORD:
# secretKeyRef:
# name: postgres-credentials
# key: password
# WEED_POSTGRES_USERNAME:
# secretKeyRef:
# name: postgres-credentials
# key: username
# WEED_POSTGRES_PASSWORD:
# secretKeyRef:
# name: postgres-credentials
# key: password
s3:
enabled: false
@@ -1011,7 +1012,7 @@ s3:
ingress:
enabled: false
className: ""
# host: false for "*" hostname, or an array for multiple hostnames
# host: false for "*" hostname
host: "seaweedfs.cluster.local"
path: "/"
pathType: Prefix
@@ -1098,6 +1099,7 @@ admin:
replicas: 1
port: 23646 # Default admin port
grpcPort: 33646 # Default gRPC port for worker connections
metricsPort: 9327
loggingOverrideLevel: null
# Admin authentication
@@ -1229,6 +1231,7 @@ worker:
metricsPort: 9327
# Admin server to connect to
# Format: "host:port" or auto-discover from admin service
adminServer: ""
# Worker capabilities - comma-separated list
@@ -3,14 +3,13 @@ package main
import (
"flag"
"fmt"
"github.com/seaweedfs/seaweedfs/other/mq_client_example/example"
"github.com/seaweedfs/seaweedfs/weed/mq/client/agent_client"
"github.com/seaweedfs/seaweedfs/weed/mq/schema"
"log"
"sync"
"sync/atomic"
"time"
"github.com/seaweedfs/seaweedfs/other/mq_client_example/example"
"github.com/seaweedfs/seaweedfs/weed/mq/client/agent_client"
"github.com/seaweedfs/seaweedfs/weed/mq/schema"
)
var (
@@ -3,14 +3,13 @@ package main
import (
"flag"
"fmt"
"log"
"time"
"github.com/seaweedfs/seaweedfs/other/mq_client_example/example"
"github.com/seaweedfs/seaweedfs/weed/mq/client/agent_client"
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"log"
"time"
)
var (
+17 -17
View File
@@ -1,6 +1,6 @@
module seaweedfs-rdma-sidecar
go 1.24.9
go 1.24
require (
github.com/seaweedfs/seaweedfs v0.0.0-00010101000000-000000000000
@@ -18,27 +18,27 @@ require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/prometheus/client_golang v1.23.2 // indirect
github.com/prometheus/client_golang v1.23.0 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.19.2 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/prometheus/common v0.65.0 // indirect
github.com/prometheus/procfs v0.17.0 // indirect
github.com/sagikazarmark/locafero v0.7.0 // indirect
github.com/seaweedfs/goexif v1.0.3 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/spf13/viper v1.21.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.12.0 // indirect
github.com/spf13/cast v1.7.1 // indirect
github.com/spf13/pflag v1.0.6 // indirect
github.com/spf13/viper v1.20.1 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/image v0.34.0 // indirect
golang.org/x/net v0.48.0 // indirect
golang.org/x/sys v0.39.0 // indirect
golang.org/x/text v0.32.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect
google.golang.org/grpc v1.77.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect
golang.org/x/image v0.30.0 // indirect
golang.org/x/net v0.43.0 // indirect
golang.org/x/sys v0.35.0 // indirect
golang.org/x/text v0.28.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0 // indirect
google.golang.org/grpc v1.74.2 // indirect
google.golang.org/protobuf v1.36.7 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
-16
View File
@@ -44,43 +44,34 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc=
github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE=
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE=
github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8=
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0=
github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw=
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo=
github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs=
github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4=
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4=
github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
@@ -109,26 +100,19 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
golang.org/x/image v0.30.0 h1:jD5RhkmVAnjqaCUXfbGBrn3lpxbknfN9w2UhHHU+5B4=
golang.org/x/image v0.30.0/go.mod h1:SAEUTxCCMWSrJcCy/4HwavEsfZZJlYxeHLc6tTiAe/c=
golang.org/x/image v0.34.0/go.mod h1:2RNFBZRB+vnwwFil8GkMdRvrJOFd1AzdZI6vOY+eJVU=
golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0 h1:MAKi5q709QWfnkkpNQ0M12hYJ1+e8qYVDyowc4U1XZM=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4=
google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM=
google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig=
google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A=
google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+2 -2
View File
@@ -419,7 +419,7 @@ func (c *Client) Read(ctx context.Context, req *ReadRequest) (*ReadResponse, err
}).Info("✅ RDMA read completed successfully")
// MOCK DATA IMPLEMENTATION - FOR DEVELOPMENT/TESTING ONLY
//
//
// This section generates placeholder data for the mock RDMA implementation.
// In a production RDMA implementation, this should be replaced with:
//
@@ -472,7 +472,7 @@ func (c *Client) ReadFileRange(ctx context.Context, fileID string, offset, size
if err != nil {
return nil, fmt.Errorf("invalid file ID %s: %w", fileID, err)
}
req := &ReadRequest{
VolumeID: volumeID,
NeedleID: needleID,
@@ -8,7 +8,6 @@ import (
"net/http"
"os"
"path/filepath"
"strings"
"time"
"seaweedfs-rdma-sidecar/pkg/rdma"
+2 -3
View File
@@ -7,11 +7,10 @@
package proto
import (
reflect "reflect"
sync "sync"
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
)
const (
+3 -3
View File
@@ -7,7 +7,7 @@ toolchain go1.25.0
require (
github.com/prometheus/client_golang v1.23.2
github.com/seaweedfs/seaweedfs v0.0.0-00010101000000-000000000000
google.golang.org/protobuf v1.36.11
google.golang.org/protobuf v1.36.8
)
require (
@@ -16,9 +16,9 @@ require (
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.19.2 // indirect
github.com/prometheus/procfs v0.17.0 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
golang.org/x/sys v0.39.0 // indirect
golang.org/x/sys v0.36.0 // indirect
)
replace github.com/seaweedfs/seaweedfs => ../..
-3
View File
@@ -26,7 +26,6 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0=
github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw=
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
@@ -37,10 +36,8 @@ go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+26 -51
View File
@@ -18,8 +18,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/shell"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -35,8 +33,9 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) {
}
// Create temporary directory for test data
// Using t.TempDir() automatically preserves logs when tests fail
testDir := t.TempDir()
testDir, err := os.MkdirTemp("", "seaweedfs_ec_integration_test_")
require.NoError(t, err)
defer os.RemoveAll(testDir)
// Start SeaweedFS cluster with multiple volume servers
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
@@ -245,8 +244,9 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) {
}
// Create temporary directory for test data
// Using t.TempDir() automatically preserves logs when tests fail
testDir := t.TempDir()
testDir, err := os.MkdirTemp("", "seaweedfs_ec_race_test_")
require.NoError(t, err)
defer os.RemoveAll(testDir)
// Start SeaweedFS cluster
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
@@ -783,8 +783,9 @@ func TestDiskAwareECRebalancing(t *testing.T) {
t.Skip("Skipping disk-aware integration test in short mode")
}
// Using t.TempDir() automatically preserves logs when tests fail
testDir := t.TempDir()
testDir, err := os.MkdirTemp("", "seaweedfs_disk_aware_ec_test_")
require.NoError(t, err)
defer os.RemoveAll(testDir)
ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
defer cancel()
@@ -1216,8 +1217,9 @@ func TestECDiskTypeSupport(t *testing.T) {
t.Skip("Skipping disk type integration test in short mode")
}
// Using t.TempDir() automatically preserves logs when tests fail
testDir := t.TempDir()
testDir, err := os.MkdirTemp("", "seaweedfs_ec_disktype_test_")
require.NoError(t, err)
defer os.RemoveAll(testDir)
ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
defer cancel()
@@ -1556,8 +1558,9 @@ func TestECDiskTypeMixedCluster(t *testing.T) {
t.Skip("Skipping mixed disk type integration test in short mode")
}
// Using t.TempDir() automatically preserves logs when tests fail
testDir := t.TempDir()
testDir, err := os.MkdirTemp("", "seaweedfs_ec_mixed_disktype_test_")
require.NoError(t, err)
defer os.RemoveAll(testDir)
ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
defer cancel()
@@ -1745,8 +1748,9 @@ func TestEvacuationFallbackBehavior(t *testing.T) {
t.Skip("Skipping evacuation fallback test in short mode")
}
// Using t.TempDir() automatically preserves logs when tests fail
testDir := t.TempDir()
testDir, err := os.MkdirTemp("", "seaweedfs_evacuation_fallback_test_")
require.NoError(t, err)
defer os.RemoveAll(testDir)
ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
defer cancel()
@@ -1838,8 +1842,9 @@ func TestCrossRackECPlacement(t *testing.T) {
t.Skip("Skipping cross-rack EC placement test in short mode")
}
// Using t.TempDir() automatically preserves logs when tests fail
testDir := t.TempDir()
testDir, err := os.MkdirTemp("", "seaweedfs_cross_rack_ec_test_")
require.NoError(t, err)
defer os.RemoveAll(testDir)
ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
defer cancel()
@@ -2191,8 +2196,9 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) {
}
// Create temporary directory for test data
// Using t.TempDir() automatically preserves logs when tests fail
testDir := t.TempDir()
testDir, err := os.MkdirTemp("", "seaweedfs_ec_replica_sync_test_")
require.NoError(t, err)
defer os.RemoveAll(testDir)
// Start SeaweedFS cluster
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
@@ -2365,47 +2371,16 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) {
// injectNeedleToOneReplica writes a needle directly to one replica to create divergence
func injectNeedleToOneReplica(grpcDialOption grpc.DialOption, vid needle.VolumeId, location wdclient.Location, needleId uint64, data []byte) error {
return operation.WithVolumeServerClient(false, location.ServerAddress(), grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
blob, size := generateNeedleBlobV3(needleId, data)
_, err := client.WriteNeedleBlob(context.Background(), &volume_server_pb.WriteNeedleBlobRequest{
VolumeId: uint32(vid),
NeedleId: needleId,
Size: size,
NeedleBlob: blob,
Size: int32(len(data)),
NeedleBlob: data,
})
return err
})
}
func generateNeedleBlobV3(needleId uint64, data []byte) ([]byte, int32) {
// Overhead: DataSize(4) + Data(N) + Flags(1)
size := 4 + len(data) + 1
// Create struct to calculate padding
// Padding is based on the Size field (DataSize + Overhead)
padding := needle.PaddingLength(types.Size(size), needle.Version3)
blob := make([]byte, 16+size+4+8+int(padding)) // Header(16) + Body(Size) + Checksum(4) + Timestamp(8) + Padding
// Header
util.Uint32toBytes(blob[0:4], 0x12345678) // Cookie
util.Uint64toBytes(blob[4:12], needleId)
util.Uint32toBytes(blob[12:16], uint32(size))
// Body
util.Uint32toBytes(blob[16:20], uint32(len(data)))
copy(blob[20:], data)
blob[20+len(data)] = 0 // Flags
// Checksum
crc := needle.NewCRC(data)
util.Uint32toBytes(blob[20+len(data)+1:20+len(data)+1+4], crc.Value())
// Timestamp (space reserved, filled by server)
// Padding (zeroes)
return blob, int32(size)
}
// uploadTestDataWithReplication uploads test data and returns the volume ID
// using the specified collection and replication level. All files are uploaded to the same volume.
func uploadTestDataWithReplication(t *testing.T, masterAddr string, collection string, replication string) (needle.VolumeId, error) {
-1
View File
@@ -3,7 +3,6 @@ package fuse_test
import (
"fmt"
"io/fs"
"net"
"os"
"os/exec"
"path/filepath"
+1 -1
View File
@@ -3,5 +3,5 @@ package fuse_test
import "testing"
func TestMinimal(t *testing.T) {
t.Log("minimal test")
t.Log("minimal test")
}
+12 -10
View File
@@ -1,6 +1,8 @@
module github.com/seaweedfs/seaweedfs/test/kafka
go 1.24.9
go 1.24.0
toolchain go1.24.7
require (
github.com/IBM/sarama v1.46.0
@@ -14,7 +16,7 @@ require (
replace github.com/seaweedfs/seaweedfs => ../../
require (
cloud.google.com/go/auth v0.17.0 // indirect
cloud.google.com/go/auth v0.16.5 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect
@@ -46,7 +48,7 @@ require (
github.com/aws/aws-sdk-go-v2/config v1.32.6 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.19.6 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.3 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
@@ -115,7 +117,7 @@ require (
github.com/google/btree v1.1.3 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
github.com/googleapis/gax-go/v2 v2.15.0 // indirect
github.com/gorilla/mux v1.8.1 // indirect
github.com/gorilla/schema v1.4.1 // indirect
@@ -161,7 +163,7 @@ require (
github.com/oracle/oci-go-sdk/v65 v65.98.0 // indirect
github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect
github.com/panjf2000/ants/v2 v2.11.3 // indirect
github.com/parquet-go/parquet-go v0.26.3 // indirect
github.com/parquet-go/parquet-go v0.25.1 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
@@ -233,16 +235,16 @@ require (
golang.org/x/exp v0.0.0-20250811191247-51f88131bc50 // indirect
golang.org/x/image v0.34.0 // indirect
golang.org/x/net v0.48.0 // indirect
golang.org/x/oauth2 v0.34.0 // indirect
golang.org/x/oauth2 v0.32.0 // indirect
golang.org/x/sync v0.19.0 // indirect
golang.org/x/sys v0.39.0 // indirect
golang.org/x/term v0.38.0 // indirect
golang.org/x/text v0.32.0 // indirect
golang.org/x/time v0.14.0 // indirect
google.golang.org/api v0.258.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect
golang.org/x/time v0.12.0 // indirect
google.golang.org/api v0.247.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect
google.golang.org/grpc/security/advancedtls v1.0.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect
google.golang.org/protobuf v1.36.10 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/validator.v2 v2.0.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
-9
View File
@@ -15,7 +15,6 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
cloud.google.com/go/auth v0.16.5 h1:mFWNQ2FEVWAliEQWpAdH80omXFokmrnbDhUS9cBywsI=
cloud.google.com/go/auth v0.16.5/go.mod h1:utzRfHMP+Vv0mpOkTRQoWD2q3BatTOoWbA7gCc2dUhQ=
cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ=
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
@@ -115,7 +114,6 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBU
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.3/go.mod h1:br7KA6edAAqDGUYJ+zVVPAyMrPhnN+zdt17yTUT6FPw=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U=
@@ -357,7 +355,6 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4=
github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo=
@@ -508,7 +505,6 @@ github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZ
github.com/panjf2000/ants/v2 v2.11.3/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
github.com/parquet-go/parquet-go v0.25.1 h1:l7jJwNM0xrk0cnIIptWMtnSnuxRkwq53S+Po3KG8Xgo=
github.com/parquet-go/parquet-go v0.25.1/go.mod h1:AXBuotO1XiBtcqJb/FKFyjBG4aqa3aQAAWF3ZPzCanY=
github.com/parquet-go/parquet-go v0.26.3/go.mod h1:h9GcSt41Knf5qXI1tp1TfR8bDBUtvdUMzSKe26aZcHk=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
@@ -829,7 +825,6 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4Iltr
golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY=
golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -937,7 +932,6 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -1011,7 +1005,6 @@ google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSr
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
google.golang.org/api v0.247.0 h1:tSd/e0QrUlLsrwMKmkbQhYVa109qIintOls2Wh6bngc=
google.golang.org/api v0.247.0/go.mod h1:r1qZOPmxXffXg6xS5uhx16Fa/UFY8QU/K4bfKrnvovM=
google.golang.org/api v0.258.0/go.mod h1:qhOMTQEZ6lUps63ZNq9jhODswwjkjYYguA7fA3TBFww=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1053,7 +1046,6 @@ google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 h1:
google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba h1:UKgtfRM7Yh93Sya0Fo8ZzhDP4qBckrrxEr2oF5UIVb8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -1084,7 +1076,6 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+5 -5
View File
@@ -152,7 +152,7 @@ func testTwoConsumersRebalance(t *testing.T, addr, topicName, groupID string) {
// Wait for rebalancing to occur - both consumers should get new assignments
var rebalancedAssignment1, rebalancedAssignment2 []int32
// Consumer1 should get a rebalance assignment
select {
case partitions := <-handler1.assignments:
@@ -372,7 +372,7 @@ func testMultipleConsumersJoin(t *testing.T, addr, topicName, groupID string) {
t.Errorf("Partition %d assigned to multiple consumers", partition)
}
}
// Each consumer should get exactly 1 partition (4 partitions / 4 consumers)
if len(assignment) != 1 {
t.Errorf("Consumer%d should get exactly 1 partition, got %d", i, len(assignment))
@@ -408,7 +408,7 @@ func (h *RebalanceTestHandler) Setup(session sarama.ConsumerGroupSession) error
h.readyOnce.Do(func() {
close(h.ready)
})
// Send partition assignment
partitions := make([]int32, 0)
for topic, partitionList := range session.Claims() {
@@ -417,13 +417,13 @@ func (h *RebalanceTestHandler) Setup(session sarama.ConsumerGroupSession) error
partitions = append(partitions, partition)
}
}
select {
case h.assignments <- partitions:
default:
// Channel might be full, that's ok
}
return nil
}
@@ -86,7 +86,7 @@ func TestSchemaEndToEnd_AvroRoundTrip(t *testing.T) {
// Verify all fields
assert.Equal(t, int32(12345), decodedMap["id"])
assert.Equal(t, "Alice Johnson", decodedMap["name"])
// Verify union fields
emailUnion, ok := decodedMap["email"].(map[string]interface{})
require.True(t, ok, "Email should be a union")
@@ -126,7 +126,7 @@ func TestSchemaEndToEnd_ProtobufRoundTrip(t *testing.T) {
require.Equal(t, uint32(2), envelope.SchemaID, "Schema ID should match")
// Note: ParseConfluentEnvelope defaults to FormatAvro; format detection requires schema registry
require.Equal(t, schema.FormatAvro, envelope.Format, "Format defaults to Avro without schema registry lookup")
// For Protobuf with indexes, we need to use the specialized parser
protobufEnvelope, ok := schema.ParseConfluentProtobufEnvelopeWithIndexCount(confluentMsg, 1)
require.True(t, ok, "Message should be a valid Protobuf envelope")
@@ -269,6 +269,7 @@ func createMockSchemaRegistryForE2E(t *testing.T) *httptest.Server {
}))
}
func getUserAvroSchemaForE2E() string {
return `{
"type": "record",
@@ -5,7 +5,7 @@ import (
)
// TestConsumerStallingPattern is a REPRODUCER for the consumer stalling bug.
//
//
// This test simulates the exact pattern that causes consumers to stall:
// 1. Consumer reads messages in batches
// 2. Consumer commits offset after each batch
@@ -24,7 +24,7 @@ import (
// If the test PASSES, it means consumer successfully fetches all messages (bug fixed)
func TestConsumerStallingPattern(t *testing.T) {
t.Skip("REPRODUCER TEST: Requires running load test infrastructure. See comments for setup.")
// This test documents the exact stalling pattern:
// - Consumers consume messages 0-163, commit offset 163
// - Next iteration: fetch offset 164+
@@ -36,7 +36,7 @@ func TestConsumerStallingPattern(t *testing.T) {
// 2. Empty fetch doesn't mean "end of partition" (could be transient)
// 3. Consumer retries on empty fetch instead of giving up
// 4. Logging shows why fetch stopped
t.Logf("=== CONSUMER STALLING REPRODUCER ===")
t.Logf("")
t.Logf("Setup Steps:")
@@ -72,27 +72,27 @@ func TestConsumerStallingPattern(t *testing.T) {
// This is a UNIT reproducer that can run standalone
func TestOffsetPlusOneCalculation(t *testing.T) {
testCases := []struct {
name string
committedOffset int64
name string
committedOffset int64
expectedNextOffset int64
}{
{"Offset 0", 0, 1},
{"Offset 99", 99, 100},
{"Offset 163", 163, 164}, // The exact stalling point!
{"Offset 163", 163, 164}, // The exact stalling point!
{"Offset 999", 999, 1000},
{"Large offset", 10000, 10001},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
// This is the critical calculation
nextOffset := tc.committedOffset + 1
if nextOffset != tc.expectedNextOffset {
t.Fatalf("OFFSET MATH BUG: committed=%d, next=%d (expected %d)",
tc.committedOffset, nextOffset, tc.expectedNextOffset)
}
t.Logf("✓ offset %d → next fetch at %d", tc.committedOffset, nextOffset)
})
}
@@ -105,18 +105,18 @@ func TestEmptyFetchShouldNotStopConsumer(t *testing.T) {
// Scenario: Consumer committed offset 163, then fetches 164+
committedOffset := int64(163)
nextFetchOffset := committedOffset + 1
// First attempt: get empty (transient - data might not be available yet)
// WRONG behavior (bug): Consumer sees 0 bytes and stops
// wrongConsumerLogic := (firstFetchResult == 0) // gives up!
// CORRECT behavior: Consumer should retry
correctConsumerLogic := true // continues retrying
correctConsumerLogic := true // continues retrying
if !correctConsumerLogic {
t.Fatalf("Consumer incorrectly gave up after empty fetch at offset %d", nextFetchOffset)
}
t.Logf("✓ Empty fetch doesn't stop consumer, continues retrying")
})
}
@@ -7,12 +7,11 @@
package pb
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
unsafe "unsafe"
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
)
const (
@@ -205,3 +205,4 @@ func TestResumeMillionRecords_Fixed(t *testing.T) {
glog.Infof("🏆 MILLION RECORD KAFKA INTEGRATION TEST COMPLETED SUCCESSFULLY!")
}
+8 -8
View File
@@ -16,7 +16,7 @@ func TestGatewayBasicFunctionality(t *testing.T) {
defer gateway.CleanupAndClose()
addr := gateway.StartAndWait()
// Give the gateway a bit more time to be fully ready
time.Sleep(200 * time.Millisecond)
@@ -32,17 +32,17 @@ func TestGatewayBasicFunctionality(t *testing.T) {
func testGatewayAcceptsConnections(t *testing.T, addr string) {
// Test basic TCP connection to gateway
t.Logf("Testing connection to gateway at %s", addr)
conn, err := net.DialTimeout("tcp", addr, 5*time.Second)
if err != nil {
t.Fatalf("Failed to connect to gateway: %v", err)
}
defer conn.Close()
// Test that we can establish a connection and the gateway is listening
// We don't need to send a full Kafka request for this basic test
t.Logf("Successfully connected to gateway at %s", addr)
// Optional: Test that we can write some data without error
testData := []byte("test")
conn.SetWriteDeadline(time.Now().Add(1 * time.Second))
@@ -57,19 +57,19 @@ func testGatewayRefusesAfterClose(t *testing.T, gateway *testutil.GatewayTestSer
// Get the address from the gateway's listener
host, port := gateway.GetListenerAddr()
addr := fmt.Sprintf("%s:%d", host, port)
// Close the gateway
gateway.CleanupAndClose()
t.Log("Testing that gateway refuses connections after close")
// Attempt to connect - should fail
conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
if err == nil {
conn.Close()
t.Fatal("Expected connection to fail after gateway close, but it succeeded")
}
// Verify it's a connection refused error
if !strings.Contains(err.Error(), "connection refused") && !strings.Contains(err.Error(), "connect: connection refused") {
t.Logf("Connection failed as expected with error: %v", err)
+6 -5
View File
@@ -63,11 +63,11 @@ const (
autoChunkSize = 8 * 1024 * 1024
// Test sizes
smallFileSize = 1 * 1024 // 1KB - single chunk
mediumFileSize = 256 * 1024 // 256KB - single chunk (at threshold)
largeFileSize = 10 * 1024 * 1024 // 10MB - triggers auto-chunking (2 chunks)
xlFileSize = 25 * 1024 * 1024 // 25MB - triggers auto-chunking (4 chunks)
multipartSize = 5 * 1024 * 1024 // 5MB per part for multipart uploads
smallFileSize = 1 * 1024 // 1KB - single chunk
mediumFileSize = 256 * 1024 // 256KB - single chunk (at threshold)
largeFileSize = 10 * 1024 * 1024 // 10MB - triggers auto-chunking (2 chunks)
xlFileSize = 25 * 1024 * 1024 // 25MB - triggers auto-chunking (4 chunks)
multipartSize = 5 * 1024 * 1024 // 5MB per part for multipart uploads
)
// ETag format patterns
@@ -540,3 +540,4 @@ func TestMultipleLargeFileUploads(t *testing.T) {
assert.NoError(t, err, "File %d ETag should be valid hex", i)
}
}
+3 -3
View File
@@ -443,7 +443,7 @@ func TestS3IAMBucketPolicyIntegration(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, testObjectData, string(data))
result.Body.Close()
// Clean up bucket policy after this test
_, err = adminClient.DeleteBucketPolicy(&s3.DeleteBucketPolicyInput{
Bucket: aws.String(bucketName),
@@ -481,7 +481,7 @@ func TestS3IAMBucketPolicyIntegration(t *testing.T) {
assert.Contains(t, *policyResult.Policy, "Deny")
// NOTE: Enforcement test is commented out due to known architectural limitation:
//
//
// KNOWN LIMITATION: DeleteObject uses the coarse-grained ACTION_WRITE constant,
// which convertActionToS3Format maps to "s3:PutObject" (not "s3:DeleteObject").
// This means the policy engine evaluates the deny policy against "s3:PutObject",
@@ -499,7 +499,7 @@ func TestS3IAMBucketPolicyIntegration(t *testing.T) {
// awsErr, ok := err.(awserr.Error)
// require.True(t, ok, "Error should be an awserr.Error")
// assert.Equal(t, "AccessDenied", awsErr.Code(), "Expected AccessDenied error code")
// Clean up bucket policy after this test
_, err = adminClient.DeleteBucketPolicy(&s3.DeleteBucketPolicyInput{
Bucket: aws.String(bucketName),
+1 -2
View File
@@ -2,13 +2,12 @@ package main
import (
"context"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/retry"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"time"
)
func main() {
+1
View File
@@ -502,3 +502,4 @@ func TestGitHub7562LargeFile(t *testing.T) {
t.Log("Large file test passed!")
}
@@ -1,274 +0,0 @@
package s3api
import (
"context"
"fmt"
"testing"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestListObjectVersionsDelimiter tests delimiter functionality in ListObjectVersions
func TestListObjectVersionsDelimiter(t *testing.T) {
client := getS3Client(t)
bucketName := getNewBucketName()
// Create bucket and enable versioning
createBucket(t, client, bucketName)
defer deleteBucket(t, client, bucketName)
enableVersioning(t, client, bucketName)
// Create test structure:
// - folder1/file1.txt
// - folder1/file2.txt
// - folder2/file3.txt
// - root-file.txt
testObjects := []string{
"folder1/file1.txt",
"folder1/file2.txt",
"folder2/file3.txt",
"root-file.txt",
}
for _, key := range testObjects {
putObject(t, client, bucketName, key, fmt.Sprintf("Content of %s", key))
}
t.Run("Delimiter groups folders correctly", func(t *testing.T) {
// List with delimiter='/' and no prefix
// Should return: root-file.txt and CommonPrefixes: folder1/, folder2/
resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
Bucket: aws.String(bucketName),
Delimiter: aws.String("/"),
})
require.NoError(t, err)
// Extract keys and prefixes
versionKeys := make([]string, 0)
for _, v := range resp.Versions {
versionKeys = append(versionKeys, *v.Key)
}
prefixValues := make([]string, 0)
for _, p := range resp.CommonPrefixes {
prefixValues = append(prefixValues, *p.Prefix)
}
// Verify results
assert.Contains(t, versionKeys, "root-file.txt", "Should include root-file.txt")
assert.Contains(t, prefixValues, "folder1/", "Should include folder1/ prefix")
assert.Contains(t, prefixValues, "folder2/", "Should include folder2/ prefix")
assert.NotContains(t, versionKeys, "folder1/file1.txt", "folder1/file1.txt should be grouped under folder1/")
assert.NotContains(t, versionKeys, "folder2/file3.txt", "folder2/file3.txt should be grouped under folder2/")
t.Logf("✓ Versions: %v", versionKeys)
t.Logf("✓ CommonPrefixes: %v", prefixValues)
})
t.Run("Prefix filtering with delimiter", func(t *testing.T) {
// List with delimiter='/' and prefix='folder1/'
// Should return: folder1/file1.txt, folder1/file2.txt
resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
Bucket: aws.String(bucketName),
Prefix: aws.String("folder1/"),
Delimiter: aws.String("/"),
})
require.NoError(t, err)
versionKeys := make([]string, 0)
for _, v := range resp.Versions {
versionKeys = append(versionKeys, *v.Key)
}
assert.Len(t, versionKeys, 2, "Should have 2 versions")
assert.Contains(t, versionKeys, "folder1/file1.txt")
assert.Contains(t, versionKeys, "folder1/file2.txt")
assert.Empty(t, resp.CommonPrefixes, "Should have no common prefixes")
t.Logf("✓ Prefix filtering works: %v", versionKeys)
})
t.Run("Without delimiter returns all versions", func(t *testing.T) {
// List without delimiter - should return all files
resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err)
assert.Len(t, resp.Versions, 4, "Should have all 4 versions")
assert.Empty(t, resp.CommonPrefixes, "Should have no common prefixes without delimiter")
t.Logf("✓ Without delimiter returns all %d versions", len(resp.Versions))
})
}
// TestListObjectVersionsDelimiterTruncation tests MaxKeys with delimiter
func TestListObjectVersionsDelimiterTruncation(t *testing.T) {
client := getS3Client(t)
bucketName := getNewBucketName()
// Create bucket and enable versioning
createBucket(t, client, bucketName)
defer deleteBucket(t, client, bucketName)
enableVersioning(t, client, bucketName)
// Create multiple folders to test truncation
for i := 0; i < 5; i++ {
key := fmt.Sprintf("folder%d/file.txt", i)
putObject(t, client, bucketName, key, fmt.Sprintf("Content %d", i))
}
// Add a root file
putObject(t, client, bucketName, "root.txt", "Root content")
t.Run("MaxKeys limits total items", func(t *testing.T) {
resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
Bucket: aws.String(bucketName),
Delimiter: aws.String("/"),
MaxKeys: aws.Int32(3),
})
require.NoError(t, err)
assert.NotNil(t, resp.IsTruncated, "IsTruncated should not be nil")
assert.True(t, *resp.IsTruncated, "Should be truncated")
assert.NotNil(t, resp.NextKeyMarker, "Should have NextKeyMarker for pagination")
count := len(resp.Versions) + len(resp.CommonPrefixes)
t.Logf("✓ MaxKeys truncation: %d items (versions: %d, prefixes: %d)",
count, len(resp.Versions), len(resp.CommonPrefixes))
})
t.Run("Pagination with delimiter", func(t *testing.T) {
// Collect all items through pagination
allKeys := make([]string, 0)
allPrefixes := make([]string, 0)
var keyMarker *string
var versionMarker *string
for {
input := &s3.ListObjectVersionsInput{
Bucket: aws.String(bucketName),
Delimiter: aws.String("/"),
MaxKeys: aws.Int32(2),
}
if keyMarker != nil {
input.KeyMarker = keyMarker
}
if versionMarker != nil {
input.VersionIdMarker = versionMarker
}
resp, err := client.ListObjectVersions(context.TODO(), input)
require.NoError(t, err)
// Collect versions
for _, v := range resp.Versions {
allKeys = append(allKeys, *v.Key)
}
// Collect prefixes
for _, p := range resp.CommonPrefixes {
allPrefixes = append(allPrefixes, *p.Prefix)
}
require.NotNil(t, resp.IsTruncated, "IsTruncated should not be nil")
if !*resp.IsTruncated {
break
}
keyMarker = resp.NextKeyMarker
versionMarker = resp.NextVersionIdMarker
}
// Should have collected all items
itemsCount := len(allKeys) + len(allPrefixes)
assert.GreaterOrEqual(t, itemsCount, 6, "Should collect all items through pagination")
t.Logf("✓ Pagination collected %d total items (keys: %d, prefixes: %d)",
itemsCount, len(allKeys), len(allPrefixes))
})
t.Run("CommonPrefixes are filtered by keyMarker (exclusive)", func(t *testing.T) {
// List with keyMarker that should skip some prefixes
// We have folder0/, folder1/, folder2/, folder3/, folder4/
// Setting keyMarker to "folder2/" should return folder3/, folder4/ and root.txt (if it's > folder2/)
resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
Bucket: aws.String(bucketName),
Delimiter: aws.String("/"),
KeyMarker: aws.String("folder2/"),
})
require.NoError(t, err)
prefixValues := make([]string, 0)
for _, p := range resp.CommonPrefixes {
prefixValues = append(prefixValues, *p.Prefix)
}
assert.NotContains(t, prefixValues, "folder0/", "Should skip folder0/")
assert.NotContains(t, prefixValues, "folder1/", "Should skip folder1/")
assert.NotContains(t, prefixValues, "folder2/", "Should skip folder2/ (exclusive marker)")
assert.Contains(t, prefixValues, "folder3/", "Should include folder3/")
assert.Contains(t, prefixValues, "folder4/", "Should include folder4/")
t.Logf("✓ CommonPrefixes filtered by keyMarker: %v", prefixValues)
})
}
// TestListObjectVersionsDelimiterWithMultipleVersions tests delimiter with multiple versions of same object
func TestListObjectVersionsDelimiterWithMultipleVersions(t *testing.T) {
client := getS3Client(t)
bucketName := getNewBucketName()
// Create bucket and enable versioning
createBucket(t, client, bucketName)
defer deleteBucket(t, client, bucketName)
enableVersioning(t, client, bucketName)
// Create multiple versions of objects in different folders
for i := 1; i <= 3; i++ {
putObject(t, client, bucketName, "folder1/file.txt", fmt.Sprintf("Version %d", i))
putObject(t, client, bucketName, "folder2/file.txt", fmt.Sprintf("Version %d", i))
}
t.Run("Delimiter groups all versions under prefix", func(t *testing.T) {
resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
Bucket: aws.String(bucketName),
Delimiter: aws.String("/"),
})
require.NoError(t, err)
// Should have no versions at root level, only common prefixes
assert.Empty(t, resp.Versions, "Should have no versions at root")
assert.Len(t, resp.CommonPrefixes, 2, "Should have 2 common prefixes")
prefixValues := make([]string, 0)
for _, p := range resp.CommonPrefixes {
prefixValues = append(prefixValues, *p.Prefix)
}
assert.Contains(t, prefixValues, "folder1/")
assert.Contains(t, prefixValues, "folder2/")
t.Logf("✓ All versions grouped under prefixes: %v", prefixValues)
})
t.Run("Listing within prefix shows all versions", func(t *testing.T) {
resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
Bucket: aws.String(bucketName),
Prefix: aws.String("folder1/"),
Delimiter: aws.String("/"),
})
require.NoError(t, err)
assert.Len(t, resp.Versions, 3, "Should have 3 versions of folder1/file.txt")
assert.Empty(t, resp.CommonPrefixes, "Should have no common prefixes")
// Verify all versions are for the same key
for _, v := range resp.Versions {
assert.Equal(t, "folder1/file.txt", *v.Key)
}
t.Logf("✓ Found %d versions within prefix", len(resp.Versions))
})
}
@@ -518,3 +518,4 @@ func TestMultipartUploadDeleteMarkerListBehavior(t *testing.T) {
t.Logf("Object restored after delete marker removal, ETag=%s", multipartETag)
}
@@ -319,3 +319,4 @@ func listAllVersions(t *testing.T, client *s3.Client, bucketName, objectKey stri
t.Logf("Total: %d versions in %d pages", len(allVersions), pageCount)
return allVersions
}
+3 -2
View File
@@ -540,11 +540,11 @@ func TestPathEdgeCases(t *testing.T) {
// Therefore, we cannot trigger the server-side path traversal block with this client.
// Instead, we verify that the file is created successfully within the jail (contained).
// The server-side protection logic is verified in unit tests (sftpd/sftp_server_test.go).
file, err := sftpClient.Create(traversalPath)
require.NoError(t, err, "creation should succeed because client sanitizes path")
file.Close()
// Clean up
err = sftpClient.Remove(traversalPath)
require.NoError(t, err)
@@ -649,3 +649,4 @@ func TestFileContent(t *testing.T) {
sftpClient.Remove(filename)
})
}
+1
View File
@@ -420,3 +420,4 @@ func findTestDataPath() string {
return "./testdata"
}
@@ -4,10 +4,6 @@ import (
"context"
"flag"
"fmt"
"strconv"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -15,6 +11,9 @@ import (
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"strconv"
"strings"
"time"
)
var (
-25
View File
@@ -1,25 +0,0 @@
# Makefile for managing local Nginx instance
# Current directory
PWD := $(shell pwd)
.PHONY: start stop reload clean
logs:
mkdir -p logs
start: logs
nginx -p $(PWD) -c nginx.conf
stop:
nginx -p $(PWD) -s stop
reload:
nginx -p $(PWD) -s reload
check:
nginx -p $(PWD) -c nginx.conf -t
clean:
rm -rf logs
-22
View File
@@ -1,22 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIDrTCCApWgAwIBAgIUHqubAnjJikbPDqGiqDT87mbcB+owDQYJKoZIhvcNAQEL
BQAwZjELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5
MRUwEwYDVQQKDAxPcmdhbml6YXRpb24xDTALBgNVBAsMBFVuaXQxEjAQBgNVBAMM
CWxvY2FsaG9zdDAeFw0yNjAxMDgwODAyMzJaFw0yNzAxMDgwODAyMzJaMGYxCzAJ
BgNVBAYTAlVTMQ4wDAYDVQQIDAVTdGF0ZTENMAsGA1UEBwwEQ2l0eTEVMBMGA1UE
CgwMT3JnYW5pemF0aW9uMQ0wCwYDVQQLDARVbml0MRIwEAYDVQQDDAlsb2NhbGhv
c3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC3gozRVkW8EFsOv75Q
AgzN0RlOkt+cyesmI71YF0NXOGlYG68Ibb4H7yemX1z663pK3e2MK0k94innW02q
0TcBvlu1Emv3Uo+iSTC6hkWR51ZkbfJDjERpO37fZhmdGexHAR2JOLGGTkTkbsRu
/lKg8HBHKcEL2NBI1D6LsOU3n+m9p0eSytzE92qY4hRtxaPgK4UUpMaEq64v8NDk
HfMFjLXmuxCSUpRTowh4MCcSS1hJCD7SJmsdXHOTQc3dgc/WJOP1rkvCZRUHVaR6
ujAIsUo9R2esevrWvxZGXFmJJDuq7u1SNnkArQludD93kovkITjb8+KsdhFGkMJR
xzdxAgMBAAGjUzBRMB0GA1UdDgQWBBSaoIBAFJGjfUpC04hJ0AlMVBuXOzAfBgNV
HSMEGDAWgBSaoIBAFJGjfUpC04hJ0AlMVBuXOzAPBgNVHRMBAf8EBTADAQH/MA0G
CSqGSIb3DQEBCwUAA4IBAQCSuLPsFF4ETql1OLr6p5FqzP6+Cg8VVkzaMJskUBA/
MEwZij+wR6teoAv8AzptByHoycOLRjYXDgweL5Z37Vr1Fo8bSqzGPqrGh6YkBNTZ
heXFf8g5lOWiv3OypxVtkEFD1tYhdU//5tnipHh08dEjZBuTumMMYhr6xcYBecI7
Lw6uo+TRrPrPLgcJU53Sivm8c2fGVexGfcklvtW4qvkQik+r9BfnoJ/f12PBq2J5
nKEQFR9EwULnkqREf0Iw/mDPS275brhdDA8JNlB49GGcRGcYu9tOVGDX5rfdIwXB
PTaWwrOoY5F4NVPlY7d5E8eLHSt8/zd+56w4g5YaQUaH
-----END CERTIFICATE-----
-11
View File
@@ -1,11 +0,0 @@
#!/bin/bash
set -e
# Generate a private key
openssl genrsa -out key.pem 2048
# Generate a self-signed certificate
openssl req -new -x509 -key key.pem -out cert.pem -days 365 \
-subj "/C=US/ST=State/L=City/O=Organization/OU=Unit/CN=localhost"
echo "Certificate (cert.pem) and Key (key.pem) generated successfully in $(pwd)"
-28
View File
@@ -1,28 +0,0 @@
-----BEGIN PRIVATE KEY-----
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC3gozRVkW8EFsO
v75QAgzN0RlOkt+cyesmI71YF0NXOGlYG68Ibb4H7yemX1z663pK3e2MK0k94inn
W02q0TcBvlu1Emv3Uo+iSTC6hkWR51ZkbfJDjERpO37fZhmdGexHAR2JOLGGTkTk
bsRu/lKg8HBHKcEL2NBI1D6LsOU3n+m9p0eSytzE92qY4hRtxaPgK4UUpMaEq64v
8NDkHfMFjLXmuxCSUpRTowh4MCcSS1hJCD7SJmsdXHOTQc3dgc/WJOP1rkvCZRUH
VaR6ujAIsUo9R2esevrWvxZGXFmJJDuq7u1SNnkArQludD93kovkITjb8+KsdhFG
kMJRxzdxAgMBAAECggEAUUf932r7EEWeT3YWcZG24iLPtsHPUb8N7KK/5xjGtwdB
gckhSfZxyUngogxzPqC2VhrecMtrcYGPVe5XBGYOmzMZo6Y5NB7Zh+QrNTLr9agl
1+3q9IhYdNPpakHq7m7g/MI9tt3po7dfRF7bmt4C804MLTkNTJYIC3oXCgCcRnpJ
NYLtombx6FNzivUMB9/FGoR3MeVsenCNmbfGyqssKafsoNfBnszg2XT4W9TpynA5
ThER0ZjAu8Wi4QJ4rGuwgCqoobe9qw5xUdnerj84O/FDxY38zK92yZzxPG39O61U
kSna0WKAhtvOqYvYe5KvIFs8AbFNBn+/mCEhaPmYEQKBgQDvXjy7maM0L8yDYNs5
2Mups0DC6E3Xkx5k6O6/TPfBBY949ZBsAT+hDpJA119/UOxGRE/wZaQ+09blyjOD
B8Nw0m0OOgpERIHhm+uDXpkVhFJXkBQMB7Jsc2tZKxHNY78UD4K4YcsbRGBNZryf
6Fd6uVad0qMKJF825gN3f7d5twKBgQDEQrxBEensG6D5XWoYTqwvNfgc9BqOAWrJ
IkX4l99n9yJUHJ+h7wcrzA1FfKVJU5LkvR+YqCxcSyil5BnQFU7s3JqnhIkwfk9R
wS2pPXLaJS6oQQcr2yRQgP8xDDMlBNABIkw/JARXsPkdxM+y93G4s+O4GIsrOJHX
zZWjoK74FwKBgDsKjoZjZfsIUzSQnmJNVCbefPnWXOTwd83CGql67Z1hprraLy5E
vJIYSwDNXtPDfRPz45fTKkkI7XeugTgvoQhiDt40eLXQBA1G7TdxDOdexQSuGBPD
jkHr0i2jTvfPQ6FKFh8ul+941RwiXCJBJ9qwfmndO7PTttyOrYo7KWFPAoGBAJHb
jxFFAPaKag4pkKnT/j9v+p96Ayo6f2RP5nSkWlJn5slgJHsyq7+u/MYBps1iIS+C
JrNLePx2lxUC+vQcgfWi1gVdiJ/Tn5uA2DCJDIx3Sb+d5HiBevGhjQakrHX8p6HQ
HZ5nZBrP2bR4CmxVXd0JRFoTU4qhljIuBeH/j0mtAoGAN9Hf3txLTm7PbuNy0xOM
Us+nHsVjIzZvZ6H413kzs85QIf22lw0JEoRBnhtZmxATERrvYJ9YtyZH0Tf+b+eU
TioM9wDi62tDjgMK+jgoIPid4hgMdHzdjjxJ6kZ9sFjR/3dOKxDwe8jU1YybFqxH
yYRlpVwN2c++XrbOPfE+m7E=
-----END PRIVATE KEY-----
-85
View File
@@ -1,85 +0,0 @@
types {
text/html html htm shtml;
text/css css;
text/xml xml;
image/gif gif;
image/jpeg jpeg jpg;
application/javascript js;
application/atom+xml atom;
application/rss+xml rss;
text/mathml mml;
text/plain txt;
text/vnd.sun.j2me.app-descriptor jad;
text/vnd.wap.wml wml;
text/x-component htc;
image/png png;
image/tiff tif tiff;
image/vnd.wap.wbmp wbmp;
image/x-icon ico;
image/x-jng jng;
image/x-ms-bmp bmp;
image/svg+xml svg svgz;
image/webp webp;
application/font-woff woff;
application/java-archive jar war ear;
application/json json;
application/mac-binhex40 hqx;
application/msword doc;
application/pdf pdf;
application/postscript ps eps ai;
application/rtf rtf;
application/vnd.apple.mpegurl m3u8;
application/vnd.ms-excel xls;
application/vnd.ms-fontobject eot;
application/vnd.ms-powerpoint ppt;
application/vnd.wap.wmlc wmlc;
application/vnd.google-earth.kml+xml kml;
application/vnd.google-earth.kmz kmz;
application/x-7z-compressed 7z;
application/x-cocoa cco;
application/x-java-archive-diff jardiff;
application/x-java-jnlp-file jnlp;
application/x-makeself run;
application/x-perl pl pm;
application/x-pilot prc pdb;
application/x-rar-compressed rar;
application/x-redhat-package-manager rpm;
application/x-sea sea;
application/x-shockwave-flash swf;
application/x-stuffit sit;
application/x-tcl tcl tk;
application/x-x509-ca-cert der pem crt;
application/x-xpinstall xpi;
application/xhtml+xml xhtml;
application/xspf+xml xspf;
application/zip zip;
application/octet-stream bin exe dll;
application/octet-stream deb;
application/octet-stream dmg;
application/octet-stream iso img;
application/octet-stream msi msp msm;
audio/midi mid midi kar;
audio/mpeg mp3;
audio/ogg ogg;
audio/x-m4a m4a;
audio/x-realaudio ra;
video/3gpp 3gpp 3gp;
video/mp2t ts;
video/mp4 mp4;
video/mpeg mpeg mpg;
video/quicktime mov;
video/webm webm;
video/x-flv flv;
video/x-m4v m4v;
video/x-mng mng;
video/x-ms-asf asf x-ms-asf;
video/x-ms-wmv wmv;
video/x-msvideo avi;
}
-55
View File
@@ -1,55 +0,0 @@
worker_processes 1;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
sendfile on;
keepalive_timeout 65;
# Standard Configuration from Wiki
upstream seaweedfs {
server 127.0.0.1:8334;
keepalive 20;
}
server {
listen 8333 ssl;
server_name localhost;
ssl_certificate cert.pem;
ssl_certificate_key key.pem;
ssl_session_cache shared:SSL:1m;
ssl_session_timeout 5m;
ssl_ciphers HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
ignore_invalid_headers off;
client_max_body_size 0;
proxy_buffering off;
location / {
proxy_pass http://seaweedfs;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Port $server_port;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_connect_timeout 300;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_request_buffering off;
chunked_transfer_encoding off;
}
}
}
@@ -4,16 +4,17 @@ import (
"context"
"flag"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/pb"
"log"
"math/rand"
"time"
"google.golang.org/grpc"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
"google.golang.org/grpc"
)
var (
@@ -4,14 +4,13 @@ import (
"crypto/md5"
"encoding/base64"
"fmt"
"net/http"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
"net/http"
"strings"
"time"
)
// Downloads an item from an S3 Bucket in the region configured in the shared config
@@ -4,6 +4,7 @@ import (
"bytes"
"flag"
"fmt"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
"io"
"log"
"math/rand"
@@ -13,8 +14,6 @@ import (
"strings"
"sync"
"time"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)
var (
@@ -4,6 +4,7 @@ import (
"bytes"
"flag"
"fmt"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
"io"
"log"
"math/rand"
@@ -14,8 +15,6 @@ import (
"strings"
"sync"
"time"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)
var (
+1 -1
View File
@@ -3,11 +3,11 @@ package main
import (
"context"
"flag"
"github.com/seaweedfs/seaweedfs/weed/pb"
"log"
"time"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
util2 "github.com/seaweedfs/seaweedfs/weed/util"
+6 -10
View File
@@ -37,7 +37,6 @@ type ObjectStoreUser struct {
AccessKey string `json:"access_key"`
SecretKey string `json:"secret_key"`
Permissions []string `json:"permissions"`
PolicyNames []string `json:"policy_names"`
}
type ObjectStoreUsersData struct {
@@ -53,13 +52,11 @@ type CreateUserRequest struct {
Email string `json:"email"`
Actions []string `json:"actions"`
GenerateKey bool `json:"generate_key"`
PolicyNames []string `json:"policy_names"`
}
type UpdateUserRequest struct {
Email string `json:"email"`
Actions []string `json:"actions"`
PolicyNames []string `json:"policy_names"`
Email string `json:"email"`
Actions []string `json:"actions"`
}
type UpdateUserPoliciesRequest struct {
@@ -73,11 +70,10 @@ type AccessKeyInfo struct {
}
type UserDetails struct {
Username string `json:"username"`
Email string `json:"email"`
Actions []string `json:"actions"`
PolicyNames []string `json:"policy_names"`
AccessKeys []AccessKeyInfo `json:"access_keys"`
Username string `json:"username"`
Email string `json:"email"`
Actions []string `json:"actions"`
AccessKeys []AccessKeyInfo `json:"access_keys"`
}
type FilerNode struct {
+162 -119
View File
@@ -23,7 +23,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"google.golang.org/grpc"
@@ -97,11 +96,6 @@ type AdminServer struct {
// Worker gRPC server
workerGrpcServer *WorkerGrpcServer
// Collection statistics caching
collectionStatsCache map[string]collectionStats
lastCollectionStatsUpdate time.Time
collectionStatsCacheThreshold time.Duration
}
// Type definitions moved to types.go
@@ -125,14 +119,13 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string)
go masterClient.KeepConnectedToMaster(ctx)
server := &AdminServer{
masterClient: masterClient,
templateFS: templateFS,
dataDir: dataDir,
grpcDialOption: grpcDialOption,
cacheExpiration: 10 * time.Second,
filerCacheExpiration: 30 * time.Second, // Cache filers for 30 seconds
configPersistence: NewConfigPersistence(dataDir),
collectionStatsCacheThreshold: 30 * time.Second,
masterClient: masterClient,
templateFS: templateFS,
dataDir: dataDir,
grpcDialOption: grpcDialOption,
cacheExpiration: 10 * time.Second,
filerCacheExpiration: 30 * time.Second, // Cache filers for 30 seconds
configPersistence: NewConfigPersistence(dataDir),
}
// Initialize topic retention purger
@@ -243,32 +236,57 @@ func (s *AdminServer) GetCredentialManager() *credential.CredentialManager {
// InvalidateCache method moved to cluster_topology.go
// GetS3BucketsData retrieves all Object Store buckets and aggregates total storage metrics
func (s *AdminServer) GetS3BucketsData() (S3BucketsData, error) {
buckets, err := s.GetS3Buckets()
if err != nil {
return S3BucketsData{}, err
}
var totalSize int64
for _, bucket := range buckets {
totalSize += bucket.PhysicalSize
}
return S3BucketsData{
Buckets: buckets,
TotalBuckets: len(buckets),
TotalSize: totalSize,
LastUpdated: time.Now(),
}, nil
}
// GetS3Buckets retrieves all Object Store buckets from the filer and collects size/object data from collections
func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
var buckets []S3Bucket
// Collect volume information by collection with caching
collectionMap, _ := s.getCollectionStats()
// Build a map of collection name to collection data
collectionMap := make(map[string]struct {
Size int64
FileCount int64
})
// Collect volume information by collection
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := client.VolumeList(context.Background(), &master_pb.VolumeListRequest{})
if err != nil {
return err
}
if resp.TopologyInfo != nil {
for _, dc := range resp.TopologyInfo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for _, node := range rack.DataNodeInfos {
for _, diskInfo := range node.DiskInfos {
for _, volInfo := range diskInfo.VolumeInfos {
collection := volInfo.Collection
if collection == "" {
collection = "default"
}
if _, exists := collectionMap[collection]; !exists {
collectionMap[collection] = struct {
Size int64
FileCount int64
}{}
}
data := collectionMap[collection]
data.Size += int64(volInfo.Size)
data.FileCount += int64(volInfo.FileCount)
collectionMap[collection] = data
}
}
}
}
}
}
return nil
})
if err != nil {
return nil, fmt.Errorf("failed to get volume information: %w", err)
}
// Get filer configuration (buckets path and filer group)
filerConfig, err := s.getFilerConfig()
@@ -306,12 +324,10 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
collectionName := getCollectionName(filerConfig.FilerGroup, bucketName)
// Get size and object count from collection data
var physicalSize int64
var logicalSize int64
var size int64
var objectCount int64
if collectionData, exists := collectionMap[collectionName]; exists {
physicalSize = collectionData.PhysicalSize
logicalSize = collectionData.LogicalSize
size = collectionData.Size
objectCount = collectionData.FileCount
}
@@ -347,8 +363,7 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
bucket := S3Bucket{
Name: bucketName,
CreatedAt: time.Unix(resp.Entry.Attributes.Crtime, 0),
LogicalSize: logicalSize,
PhysicalSize: physicalSize,
Size: size,
ObjectCount: objectCount,
LastModified: time.Unix(resp.Entry.Attributes.Mtime, 0),
Quota: quota,
@@ -374,7 +389,6 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
}
// GetBucketDetails retrieves detailed information about a specific bucket
// Note: This no longer lists objects for performance reasons. Use GetS3Buckets for size/count data.
func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error) {
// Get filer configuration (buckets path)
filerConfig, err := s.getFilerConfig()
@@ -382,25 +396,16 @@ func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error
glog.Warningf("Failed to get filer configuration, using defaults: %v", err)
}
bucketPath := fmt.Sprintf("%s/%s", filerConfig.BucketsPath, bucketName)
details := &BucketDetails{
Bucket: S3Bucket{
Name: bucketName,
},
Objects: []S3Object{},
UpdatedAt: time.Now(),
}
// Get collection data for size and object count with caching
collectionName := getCollectionName(filerConfig.FilerGroup, bucketName)
stats, err := s.getCollectionStats()
if err != nil {
glog.Warningf("Failed to get collection data: %v", err)
// Continue without collection data - use zero values
} else if data, ok := stats[collectionName]; ok {
details.Bucket.LogicalSize = data.LogicalSize
details.Bucket.PhysicalSize = data.PhysicalSize
details.Bucket.ObjectCount = data.FileCount
}
err = s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
// Get bucket info
bucketResp, err := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
@@ -451,7 +456,8 @@ func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error
details.Bucket.ObjectLockDuration = objectLockDuration
details.Bucket.Owner = owner
return nil
// List objects in bucket (recursively)
return s.listBucketObjects(client, bucketPath, bucketPath, "", details)
})
if err != nil {
@@ -461,6 +467,106 @@ func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error
return details, nil
}
// listBucketObjects recursively lists all objects in a bucket
// bucketBasePath is the full path to the bucket (e.g., /buckets/mybucket)
func (s *AdminServer) listBucketObjects(client filer_pb.SeaweedFilerClient, bucketBasePath, directory, prefix string, details *BucketDetails) error {
stream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
Directory: directory,
Prefix: prefix,
StartFromFileName: "",
InclusiveStartFrom: false,
Limit: 1000,
})
if err != nil {
return err
}
for {
resp, err := stream.Recv()
if err != nil {
if err.Error() == "EOF" {
break
}
return err
}
entry := resp.Entry
if entry.IsDirectory {
// Check if this is a .versions directory (represents a versioned object)
if strings.HasSuffix(entry.Name, ".versions") {
// This directory represents an object, add it as an object without the .versions suffix
objectName := strings.TrimSuffix(entry.Name, ".versions")
objectKey := objectName
if directory != bucketBasePath {
relativePath := directory[len(bucketBasePath)+1:]
objectKey = fmt.Sprintf("%s/%s", relativePath, objectName)
}
// Extract latest version metadata from extended attributes
var size int64 = 0
var mtime int64 = entry.Attributes.Mtime
if entry.Extended != nil {
// Get size of latest version
if sizeBytes, ok := entry.Extended[s3_constants.ExtLatestVersionSizeKey]; ok && len(sizeBytes) == 8 {
size = int64(util.BytesToUint64(sizeBytes))
}
// Get mtime of latest version
if mtimeBytes, ok := entry.Extended[s3_constants.ExtLatestVersionMtimeKey]; ok && len(mtimeBytes) == 8 {
mtime = int64(util.BytesToUint64(mtimeBytes))
}
}
obj := S3Object{
Key: objectKey,
Size: size,
LastModified: time.Unix(mtime, 0),
ETag: "",
StorageClass: "STANDARD",
}
details.Objects = append(details.Objects, obj)
details.TotalCount++
details.TotalSize += size
// Don't recurse into .versions directories
continue
}
// Recursively list subdirectories
subDir := fmt.Sprintf("%s/%s", directory, entry.Name)
err := s.listBucketObjects(client, bucketBasePath, subDir, "", details)
if err != nil {
return err
}
} else {
// Add file object
objectKey := entry.Name
if directory != bucketBasePath {
// Remove bucket prefix to get relative path
relativePath := directory[len(bucketBasePath)+1:]
objectKey = fmt.Sprintf("%s/%s", relativePath, entry.Name)
}
obj := S3Object{
Key: objectKey,
Size: int64(entry.Attributes.FileSize),
LastModified: time.Unix(entry.Attributes.Mtime, 0),
ETag: "", // Could be calculated from chunks if needed
StorageClass: "STANDARD",
}
details.Objects = append(details.Objects, obj)
details.TotalSize += obj.Size
details.TotalCount++
}
}
// Update bucket totals
details.Bucket.Size = details.TotalSize
details.Bucket.ObjectCount = details.TotalCount
return nil
}
// CreateS3Bucket creates a new S3 bucket
func (s *AdminServer) CreateS3Bucket(bucketName string) error {
return s.CreateS3BucketWithQuota(bucketName, 0, false)
@@ -2002,66 +2108,3 @@ func getBoolFromMap(m map[string]interface{}, key string) bool {
}
return false
}
type collectionStats struct {
PhysicalSize int64
LogicalSize int64
FileCount int64
}
func collectCollectionStats(topologyInfo *master_pb.TopologyInfo) map[string]collectionStats {
collectionMap := make(map[string]collectionStats)
for _, dc := range topologyInfo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for _, node := range rack.DataNodeInfos {
for _, diskInfo := range node.DiskInfos {
for _, volInfo := range diskInfo.VolumeInfos {
collection := volInfo.Collection
if collection == "" {
collection = "default"
}
data := collectionMap[collection]
data.PhysicalSize += int64(volInfo.Size)
rp, _ := super_block.NewReplicaPlacementFromByte(byte(volInfo.ReplicaPlacement))
// NewReplicaPlacementFromByte never returns a nil rp. If there's an error,
// it returns a zero-valued ReplicaPlacement, for which GetCopyCount() is 1.
// This provides a safe fallback, so we can ignore the error.
replicaCount := int64(rp.GetCopyCount())
if volInfo.Size >= volInfo.DeletedByteCount {
data.LogicalSize += int64(volInfo.Size-volInfo.DeletedByteCount) / replicaCount
}
if volInfo.FileCount >= volInfo.DeleteCount {
data.FileCount += int64(volInfo.FileCount-volInfo.DeleteCount) / replicaCount
}
collectionMap[collection] = data
}
}
}
}
}
return collectionMap
}
// getCollectionStats returns current collection statistics with caching
func (s *AdminServer) getCollectionStats() (map[string]collectionStats, error) {
now := time.Now()
if s.collectionStatsCache != nil && now.Sub(s.lastCollectionStatsUpdate) < s.collectionStatsCacheThreshold {
return s.collectionStatsCache, nil
}
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := client.VolumeList(context.Background(), &master_pb.VolumeListRequest{})
if err != nil {
return err
}
if resp.TopologyInfo != nil {
s.collectionStatsCache = collectCollectionStats(resp.TopologyInfo)
s.lastCollectionStatsUpdate = now
}
return nil
})
return s.collectionStatsCache, err
}
+15 -2
View File
@@ -48,13 +48,26 @@ type CreateBucketRequest struct {
func (s *AdminServer) ShowS3Buckets(c *gin.Context) {
username := c.GetString("username")
data, err := s.GetS3BucketsData()
buckets, err := s.GetS3Buckets()
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get Object Store buckets: " + err.Error()})
return
}
data.Username = username
// Calculate totals
var totalSize int64
for _, bucket := range buckets {
totalSize += bucket.Size
}
data := S3BucketsData{
Username: username,
Buckets: buckets,
TotalBuckets: len(buckets),
TotalSize: totalSize,
LastUpdated: time.Now(),
}
c.JSON(http.StatusOK, data)
}
+2 -4
View File
@@ -120,10 +120,8 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error {
// InvalidateCache forces a refresh of cached data
func (s *AdminServer) InvalidateCache() {
s.lastCacheUpdate = time.Now().Add(-s.cacheExpiration)
s.lastCacheUpdate = time.Time{}
s.cachedTopology = nil
s.lastFilerUpdate = time.Now().Add(-s.filerCacheExpiration)
s.lastFilerUpdate = time.Time{}
s.cachedFilers = nil
s.lastCollectionStatsUpdate = time.Now().Add(-s.collectionStatsCacheThreshold)
s.collectionStatsCache = nil
}
+6 -4
View File
@@ -73,8 +73,7 @@ type VolumeServerEcInfo struct {
type S3Bucket struct {
Name string `json:"name"`
CreatedAt time.Time `json:"created_at"`
LogicalSize int64 `json:"logical_size"` // Actual data size (used space)
PhysicalSize int64 `json:"physical_size"` // Total allocated volume space
Size int64 `json:"size"`
ObjectCount int64 `json:"object_count"`
LastModified time.Time `json:"last_modified"`
Quota int64 `json:"quota"` // Quota in bytes, 0 means no quota
@@ -95,8 +94,11 @@ type S3Object struct {
}
type BucketDetails struct {
Bucket S3Bucket `json:"bucket"`
UpdatedAt time.Time `json:"updated_at"`
Bucket S3Bucket `json:"bucket"`
Objects []S3Object `json:"objects"`
TotalSize int64 `json:"total_size"`
TotalCount int64 `json:"total_count"`
UpdatedAt time.Time `json:"updated_at"`
}
// ObjectStoreUser is defined in admin_data.go
+5 -15
View File
@@ -21,9 +21,8 @@ func (s *AdminServer) CreateObjectStoreUser(req CreateUserRequest) (*ObjectStore
// Create new identity
newIdentity := &iam_pb.Identity{
Name: req.Username,
Actions: req.Actions,
PolicyNames: req.PolicyNames,
Name: req.Username,
Actions: req.Actions,
}
// Add account if email is provided
@@ -64,7 +63,6 @@ func (s *AdminServer) CreateObjectStoreUser(req CreateUserRequest) (*ObjectStore
AccessKey: accessKey,
SecretKey: secretKey,
Permissions: req.Actions,
PolicyNames: req.PolicyNames,
}
return user, nil
@@ -93,17 +91,12 @@ func (s *AdminServer) UpdateObjectStoreUser(username string, req UpdateUserReque
Account: identity.Account,
Credentials: identity.Credentials,
Actions: identity.Actions,
PolicyNames: identity.PolicyNames,
}
// Update actions if provided
if req.Actions != nil {
if len(req.Actions) > 0 {
updatedIdentity.Actions = req.Actions
}
// Always update policy names when present in request (even if empty to allow clearing)
if req.PolicyNames != nil {
updatedIdentity.PolicyNames = req.PolicyNames
}
// Update email if provided
if req.Email != "" {
@@ -127,7 +120,6 @@ func (s *AdminServer) UpdateObjectStoreUser(username string, req UpdateUserReque
Username: username,
Email: req.Email,
Permissions: updatedIdentity.Actions,
PolicyNames: updatedIdentity.PolicyNames,
}
// Get first access key for display
@@ -177,9 +169,8 @@ func (s *AdminServer) GetObjectStoreUserDetails(username string) (*UserDetails,
}
details := &UserDetails{
Username: username,
Actions: identity.Actions,
PolicyNames: identity.PolicyNames,
Username: username,
Actions: identity.Actions,
}
// Set email from account if available
@@ -304,7 +295,6 @@ func (s *AdminServer) UpdateUserPolicies(username string, actions []string) erro
Account: identity.Account,
Credentials: identity.Credentials,
Actions: actions,
PolicyNames: identity.PolicyNames,
}
// Update user using credential manager
+17 -27
View File
@@ -225,11 +225,11 @@ func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStr
select {
case <-ctx.Done():
glog.Infof("Worker %s connection closed: %v", workerID, ctx.Err())
s.unregisterWorker(conn)
s.unregisterWorker(workerID)
return nil
case <-connCtx.Done():
glog.Infof("Worker %s connection cancelled", workerID)
s.unregisterWorker(conn)
s.unregisterWorker(workerID)
return nil
default:
}
@@ -241,7 +241,7 @@ func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStr
} else {
glog.Errorf("Error receiving from worker %s: %v", workerID, err)
}
s.unregisterWorker(conn)
s.unregisterWorker(workerID)
return err
}
@@ -294,7 +294,7 @@ func (s *WorkerGrpcServer) handleWorkerMessage(conn *WorkerConnection, msg *work
case *worker_pb.WorkerMessage_Shutdown:
glog.Infof("Worker %s shutting down: %s", workerID, m.Shutdown.Reason)
s.unregisterWorker(conn)
s.unregisterWorker(workerID)
default:
glog.Warningf("Unknown message type from worker %s", workerID)
@@ -463,24 +463,17 @@ func (s *WorkerGrpcServer) safeCloseOutgoingChannel(conn *WorkerConnection, sour
}
// unregisterWorker removes a worker connection
func (s *WorkerGrpcServer) unregisterWorker(conn *WorkerConnection) {
func (s *WorkerGrpcServer) unregisterWorker(workerID string) {
s.connMutex.Lock()
existingConn, exists := s.connections[conn.workerID]
conn, exists := s.connections[workerID]
if !exists {
s.connMutex.Unlock()
glog.V(2).Infof("unregisterWorker: worker %s not found in connections map (already unregistered)", conn.workerID)
return
}
// Only remove if it matches the specific connection instance
if existingConn != conn {
s.connMutex.Unlock()
glog.V(1).Infof("unregisterWorker: worker %s connection replaced, skipping unregister for old connection", conn.workerID)
glog.V(2).Infof("unregisterWorker: worker %s not found in connections map (already unregistered)", workerID)
return
}
// Remove from map first to prevent duplicate cleanup attempts
delete(s.connections, conn.workerID)
delete(s.connections, workerID)
s.connMutex.Unlock()
// Cancel context to signal goroutines to stop
@@ -489,7 +482,7 @@ func (s *WorkerGrpcServer) unregisterWorker(conn *WorkerConnection) {
// Safely close the outgoing channel with recover to handle potential double-close
s.safeCloseOutgoingChannel(conn, "unregisterWorker")
glog.V(1).Infof("Unregistered worker %s", conn.workerID)
glog.V(1).Infof("Unregistered worker %s", workerID)
}
// cleanupRoutine periodically cleans up stale connections
@@ -512,18 +505,15 @@ func (s *WorkerGrpcServer) cleanupStaleConnections() {
cutoff := time.Now().Add(-2 * time.Minute)
s.connMutex.Lock()
// collect connections to remove first to avoid deadlock if unregisterWorker locks
var toRemove []*WorkerConnection
for _, conn := range s.connections {
if conn.lastSeen.Before(cutoff) {
toRemove = append(toRemove, conn)
}
}
s.connMutex.Unlock()
defer s.connMutex.Unlock()
for _, conn := range toRemove {
glog.Warningf("Cleaning up stale worker connection: %s", conn.workerID)
s.unregisterWorker(conn)
for workerID, conn := range s.connections {
if conn.lastSeen.Before(cutoff) {
glog.Warningf("Cleaning up stale worker connection: %s", workerID)
conn.cancel()
s.safeCloseOutgoingChannel(conn, "cleanupStaleConnections")
delete(s.connections, workerID)
}
}
}
+16 -10
View File
@@ -49,7 +49,7 @@ func NewAdminHandlers(adminServer *dash.AdminServer) *AdminHandlers {
}
// SetupRoutes configures all the routes for the admin interface
func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser, adminPassword, readOnlyUser, readOnlyPassword string, enableUI bool) {
func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser, adminPassword, readOnlyUser, readOnlyPassword string) {
// Health check (no auth required)
r.GET("/health", h.HealthCheck)
@@ -61,11 +61,6 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser,
c.Redirect(http.StatusMovedPermanently, "/static/favicon.ico")
})
// Skip UI routes if UI is not enabled
if !enableUI {
return
}
if authRequired {
// Authentication routes (no auth required)
r.GET("/login", h.authHandlers.ShowLogin)
@@ -414,8 +409,8 @@ func (h *AdminHandlers) getS3BucketsData(c *gin.Context) dash.S3BucketsData {
username = "admin"
}
// Get Object Store buckets data
data, err := h.adminServer.GetS3BucketsData()
// Get Object Store buckets
buckets, err := h.adminServer.GetS3Buckets()
if err != nil {
// Return empty data on error
return dash.S3BucketsData{
@@ -427,8 +422,19 @@ func (h *AdminHandlers) getS3BucketsData(c *gin.Context) dash.S3BucketsData {
}
}
data.Username = username
return data
// Calculate totals
var totalSize int64
for _, bucket := range buckets {
totalSize += bucket.Size
}
return dash.S3BucketsData{
Username: username,
Buckets: buckets,
TotalBuckets: len(buckets),
TotalSize: totalSize,
LastUpdated: time.Now(),
}
}
// getAdminData retrieves admin data from the server (now uses consolidated method)
-6
View File
@@ -1,7 +1,6 @@
package handlers
import (
"fmt"
"net/http"
"time"
@@ -30,12 +29,7 @@ func (h *UserHandlers) ShowObjectStoreUsers(c *gin.Context) {
usersData := h.getObjectStoreUsersData(c)
// Render HTML template
// Add cache-control headers to prevent browser caching of inline JavaScript
c.Header("Content-Type", "text/html")
c.Header("Cache-Control", "no-cache, no-store, must-revalidate")
c.Header("Pragma", "no-cache")
c.Header("Expires", "0")
c.Header("ETag", fmt.Sprintf("\"%d\"", time.Now().Unix()))
usersComponent := app.ObjectStoreUsers(usersData)
layoutComponent := layout.Layout(c, usersComponent)
err := layoutComponent.Render(c.Request.Context(), c.Writer)
File diff suppressed because it is too large Load Diff
+41 -98
View File
@@ -223,13 +223,6 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
</select>
<small class="form-text text-muted">Hold Ctrl/Cmd to select multiple permissions</small>
</div>
<div class="mb-3">
<label for="policies" class="form-label">Attached Policies</label>
<select multiple class="form-control" id="policies" name="policies" size="5">
<!-- Options loaded dynamically -->
</select>
<small class="form-text text-muted">Hold Ctrl/Cmd to select multiple policies</small>
</div>
<div class="mb-3 form-check">
<input type="checkbox" class="form-check-input" id="generateKey" name="generateKey" checked>
<label class="form-check-label" for="generateKey">
@@ -282,12 +275,6 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
</optgroup>
</select>
</div>
<div class="mb-3">
<label for="editPolicies" class="form-label">Attached Policies</label>
<select multiple class="form-control" id="editPolicies" name="policies" size="5">
<!-- Options loaded dynamically -->
</select>
</div>
</form>
</div>
<div class="modal-footer">
@@ -349,7 +336,6 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
<!-- JavaScript for user management -->
<script>
document.addEventListener('DOMContentLoaded', function() {
// Event delegation for user action buttons
document.addEventListener('click', function(e) {
const button = e.target.closest('[data-action]');
@@ -373,67 +359,8 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
break;
}
});
// Event delegation for delete access key buttons
document.addEventListener('click', function(e) {
const deleteBtn = e.target.closest('.delete-access-key-btn');
if (deleteBtn) {
const username = deleteBtn.getAttribute('data-username');
const accessKey = deleteBtn.getAttribute('data-access-key');
deleteAccessKey(username, accessKey);
}
});
// Load policies for dropdowns
loadPolicies();
});
// Load policies
async function loadPolicies() {
try {
const response = await fetch('/api/object-store/policies');
if (response.ok) {
const data = await response.json();
const policies = data.policies || [];
const createSelect = document.getElementById('policies');
const editSelect = document.getElementById('editPolicies');
// Check if elements exist
if (!createSelect || !editSelect) {
console.warn('Policy select elements not found');
return;
}
// Clear existing options
createSelect.innerHTML = '';
editSelect.innerHTML = '';
if (policies && policies.length > 0) {
policies.forEach(policy => {
const option = document.createElement('option');
option.value = policy.name;
option.textContent = policy.name;
createSelect.appendChild(option);
editSelect.appendChild(option.cloneNode(true));
});
} else {
const emptyOption = document.createElement('option');
emptyOption.disabled = true;
emptyOption.textContent = 'No policies available';
createSelect.appendChild(emptyOption);
editSelect.appendChild(emptyOption.cloneNode(true));
}
} else {
const error = await response.json().catch(() => ({}));
showErrorMessage('Failed to load policies: ' + (error.error || 'Unknown error'));
}
} catch (error) {
console.error('Error loading policies:', error);
showErrorMessage('Failed to load policies: ' + error.message);
}
}
// Show user details modal
async function showUserDetails(username) {
try {
@@ -468,14 +395,6 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
Array.from(actionsSelect.options).forEach(option => {
option.selected = user.actions && user.actions.includes(option.value);
});
// Set selected policies
const policiesSelect = document.getElementById('editPolicies');
if (policiesSelect) {
Array.from(policiesSelect.options).forEach(option => {
option.selected = user.policy_names && user.policy_names.includes(option.value);
});
}
// Show modal
const modal = new bootstrap.Modal(document.getElementById('editUserModal'));
@@ -539,7 +458,6 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
username: formData.get('username'),
email: formData.get('email'),
actions: Array.from(document.getElementById('actions').selectedOptions).map(option => option.value),
policy_names: Array.from(document.getElementById('policies').selectedOptions).map(option => option.value),
generate_key: document.getElementById('generateKey').checked
};
@@ -576,6 +494,41 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
}
}
// Handle update user form submission
async function handleUpdateUser() {
const username = document.getElementById('editUsername').value;
const formData = new FormData(document.getElementById('editUserForm'));
const userData = {
email: formData.get('email'),
actions: Array.from(document.getElementById('editActions').selectedOptions).map(option => option.value)
};
try {
const response = await fetch(`/api/users/${username}`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(userData)
});
if (response.ok) {
showSuccessMessage('User updated successfully');
// Close modal and refresh page
const modal = bootstrap.Modal.getInstance(document.getElementById('editUserModal'));
modal.hide();
setTimeout(() => window.location.reload(), 1000);
} else {
const error = await response.json();
showErrorMessage('Failed to update user: ' + (error.error || 'Unknown error'));
}
} catch (error) {
console.error('Error updating user:', error);
showErrorMessage('Failed to update user: ' + error.message);
}
}
// Create user details content
function createUserDetailsContent(user) {
@@ -592,27 +545,17 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
detailsHtml += '<div class="mb-3">';
if (user.actions && user.actions.length > 0) {
detailsHtml += user.actions.map(function(action) {
return '<span class="badge bg-info me-1">' + escapeHtml(action) + '</span>';
return '<span class="badge bg-info me-1">' + action + '</span>';
}).join('');
} else {
detailsHtml += '<span class="text-muted">No permissions assigned</span>';
}
detailsHtml += '</div>';
detailsHtml += '<h6 class="text-muted">Attached Policy Names</h6>';
detailsHtml += '<div class="mb-3">';
if (user.policy_names && user.policy_names.length > 0) {
detailsHtml += user.policy_names.map(function(policy) {
return '<span class="badge bg-secondary me-1">' + escapeHtml(policy) + '</span>';
}).join('');
} else {
detailsHtml += '<span class="text-muted">No policies attached</span>';
}
detailsHtml += '</div>';
detailsHtml += '<h6 class="text-muted">Access Keys</h6>';
detailsHtml += '</div>';
detailsHtml += '<h6 class="text-muted">Access Keys</h6>';
if (user.access_keys && user.access_keys.length > 0) {
detailsHtml += '<div class="mb-2">';
user.access_keys.forEach(function(key) {
detailsHtml += '<div><code class="text-muted">' + escapeHtml(key.access_key) + '</code></div>';
detailsHtml += '<div><code class="text-muted">' + key.access_key + '</code></div>';
});
detailsHtml += '</div>';
} else {
@@ -636,10 +579,10 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
user.access_keys.forEach(function(key) {
keysHtml += '<tr>';
keysHtml += '<td><code>' + escapeHtml(key.access_key) + '</code></td>';
keysHtml += '<td><code>' + key.access_key + '</code></td>';
keysHtml += '<td><span class="badge bg-success">Active</span></td>';
keysHtml += '<td>';
keysHtml += '<button class="btn btn-outline-danger btn-sm delete-access-key-btn" data-username="' + escapeHtml(user.username) + '" data-access-key="' + escapeHtml(key.access_key) + '">';
keysHtml += '<button class="btn btn-outline-danger btn-sm" onclick="deleteAccessKey(\'' + user.username + '\', \'' + key.access_key + '\')">';
keysHtml += '<i class="fas fa-trash"></i> Delete';
keysHtml += '</button>';
keysHtml += '</td>';
File diff suppressed because one or more lines are too long
+155 -112
View File
@@ -116,8 +116,7 @@ templ S3Buckets(data dash.S3BucketsData) {
<th>Owner</th>
<th>Created</th>
<th>Objects</th>
<th>Logical Size</th>
<th>Physical Size</th>
<th>Size</th>
<th>Quota</th>
<th>Versioning</th>
<th>Object Lock</th>
@@ -128,7 +127,7 @@ templ S3Buckets(data dash.S3BucketsData) {
for _, bucket := range data.Buckets {
<tr>
<td>
<a href={templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))}
<a href={templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))}
class="text-decoration-none">
<i class="fas fa-cube me-2"></i>
{bucket.Name}
@@ -145,24 +144,16 @@ templ S3Buckets(data dash.S3BucketsData) {
</td>
<td>{bucket.CreatedAt.Format("2006-01-02 15:04")}</td>
<td>{fmt.Sprintf("%d", bucket.ObjectCount)}</td>
<td>
<div>{formatBytes(bucket.LogicalSize)}</div>
if bucket.PhysicalSize > 0 && bucket.LogicalSize > 0 && bucket.PhysicalSize > bucket.LogicalSize {
<div class="small text-muted">
{fmt.Sprintf("%.1fx overhead", float64(bucket.PhysicalSize)/float64(bucket.LogicalSize))}
</div>
}
</td>
<td>{formatBytes(bucket.PhysicalSize)}</td>
<td>{formatBytes(bucket.Size)}</td>
<td>
if bucket.Quota > 0 {
<div>
<span class={fmt.Sprintf("badge bg-%s", getQuotaStatusColor(bucket.LogicalSize, bucket.Quota, bucket.QuotaEnabled))}>
<span class={fmt.Sprintf("badge bg-%s", getQuotaStatusColor(bucket.Size, bucket.Quota, bucket.QuotaEnabled))}>
{formatBytes(bucket.Quota)}
</span>
if bucket.QuotaEnabled {
<div class="small text-muted">
{fmt.Sprintf("%.1f%% used", float64(bucket.LogicalSize)/float64(bucket.Quota)*100)}
{fmt.Sprintf("%.1f%% used", float64(bucket.Size)/float64(bucket.Quota)*100)}
</div>
} else {
<div class="small text-muted">Disabled</div>
@@ -201,7 +192,7 @@ templ S3Buckets(data dash.S3BucketsData) {
</td>
<td>
<div class="btn-group btn-group-sm" role="group">
<a href={templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))}
<a href={templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))}
class="btn btn-outline-success btn-sm"
title="Browse Files">
<i class="fas fa-folder-open"></i>
@@ -239,7 +230,7 @@ templ S3Buckets(data dash.S3BucketsData) {
}
if len(data.Buckets) == 0 {
<tr>
<td colspan="10" class="text-center text-muted py-4">
<td colspan="9" class="text-center text-muted py-4">
<i class="fas fa-cube fa-3x mb-3 text-muted"></i>
<div>
<h5>No Object Store buckets found</h5>
@@ -889,9 +880,9 @@ templ S3Buckets(data dash.S3BucketsData) {
'<div class="text-center py-4">' +
'<div class="spinner-border text-primary" role="status">' +
'<span class="visually-hidden">Loading...</span>' +
'<\\/div>' +
'</div>' +
'<div class="mt-2">Loading bucket details...</div>' +
'<\\/div>';
'</div>';
detailsModalInstance.show();
@@ -904,7 +895,7 @@ templ S3Buckets(data dash.S3BucketsData) {
'<div class="alert alert-danger">' +
'<i class="fas fa-exclamation-triangle me-2"></i>' +
'Error loading bucket details: ' + data.error +
'<\\/div>';
'</div>';
} else {
displayBucketDetails(data);
}
@@ -915,7 +906,7 @@ templ S3Buckets(data dash.S3BucketsData) {
'<div class="alert alert-danger">' +
'<i class="fas fa-exclamation-triangle me-2"></i>' +
'Error loading bucket details: ' + error.message +
'<\\/div>';
'</div>';
});
});
});
@@ -947,91 +938,146 @@ templ S3Buckets(data dash.S3BucketsData) {
});
}
function displayBucketDetails(data) {
const bucket = data.bucket;
function displayBucketDetails(data) {
const bucket = data.bucket;
const objects = data.objects || [];
function escapeHtml(v) {
return String(v ?? '')
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&#39;');
}
function formatBytes(bytes) {
if (bytes === 0) return '0 Bytes';
const k = 1024;
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
}
function formatDate(dateString) {
const date = new Date(dateString);
return date.toLocaleString();
}
let ownerHtml = '<span class="text-muted">No owner (admin-only)</span>';
if (bucket.owner) {
ownerHtml = '<span class="badge bg-info"><i class="fas fa-user me-1"></i>' + escapeHtml(bucket.owner) + '</span>';
}
let usageHtml = '';
if (bucket.physical_size > 0 && bucket.logical_size > 0 && bucket.physical_size > bucket.logical_size) {
const overhead = (bucket.physical_size / bucket.logical_size).toFixed(1);
usageHtml = '<br><small class="text-muted">' + overhead + 'x overhead<\/small>';
}
let quotaHtml = '<span class="badge bg-secondary">Disabled</span>';
if (bucket.quota_enabled) {
quotaHtml = '<span class="badge bg-success">' + formatBytes(bucket.quota) + '</span>';
}
let versioningHtml = '<span class="text-muted">Not configured</span>';
if (bucket.versioning_status === 'Enabled') {
versioningHtml = '<span class="badge bg-success"><i class="fas fa-check me-1"></i>Enabled</span>';
} else if (bucket.versioning_status === 'Suspended') {
versioningHtml = '<span class="badge bg-warning"><i class="fas fa-pause me-1"></i>Suspended</span>';
}
let objectLockHtml = '<span class="text-muted">Not configured</span>';
if (bucket.object_lock_enabled) {
let details = '';
if (bucket.object_lock_mode && bucket.object_lock_duration > 0) {
details = '<br><small class="text-muted">' + escapeHtml(bucket.object_lock_mode) + ' • ' + bucket.object_lock_duration + ' days<\/small>';
// Helper function to escape HTML to prevent XSS
function escapeHtml(v) {
return String(v ?? '')
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&#39;');
}
objectLockHtml = '<span class="badge bg-warning"><i class="fas fa-lock me-1"></i>Enabled</span>' + details;
// Helper function to format bytes
function formatBytes(bytes) {
if (bytes === 0) return '0 Bytes';
const k = 1024;
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
}
// Helper function to format date
function formatDate(dateString) {
const date = new Date(dateString);
return date.toLocaleString();
}
// Generate objects table
let objectsTable = '';
if (objects.length > 0) {
objectsTable = '<div class="table-responsive">' +
'<table class="table table-sm table-striped">' +
'<thead>' +
'<tr>' +
'<th>Object Key</th>' +
'<th>Size</th>' +
'<th>Last Modified</th>' +
'<th>Storage Class</th>' +
'</tr>' +
'</thead>' +
'<tbody>' +
objects.map(obj =>
'<tr>' +
'<td><i class="fas fa-file me-1"></i>' + escapeHtml(obj.key) + '</td>' +
'<td>' + formatBytes(obj.size) + '</td>' +
'<td>' + formatDate(obj.last_modified) + '</td>' +
'<td><span class="badge bg-primary">' + escapeHtml(obj.storage_class) + '</span></td>' +
'</tr>'
).join('') +
'</tbody>' +
'</table>' +
'</div>';
} else {
objectsTable = '<div class="text-center py-4 text-muted">' +
'<i class="fas fa-file fa-3x mb-3"></i>' +
'<div>No objects found in this bucket</div>' +
'</div>';
}
const content = '<div class="row">' +
'<div class="col-md-6">' +
'<h6><i class="fas fa-info-circle me-2"></i>Bucket Information</h6>' +
'<table class="table table-sm">' +
'<tr>' +
'<td><strong>Name:</strong></td>' +
'<td>' + escapeHtml(bucket.name) + '</td>' +
'</tr>' +
'<tr>' +
'<td><strong>Owner:</strong></td>' +
'<td>' + (bucket.owner ? '<span class="badge bg-info"><i class="fas fa-user me-1"></i>' + escapeHtml(bucket.owner) + '</span>' : '<span class="text-muted">No owner (admin-only)</span>') + '</td>' +
'</tr>' +
'<tr>' +
'<td><strong>Created:</strong></td>' +
'<td>' + formatDate(bucket.created_at) + '</td>' +
'</tr>' +
'<tr>' +
'<td><strong>Last Modified:</strong></td>' +
'<td>' + formatDate(bucket.last_modified) + '</td>' +
'</tr>' +
'<tr>' +
'<td><strong>Total Size:</strong></td>' +
'<td>' + formatBytes(bucket.size) + '</td>' +
'</tr>' +
'<tr>' +
'<td><strong>Object Count:</strong></td>' +
'<td>' + bucket.object_count + '</td>' +
'</tr>' +
'</table>' +
'</div>' +
'<div class="col-md-6">' +
'<h6><i class="fas fa-cogs me-2"></i>Configuration</h6>' +
'<table class="table table-sm">' +
'<tr>' +
'<td><strong>Quota:</strong></td>' +
'<td>' +
(bucket.quota_enabled ?
'<span class="badge bg-success">' + formatBytes(bucket.quota) + '</span>' :
'<span class="badge bg-secondary">Disabled</span>'
) +
'</td>' +
'</tr>' +
'<tr>' +
'<td><strong>Versioning:</strong></td>' +
'<td>' +
(bucket.versioning_status === 'Enabled' ?
'<span class="badge bg-success"><i class="fas fa-check me-1"></i>Enabled</span>' :
bucket.versioning_status === 'Suspended' ?
'<span class="badge bg-warning"><i class="fas fa-pause me-1"></i>Suspended</span>' :
'<span class="text-muted">Not configured</span>'
) +
'</td>' +
'</tr>' +
'<tr>' +
'<td><strong>Object Lock:</strong></td>' +
'<td>' +
(bucket.object_lock_enabled ?
'<span class="badge bg-warning"><i class="fas fa-lock me-1"></i>Enabled</span>' +
(bucket.object_lock_mode && bucket.object_lock_duration > 0 ?
'<br><small class="text-muted">' + escapeHtml(bucket.object_lock_mode) + ' • ' + bucket.object_lock_duration + ' days</small>' :
'') :
'<span class="text-muted">Not configured</span>'
) +
'</td>' +
'</tr>' +
'</table>' +
'</div>' +
'</div>' +
'<hr>' +
'<div class="row">' +
'<div class="col-12">' +
'<h6><i class="fas fa-list me-2"></i>Objects (' + objects.length + ')</h6>' +
objectsTable +
'</div>' +
'</div>';
document.getElementById('bucketDetailsContent').innerHTML = content;
}
const rows = [
'<div class="row">',
'<div class="col-md-6">',
'<h6><i class="fas fa-info-circle me-2"></i>Bucket Information</h6>',
'<table class="table table-sm">',
'<tr><td><strong>Name:</strong></td><td>' + escapeHtml(bucket.name) + '<\/td><\/tr>',
'<tr><td><strong>Owner:</strong></td><td>' + ownerHtml + '<\/td><\/tr>',
'<tr><td><strong>Created:</strong></td><td>' + formatDate(bucket.created_at) + '<\/td><\/tr>',
'<tr><td><strong>Last Modified:</strong></td><td>' + formatDate(bucket.last_modified) + '<\/td><\/tr>',
'<tr><td><strong>Logical Size:</strong></td><td>' + formatBytes(bucket.logical_size) + '<\/td><\/tr>',
'<tr><td><strong>Physical Size:</strong></td><td>' + formatBytes(bucket.physical_size) + usageHtml + '<\/td><\/tr>',
'<tr><td><strong>Object Count:</strong></td><td>' + bucket.object_count + '<\/td><\/tr>',
'<\/table>',
'<\/div>',
'<div class="col-md-6">',
'<h6><i class="fas fa-cogs me-2"></i>Configuration</h6>',
'<table class="table table-sm">',
'<tr><td><strong>Quota:</strong></td><td>' + quotaHtml + '<\/td><\/tr>',
'<tr><td><strong>Versioning:</strong></td><td>' + versioningHtml + '<\/td><\/tr>',
'<tr><td><strong>Object Lock:</strong></td><td>' + objectLockHtml + '<\/td><\/tr>',
'<\/table>',
'<\/div>',
'<\/div>'
];
document.getElementById('bucketDetailsContent').innerHTML = rows.join('');
}
function exportBucketList() {
// RFC 4180 compliant CSV escaping: escape double quotes by doubling them
function escapeCsvField(value) {
@@ -1051,26 +1097,23 @@ function displayBucketDetails(data) {
owner: cells[1].textContent.trim(),
created: cells[2].textContent.trim(),
objects: cells[3].textContent.trim(),
logicalSize: cells[4].textContent.trim(),
physicalSize: cells[5].textContent.trim(),
quota: cells[6].textContent.trim(),
versioning: cells[7].textContent.trim(),
objectLock: cells[8].textContent.trim()
size: cells[4].textContent.trim(),
quota: cells[5].textContent.trim(),
versioning: cells[6].textContent.trim(),
objectLock: cells[7].textContent.trim()
};
}
return null;
}).filter(bucket => bucket !== null);
const csvContent = "data:text/csv;charset=utf-8," +
"Name,Owner,Logical Size,Physical Size,Object Count,Created,Quota,Versioning,Object Lock\n" +
"Name,Owner,Created,Objects,Size,Quota,Versioning,Object Lock\n" +
buckets.map(b => [
escapeCsvField(b.name),
escapeCsvField(b.owner),
escapeCsvField(b.logicalSize),
escapeCsvField(b.physicalSize),
escapeCsvField(b.objects),
escapeCsvField(b.created),
escapeCsvField(b.objects),
escapeCsvField(b.size),
escapeCsvField(b.quota),
escapeCsvField(b.versioning),
escapeCsvField(b.objectLock)
@@ -1108,4 +1151,4 @@ func getQuotaInMB(quotaBytes int64) int64 {
quotaBytes = -quotaBytes // Handle disabled quotas (negative values)
}
return quotaBytes / (1024 * 1024)
}
}
File diff suppressed because one or more lines are too long
+12 -30
View File
@@ -270,7 +270,7 @@ templ ServiceAccounts(data dash.ServiceAccountsData) {
<label class="form-label fw-bold">AWS_ACCESS_KEY_ID</label>
<div class="input-group">
<input type="text" class="form-control font-monospace" id="displayAccessKey" readonly>
<button class="btn btn-outline-secondary" type="button" onclick="copyCredentialToClipboard(this, 'displayAccessKey')">
<button class="btn btn-outline-secondary" type="button" onclick="copyToClipboard(event, 'displayAccessKey')">
<i class="fas fa-copy"></i> Copy
</button>
</div>
@@ -280,7 +280,7 @@ templ ServiceAccounts(data dash.ServiceAccountsData) {
<label class="form-label fw-bold">AWS_SECRET_ACCESS_KEY</label>
<div class="input-group">
<input type="text" class="form-control font-monospace" id="displaySecretKey" readonly>
<button class="btn btn-outline-secondary" type="button" onclick="copyCredentialToClipboard(this, 'displaySecretKey')">
<button class="btn btn-outline-secondary" type="button" onclick="copyToClipboard(event, 'displaySecretKey')">
<i class="fas fa-copy"></i> Copy
</button>
</div>
@@ -474,25 +474,27 @@ templ ServiceAccounts(data dash.ServiceAccountsData) {
setTimeout(() => window.location.reload(), 500);
}
function copyCredentialToClipboard(button, elementId) {
function copyToClipboard(event, elementId) {
const element = document.getElementById(elementId);
const textToCopy = element.value;
// Use modern Clipboard API if available
if (navigator.clipboard && navigator.clipboard.writeText) {
navigator.clipboard.writeText(textToCopy).then(() => {
showSuccessMessage('Copied to clipboard!');
if (navigator.clipboard) {
navigator.clipboard.writeText(element.value).then(() => {
// Success feedback could be added here
}).catch(err => {
console.warn('Clipboard API failed:', err);
// Fallback
fallbackCopyTextToClipboard(element);
element.select();
document.execCommand('copy');
});
} else {
// Fallback for older browsers or non-secure contexts
fallbackCopyTextToClipboard(element);
// Fallback for older browsers
element.select();
document.execCommand('copy');
}
// Visual feedback
const button = event.target.closest('button');
const originalHTML = button.innerHTML;
button.innerHTML = '<i class="fas fa-check"></i>';
@@ -506,26 +508,6 @@ templ ServiceAccounts(data dash.ServiceAccountsData) {
}, 1000);
}
function fallbackCopyTextToClipboard(element) {
element.select();
element.setSelectionRange(0, 99999); // For mobile devices
try {
const successful = document.execCommand('copy');
if (successful) {
showSuccessMessage('Copied to clipboard!');
} else {
showErrorMessage('Failed to copy to clipboard');
}
} catch (err) {
console.error('Fallback copy failed:', err);
showErrorMessage('Failed to copy to clipboard');
}
// Clear selection
window.getSelection().removeAllRanges();
}
function createSADetailsContent(sa) {
// Create DOM elements safely to prevent XSS
File diff suppressed because one or more lines are too long
+1 -2
View File
@@ -1,11 +1,10 @@
package cluster
import (
"github.com/seaweedfs/seaweedfs/weed/pb"
"strconv"
"sync"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb"
)
func TestConcurrentAddRemoveNodes(t *testing.T) {
+1 -2
View File
@@ -1,9 +1,8 @@
package cluster
import (
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"time"
)
type GroupMembers struct {
+12 -27
View File
@@ -32,18 +32,17 @@ func NewLockClient(grpcDialOption grpc.DialOption, seedFiler pb.ServerAddress) *
}
type LiveLock struct {
key string
renewToken string
expireAtNs int64
hostFiler pb.ServerAddress
cancelCh chan struct{}
grpcDialOption grpc.DialOption
isLocked int32 // 0 = unlocked, 1 = locked; use atomic operations
self string
lc *LockClient
owner string
lockTTL time.Duration
consecutiveFailures int // Track connection failures to trigger fallback
key string
renewToken string
expireAtNs int64
hostFiler pb.ServerAddress
cancelCh chan struct{}
grpcDialOption grpc.DialOption
isLocked int32 // 0 = unlocked, 1 = locked; use atomic operations
self string
lc *LockClient
owner string
lockTTL time.Duration
}
// NewShortLivedLock creates a lock with a 5-second duration
@@ -214,7 +213,6 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
glog.V(4).Infof("LOCK: DistributedLock response - key=%s err=%v", lock.key, err)
if err == nil && resp != nil {
lock.renewToken = resp.RenewToken
lock.consecutiveFailures = 0 // Reset failure counter on success
glog.V(4).Infof("LOCK: Got renewToken for key=%s", lock.key)
} else {
//this can be retried. Need to remember the last valid renewToken
@@ -227,7 +225,7 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
// Only log if the host actually changed
glog.V(2).Infof("LOCK: Host changed from %s to %s for key=%s", previousHostFiler, resp.LockHostMovedTo, lock.key)
lock.hostFiler = pb.ServerAddress(resp.LockHostMovedTo)
// Don't update seedFiler - keep original for fallback
lock.lc.seedFiler = lock.hostFiler
} else if resp.LockHostMovedTo != "" {
lock.hostFiler = pb.ServerAddress(resp.LockHostMovedTo)
}
@@ -244,19 +242,6 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
}
return err
})
if err != nil && lock.hostFiler != lock.lc.seedFiler {
lock.consecutiveFailures++
// Fall back to seed filer after 3 consecutive connection failures
if lock.consecutiveFailures >= 3 {
glog.V(0).Infof("LOCK: Connection failed %d times for key=%s filer=%s, falling back to seed filer=%s",
lock.consecutiveFailures, lock.key, lock.hostFiler, lock.lc.seedFiler)
lock.hostFiler = lock.lc.seedFiler
lock.consecutiveFailures = 0
lock.renewToken = ""
}
}
return
}
@@ -2,10 +2,9 @@ package lock_manager
import (
"fmt"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"time"
)
const RenewInterval = time.Second * 3
+2 -3
View File
@@ -2,11 +2,10 @@ package lock_manager
import (
"fmt"
"sync"
"time"
"github.com/google/uuid"
"github.com/seaweedfs/seaweedfs/weed/glog"
"sync"
"time"
)
var LockErrorNonEmptyTokenOnNewLock = fmt.Errorf("lock: non-empty token on a new lock")
+4 -4
View File
@@ -210,8 +210,8 @@ func runAdmin(cmd *Command, args []string) bool {
cancel()
}()
// Start the admin server with all masters (UI enabled by default)
err := startAdminServer(ctx, a, true)
// Start the admin server with all masters
err := startAdminServer(ctx, a)
if err != nil {
fmt.Printf("Admin server error: %v\n", err)
return false
@@ -222,7 +222,7 @@ func runAdmin(cmd *Command, args []string) bool {
}
// startAdminServer starts the actual admin server
func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool) error {
func startAdminServer(ctx context.Context, options AdminOptions) error {
// Set Gin mode
gin.SetMode(gin.ReleaseMode)
@@ -307,7 +307,7 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool)
// Create handlers and setup routes
authRequired := *options.adminPassword != ""
adminHandlers := handlers.NewAdminHandlers(adminServer)
adminHandlers.SetupRoutes(r, authRequired, *options.adminUser, *options.adminPassword, *options.readOnlyUser, *options.readOnlyPassword, enableUI)
adminHandlers.SetupRoutes(r, authRequired, *options.adminUser, *options.adminPassword, *options.readOnlyUser, *options.readOnlyPassword)
// Server configuration
addr := fmt.Sprintf(":%d", *options.port)
+3 -4
View File
@@ -2,13 +2,12 @@ package command
import (
"fmt"
"os"
"path/filepath"
"runtime"
"github.com/posener/complete"
completeinstall "github.com/posener/complete/cmd/install"
flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
"os"
"path/filepath"
"runtime"
)
func AutocompleteMain(commands []*Command) bool {
+4 -3
View File
@@ -4,6 +4,8 @@ import (
"bufio"
"context"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/util/version"
"io"
"math"
"math/rand"
@@ -14,15 +16,14 @@ import (
"sync"
"time"
"google.golang.org/grpc"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
"github.com/seaweedfs/seaweedfs/weed/util/version"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"google.golang.org/grpc"
)
type BenchmarkOptions struct {
-1
View File
@@ -137,7 +137,6 @@ func init() {
filerS3Options.concurrentUploadLimitMB = cmdFiler.Flag.Int("s3.concurrentUploadLimitMB", 0, "limit total concurrent upload size for S3, 0 means unlimited")
filerS3Options.concurrentFileUploadLimit = cmdFiler.Flag.Int("s3.concurrentFileUploadLimit", 0, "limit number of concurrent file uploads for S3, 0 means unlimited")
filerS3Options.enableIam = cmdFiler.Flag.Bool("s3.iam", true, "enable embedded IAM API on the same S3 port")
filerS3Options.cipher = cmdFiler.Flag.Bool("s3.encryptVolumeData", false, "encrypt data on volume servers for S3 uploads")
// start webdav on filer
filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
+4 -16
View File
@@ -83,6 +83,8 @@ func runFilerBackup(cmd *Command, args []string) bool {
time.Sleep(1747 * time.Millisecond)
}
}
// Unreachable: satisfies bool return type signature for daemon function
return false
}
const (
@@ -146,13 +148,13 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
}
// ignore HTTP 404 from remote reads
if errors.Is(err, http.ErrNotFound) {
glog.V(0).Infof("got 404 error for %s, ignore it: %s", getSourceKey(resp), err.Error())
glog.V(0).Infof("got 404 error, ignore it: %s", err.Error())
return nil
}
// also ignore missing volume/lookup errors coming from LookupFileId or vid map
errStr := err.Error()
if strings.Contains(errStr, "LookupFileId") || (strings.Contains(errStr, "volume id") && strings.Contains(errStr, "not found")) {
glog.V(0).Infof("got missing-volume error for %s, ignore it: %s", getSourceKey(resp), errStr)
glog.V(0).Infof("got missing-volume error, ignore it: %s", errStr)
return nil
}
return err
@@ -204,17 +206,3 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
return pb.FollowMetadata(sourceFiler, grpcDialOption, metadataFollowOption, processEventFnWithOffset)
}
func getSourceKey(resp *filer_pb.SubscribeMetadataResponse) string {
if resp == nil || resp.EventNotification == nil {
return ""
}
message := resp.EventNotification
if message.NewEntry != nil {
return string(util.FullPath(message.NewParentPath).Child(message.NewEntry.Name))
}
if message.OldEntry != nil {
return string(util.FullPath(resp.Directory).Child(message.OldEntry.Name))
}
return ""
}
+2
View File
@@ -126,6 +126,8 @@ func runFilerMetaBackup(cmd *Command, args []string) bool {
time.Sleep(1747 * time.Millisecond)
}
}
// Unreachable: satisfies bool return type signature for daemon function
return false
}
func (metaBackup *FilerMetaBackupOptions) initStore(v *viper.Viper) error {
+2 -2
View File
@@ -2,13 +2,13 @@ package command
import (
"fmt"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb"
"os"
"path/filepath"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
+1 -2
View File
@@ -5,12 +5,11 @@ package command
import (
"context"
"strings"
jsoniter "github.com/json-iterator/go"
elastic "github.com/olivere/elastic/v7"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"strings"
)
type EsDocument struct {
+2 -3
View File
@@ -3,9 +3,6 @@ package command
import (
"context"
"fmt"
"os"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -14,6 +11,8 @@ import (
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
"os"
"time"
)
type RemoteGatewayOptions struct {
+5 -6
View File
@@ -3,12 +3,6 @@ package command
import (
"context"
"fmt"
"math"
"math/rand"
"path/filepath"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -18,6 +12,11 @@ import (
"github.com/seaweedfs/seaweedfs/weed/replication/source"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/protobuf/proto"
"math"
"math/rand"
"path/filepath"
"strings"
"time"
)
func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSource *source.FilerSource) error {
+1 -2
View File
@@ -2,8 +2,6 @@ package command
import (
"fmt"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -11,6 +9,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
"time"
)
type RemoteSyncOptions struct {
+4 -3
View File
@@ -3,12 +3,11 @@ package command
import (
"context"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"os"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -41,7 +40,9 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error {
if resp.EventNotification.NewEntry != nil {
if *option.storageClass == "" {
delete(resp.EventNotification.NewEntry.Extended, s3_constants.AmzStorageClass)
if _, ok := resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass]; ok {
delete(resp.EventNotification.NewEntry.Extended, s3_constants.AmzStorageClass)
}
} else {
resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass] = []byte(*option.storageClass)
}
+2 -3
View File
@@ -1,13 +1,12 @@
package command
import (
"sync"
"sync/atomic"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"sync"
"sync/atomic"
)
type MetadataProcessor struct {
-30
View File
@@ -224,36 +224,6 @@ func runFuse(cmd *Command, args []string) bool {
fusermountPath = parameter.value
case "config_dir":
util.ConfigurationFileDirectory.Set(parameter.value)
// FUSE performance options
case "writebackCache":
if parsed, err := strconv.ParseBool(parameter.value); err == nil {
mountOptions.writebackCache = &parsed
} else {
fmt.Fprintf(os.Stderr, "failed to parse 'writebackCache' value %q: %v\n", parameter.value, err)
return false
}
case "asyncDio":
if parsed, err := strconv.ParseBool(parameter.value); err == nil {
mountOptions.asyncDio = &parsed
} else {
fmt.Fprintf(os.Stderr, "failed to parse 'asyncDio' value %q: %v\n", parameter.value, err)
return false
}
case "cacheSymlink":
if parsed, err := strconv.ParseBool(parameter.value); err == nil {
mountOptions.cacheSymlink = &parsed
} else {
fmt.Fprintf(os.Stderr, "failed to parse 'cacheSymlink' value %q: %v\n", parameter.value, err)
return false
}
// macOS-specific FUSE options
case "sys.novncache":
if parsed, err := strconv.ParseBool(parameter.value); err == nil {
mountOptions.novncache = &parsed
} else {
fmt.Fprintf(os.Stderr, "failed to parse 'sys.novncache' value %q: %v\n", parameter.value, err)
return false
}
default:
t := parameter.name
if parameter.value != "true" {
+93 -109
View File
@@ -11,8 +11,10 @@ import (
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
iam_pb "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/util"
@@ -55,9 +57,6 @@ var (
createdInitialIAM bool // Track if initial IAM config was created from env vars
// Track which port flags were explicitly passed on CLI before config file is applied
explicitPortFlags map[string]bool
miniEnableWebDAV *bool
miniEnableS3 *bool
miniEnableAdminUI *bool
)
func init() {
@@ -136,9 +135,6 @@ func initMiniCommonFlags() {
miniOptions.memprofile = cmdMini.Flag.String("memprofile", "", "memory profile output file")
miniOptions.debug = cmdMini.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:6060/debug/pprof/goroutine?debug=2")
miniOptions.debugPort = cmdMini.Flag.Int("debug.port", 6060, "http port for debugging")
miniEnableWebDAV = cmdMini.Flag.Bool("webdav", true, "enable WebDAV server")
miniEnableS3 = cmdMini.Flag.Bool("s3", true, "enable S3 server")
miniEnableAdminUI = cmdMini.Flag.Bool("admin.ui", true, "enable Admin UI")
}
// initMiniMasterFlags initializes Master server flag options
@@ -450,27 +446,10 @@ func ensureAllPortsAvailableOnIP(bindIp string) error {
{miniMasterOptions.port, "Master", "master.port", miniMasterOptions.portGrpc},
{miniFilerOptions.port, "Filer", "filer.port", miniFilerOptions.portGrpc},
{miniOptions.v.port, "Volume", "volume.port", miniOptions.v.portGrpc},
{miniS3Options.port, "S3", "s3.port", miniS3Options.portGrpc},
{miniWebDavOptions.port, "WebDAV", "webdav.port", nil},
{miniAdminOptions.port, "Admin", "admin.port", miniAdminOptions.grpcPort},
}
if *miniEnableS3 {
portConfigs = append(portConfigs, struct {
port *int
name string
flagName string
grpcPtr *int
}{miniS3Options.port, "S3", "s3.port", miniS3Options.portGrpc})
}
portConfigs = append(portConfigs, struct {
port *int
name string
flagName string
grpcPtr *int
}{miniWebDavOptions.port, "WebDAV", "webdav.port", nil},
struct {
port *int
name string
flagName string
grpcPtr *int
}{miniAdminOptions.port, "Admin", "admin.port", miniAdminOptions.grpcPort})
// First, reserve all gRPC ports that will be calculated to prevent HTTP port allocation from using them
// This prevents collisions like: HTTP port moves to X, then gRPC port is calculated as Y where Y == X
@@ -533,19 +512,9 @@ func initializeGrpcPortsOnIP(bindIp string) {
{miniMasterOptions.port, miniMasterOptions.portGrpc, "Master"},
{miniFilerOptions.port, miniFilerOptions.portGrpc, "Filer"},
{miniOptions.v.port, miniOptions.v.portGrpc, "Volume"},
{miniS3Options.port, miniS3Options.portGrpc, "S3"},
{miniAdminOptions.port, miniAdminOptions.grpcPort, "Admin"},
}
if *miniEnableS3 {
grpcConfigs = append(grpcConfigs, struct {
httpPort *int
grpcPort *int
name string
}{miniS3Options.port, miniS3Options.portGrpc, "S3"})
}
grpcConfigs = append(grpcConfigs, struct {
httpPort *int
grpcPort *int
name string
}{miniAdminOptions.port, miniAdminOptions.grpcPort, "Admin"})
for _, config := range grpcConfigs {
if config.grpcPort == nil {
@@ -852,25 +821,17 @@ func startMiniServices(miniWhiteList []string, allServicesReady chan struct{}) {
waitForServiceReady("Filer", *miniFilerOptions.port, bindIp)
// Start S3 and WebDAV in parallel (both depend on filer)
if *miniEnableS3 {
go startMiniService("S3", func() {
startS3Service()
}, *miniS3Options.port)
}
go startMiniService("S3", func() {
startS3Service()
}, *miniS3Options.port)
if *miniEnableWebDAV {
go startMiniService("WebDAV", func() {
miniWebDavOptions.startWebDav()
}, *miniWebDavOptions.port)
}
go startMiniService("WebDAV", func() {
miniWebDavOptions.startWebDav()
}, *miniWebDavOptions.port)
// Wait for both S3 and WebDAV to be ready
if *miniEnableS3 {
waitForServiceReady("S3", *miniS3Options.port, bindIp)
}
if *miniEnableWebDAV {
waitForServiceReady("WebDAV", *miniWebDavOptions.port, bindIp)
}
waitForServiceReady("S3", *miniS3Options.port, bindIp)
waitForServiceReady("WebDAV", *miniWebDavOptions.port, bindIp)
// Start Admin with worker (depends on master, filer, S3, WebDAV)
go startMiniAdminWithWorker(allServicesReady)
@@ -914,7 +875,37 @@ func startS3Service() {
secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
if accessKey != "" && secretKey != "" {
createdInitialIAM = true
user := "mini"
iamCfg := &iam_pb.S3ApiConfiguration{}
ident := &iam_pb.Identity{Name: user}
ident.Credentials = append(ident.Credentials, &iam_pb.Credential{AccessKey: accessKey, SecretKey: secretKey})
ident.Actions = append(ident.Actions, "Admin")
iamCfg.Identities = append(iamCfg.Identities, ident)
iamPath := filepath.Join(*miniDataFolders, "iam_config.json")
// Check if IAM config file already exists
if _, err := os.Stat(iamPath); err == nil {
// File exists, skip writing to preserve existing configuration
glog.V(1).Infof("IAM config file already exists at %s, preserving existing configuration", iamPath)
*miniIamConfig = iamPath
} else if os.IsNotExist(err) {
// File does not exist, create and write new configuration
f, err := os.OpenFile(iamPath, os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
glog.Fatalf("failed to create IAM config file %s: %v", iamPath, err)
}
defer f.Close()
if err := filer.ProtoToText(f, iamCfg); err != nil {
glog.Fatalf("failed to write IAM config to %s: %v", iamPath, err)
}
*miniIamConfig = iamPath
createdInitialIAM = true // Mark that we created initial IAM config
glog.V(1).Infof("Created initial IAM config at %s", iamPath)
} else {
// Error checking file existence
glog.Fatalf("failed to check IAM config file existence at %s: %v", iamPath, err)
}
}
miniS3Options.localFilerSocket = miniFilerOptions.localSocket
@@ -927,9 +918,6 @@ func startMiniAdminWithWorker(allServicesReady chan struct{}) {
ctx := context.Background()
// Determine bind IP for health checks
bindIp := getBindIp()
// Prepare master address
masterAddr := fmt.Sprintf("%s:%d", *miniIp, *miniMasterOptions.port)
@@ -967,13 +955,13 @@ func startMiniAdminWithWorker(allServicesReady chan struct{}) {
// Start admin server in background
go func() {
if err := startAdminServer(ctx, miniAdminOptions, *miniEnableAdminUI); err != nil {
if err := startAdminServer(ctx, miniAdminOptions); err != nil {
glog.Errorf("Admin server error: %v", err)
}
}()
// Wait for admin server's HTTP port to be ready before launching worker
adminAddr := fmt.Sprintf("http://%s:%d", bindIp, *miniAdminOptions.port)
adminAddr := fmt.Sprintf("http://127.0.0.1:%d", *miniAdminOptions.port)
glog.V(1).Infof("Waiting for admin server to be ready at %s...", adminAddr)
if err := waitForAdminServerReady(adminAddr); err != nil {
glog.Fatalf("Admin server readiness check failed: %v", err)
@@ -983,21 +971,20 @@ func startMiniAdminWithWorker(allServicesReady chan struct{}) {
startMiniWorker()
// Wait for worker to be ready by polling its gRPC port
workerGrpcAddr := fmt.Sprintf("%s:%d", bindIp, *miniAdminOptions.grpcPort)
workerGrpcAddr := fmt.Sprintf("127.0.0.1:%d", *miniAdminOptions.grpcPort)
waitForWorkerReady(workerGrpcAddr)
}
// waitForAdminServerReady pings the admin server HTTP endpoint to check if it's ready
func waitForAdminServerReady(adminAddr string) error {
healthAddr := fmt.Sprintf("%s/health", adminAddr)
maxAttempts := 60 // 60 * 500ms = 30 seconds max wait
maxAttempts := 40 // 40 * 500ms = 20 seconds max wait
attempt := 0
client := &http.Client{
Timeout: 1 * time.Second,
Timeout: 500 * time.Millisecond,
}
for attempt < maxAttempts {
resp, err := client.Get(healthAddr)
resp, err := client.Get(adminAddr)
if err == nil {
resp.Body.Close()
glog.V(1).Infof("Admin server is ready at %s", adminAddr)
@@ -1106,6 +1093,32 @@ func startMiniWorker() {
glog.Infof("Maintenance worker %s started successfully", workerInstance.ID())
}
const welcomeMessageTemplate = `
╔═══════════════════════════════════════════════════════════════════════════════╗
║ SeaweedFS Mini - All-in-One Mode ║
╚═══════════════════════════════════════════════════════════════════════════════╝
All components are running and ready to use:
Master UI: http://%s:%d
Filer UI: http://%s:%d
S3 Endpoint: http://%s:%d
WebDAV: http://%s:%d
Admin UI: http://%s:%d
Volume Server: http://%s:%d
Optimized Settings:
• Volume size limit: %dMB
• Volume max: auto (based on free disk space)
• Pre-stop seconds: 1 (faster shutdown)
• Master peers: none (single master mode)
• Admin UI for management and maintenance tasks
Data Directory: %s
Press Ctrl+C to stop all components
`
const credentialsInstructionTemplate = `
To create S3 credentials, you have two options:
@@ -1121,57 +1134,28 @@ const credentialsInstructionTemplate = `
`
const credentialsCreatedMessage = `
Initial S3 credentials loaded from environment variables.
Initial S3 credentials created:
user: mini
Note: credentials have been written to the IAM configuration file.
`
// printWelcomeMessage prints the welcome message after all services are running
func printWelcomeMessage() {
var sb strings.Builder
sb.WriteString("╔═══════════════════════════════════════════════════════════════════════════════╗\n")
sb.WriteString("║ SeaweedFS Mini - All-in-One Mode ║\n")
sb.WriteString("╚═══════════════════════════════════════════════════════════════════════════════╝\n\n")
sb.WriteString(" All enabled components are running and ready to use:\n\n")
fmt.Fprintf(&sb, " Master UI: http://%s:%d\n", *miniIp, *miniMasterOptions.port)
fmt.Fprintf(&sb, " Filer UI: http://%s:%d\n", *miniIp, *miniFilerOptions.port)
if *miniEnableS3 {
fmt.Fprintf(&sb, " S3 Endpoint: http://%s:%d\n", *miniIp, *miniS3Options.port)
}
if *miniEnableWebDAV {
fmt.Fprintf(&sb, " WebDAV: http://%s:%d\n", *miniIp, *miniWebDavOptions.port)
}
if *miniEnableAdminUI {
fmt.Fprintf(&sb, " Admin UI: http://%s:%d\n", *miniIp, *miniAdminOptions.port)
}
fmt.Fprintf(&sb, " Volume Server: http://%s:%d\n\n", *miniIp, *miniOptions.v.port)
sb.WriteString(" Optimized Settings:\n")
fmt.Fprintf(&sb, " • Volume size limit: %dMB\n", *miniMasterOptions.volumeSizeLimitMB)
sb.WriteString(" • Volume max: auto (based on free disk space)\n")
sb.WriteString(" • Pre-stop seconds: 1 (faster shutdown)\n")
sb.WriteString(" • Master peers: none (single master mode)\n")
if *miniEnableAdminUI {
sb.WriteString(" • Admin UI for management and maintenance tasks\n")
}
fmt.Fprintf(&sb, "\n Data Directory: %s\n\n", *miniDataFolders)
sb.WriteString(" Press Ctrl+C to stop all components")
fmt.Printf(welcomeMessageTemplate,
*miniIp, *miniMasterOptions.port,
*miniIp, *miniFilerOptions.port,
*miniIp, *miniS3Options.port,
*miniIp, *miniWebDavOptions.port,
*miniIp, *miniAdminOptions.port,
*miniIp, *miniOptions.v.port,
*miniMasterOptions.volumeSizeLimitMB,
*miniDataFolders,
)
if createdInitialIAM {
sb.WriteString(credentialsCreatedMessage)
} else if *miniEnableAdminUI {
fmt.Fprintf(&sb, credentialsInstructionTemplate, *miniIp, *miniAdminOptions.port)
fmt.Print(credentialsCreatedMessage)
} else {
sb.WriteString("\n To create S3 credentials, use environment variables:\n\n")
sb.WriteString(" export AWS_ACCESS_KEY_ID=your-access-key\\n")
sb.WriteString(" export AWS_SECRET_ACCESS_KEY=your-secret-key\\n")
sb.WriteString(" weed mini -dir=/data\\n")
sb.WriteString(" This will create initial credentials for the 'mini' user.\\n")
fmt.Printf(credentialsInstructionTemplate, *miniIp, *miniAdminOptions.port)
}
fmt.Print(sb.String())
fmt.Println("")
}
-16
View File
@@ -47,14 +47,6 @@ type MountOptions struct {
rdmaReadOnly *bool
rdmaMaxConcurrent *int
rdmaTimeoutMs *int
// FUSE performance options
writebackCache *bool
asyncDio *bool
cacheSymlink *bool
// macOS-specific FUSE options
novncache *bool
}
var (
@@ -110,14 +102,6 @@ func init() {
mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
mountReadRetryTime = cmdMount.Flag.Duration("readRetryTime", 6*time.Second, "maximum read retry wait time")
// FUSE performance options
mountOptions.writebackCache = cmdMount.Flag.Bool("writebackCache", false, "enable FUSE writeback cache for improved write performance (at risk of data loss on crash)")
mountOptions.asyncDio = cmdMount.Flag.Bool("asyncDio", false, "enable async direct I/O for better concurrency")
mountOptions.cacheSymlink = cmdMount.Flag.Bool("cacheSymlink", false, "enable symlink caching to reduce metadata lookups")
// macOS-specific FUSE options
mountOptions.novncache = cmdMount.Flag.Bool("sys.novncache", false, "(macOS only) disable vnode name caching to avoid stale data")
}
var cmdMount = &Command{
+1 -2
View File
@@ -3,11 +3,10 @@ package command
import (
"bufio"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/glog"
"io"
"os"
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
)
const (
+30 -38
View File
@@ -18,7 +18,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/util/version"
"github.com/seaweedfs/go-fuse/v2/fuse"
"github.com/hanwen/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/mount"
"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
@@ -208,20 +208,12 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
if runtime.GOARCH == "amd64" {
fuseMountOptions.Options = append(fuseMountOptions.Options, "noapplexattr")
}
if option.novncache != nil && *option.novncache {
fuseMountOptions.Options = append(fuseMountOptions.Options, "novncache")
}
// fuseMountOptions.Options = append(fuseMountOptions.Options, "novncache") // need to test effectiveness
fuseMountOptions.Options = append(fuseMountOptions.Options, "slow_statfs")
fuseMountOptions.Options = append(fuseMountOptions.Options, "volname="+serverFriendlyName)
fuseMountOptions.Options = append(fuseMountOptions.Options, fmt.Sprintf("iosize=%d", ioSizeMB*1024*1024))
}
fuseMountOptions.EnableWriteback = *option.writebackCache
fuseMountOptions.EnableAsyncDio = *option.asyncDio
if option.cacheSymlink != nil && *option.cacheSymlink {
fuseMountOptions.EnableSymlinkCaching = true
}
// find mount point
mountRoot := filerMountRootPath
if mountRoot != "/" && strings.HasSuffix(mountRoot, "/") {
@@ -234,34 +226,34 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
}
seaweedFileSystem := mount.NewSeaweedFileSystem(&mount.Option{
MountDirectory: dir,
FilerAddresses: filerAddresses,
GrpcDialOption: grpcDialOption,
FilerMountRootPath: mountRoot,
Collection: *option.collection,
Replication: *option.replication,
TtlSec: int32(*option.ttlSec),
DiskType: types.ToDiskType(*option.diskType),
ChunkSizeLimit: int64(chunkSizeLimitMB) * 1024 * 1024,
ConcurrentWriters: *option.concurrentWriters,
ConcurrentReaders: *option.concurrentReaders,
CacheDirForRead: *option.cacheDirForRead,
CacheSizeMBForRead: *option.cacheSizeMBForRead,
CacheDirForWrite: cacheDirForWrite,
CacheMetaTTlSec: *option.cacheMetaTtlSec,
DataCenter: *option.dataCenter,
Quota: int64(*option.collectionQuota) * 1024 * 1024,
MountUid: uid,
MountGid: gid,
MountMode: mountMode,
MountCtime: fileInfo.ModTime(),
MountMtime: time.Now(),
Umask: umask,
VolumeServerAccess: *mountOptions.volumeServerAccess,
Cipher: cipher,
UidGidMapper: uidGidMapper,
DisableXAttr: *option.disableXAttr,
IsMacOs: runtime.GOOS == "darwin",
MountDirectory: dir,
FilerAddresses: filerAddresses,
GrpcDialOption: grpcDialOption,
FilerMountRootPath: mountRoot,
Collection: *option.collection,
Replication: *option.replication,
TtlSec: int32(*option.ttlSec),
DiskType: types.ToDiskType(*option.diskType),
ChunkSizeLimit: int64(chunkSizeLimitMB) * 1024 * 1024,
ConcurrentWriters: *option.concurrentWriters,
ConcurrentReaders: *option.concurrentReaders,
CacheDirForRead: *option.cacheDirForRead,
CacheSizeMBForRead: *option.cacheSizeMBForRead,
CacheDirForWrite: cacheDirForWrite,
CacheMetaTTlSec: *option.cacheMetaTtlSec,
DataCenter: *option.dataCenter,
Quota: int64(*option.collectionQuota) * 1024 * 1024,
MountUid: uid,
MountGid: gid,
MountMode: mountMode,
MountCtime: fileInfo.ModTime(),
MountMtime: time.Now(),
Umask: umask,
VolumeServerAccess: *mountOptions.volumeServerAccess,
Cipher: cipher,
UidGidMapper: uidGidMapper,
DisableXAttr: *option.disableXAttr,
IsMacOs: runtime.GOOS == "darwin",
MetadataFlushSeconds: *option.metadataFlushSeconds,
// RDMA acceleration options
RdmaEnabled: *option.rdmaEnabled,
+2 -2
View File
@@ -68,7 +68,7 @@ type S3Options struct {
func init() {
cmdS3.Run = runS3 // break init cycle
s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "comma-separated filer server addresses for high availability")
s3StandaloneOptions.bindIp = cmdS3.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to 0.0.0.0.")
s3StandaloneOptions.bindIp = cmdS3.Flag.String("ip.bind", "", "ip address to bind to. Default to localhost.")
s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port")
s3StandaloneOptions.portHttps = cmdS3.Flag.Int("port.https", 0, "s3 server https listen port")
s3StandaloneOptions.portGrpc = cmdS3.Flag.Int("port.grpc", 0, "s3 server grpc listen port")
@@ -302,7 +302,7 @@ func (s3opt *S3Options) startS3Server() bool {
*s3opt.portGrpc = 10000 + *s3opt.port
}
if *s3opt.bindIp == "" {
*s3opt.bindIp = "0.0.0.0"
*s3opt.bindIp = "localhost"
}
if runtime.GOOS != "windows" {

Some files were not shown because too many files have changed in this diff Show More