mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-10 08:35:50 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0a1b54b146 | ||
|
|
b4c7d42a06 | ||
|
|
cba69f4593 | ||
|
|
91f59e73e5 | ||
|
|
98d89ffad7 | ||
|
|
f4af1cc0ba | ||
|
|
9fa95dd2c6 | ||
|
|
ff84ef880d | ||
|
|
2d65d7f499 | ||
|
|
3f58e3bf8f | ||
|
|
016391530b | ||
|
|
8d59ef41d5 | ||
|
|
e596542295 | ||
|
|
61db4d0966 | ||
|
|
74de694447 | ||
|
|
c5e8e4f049 | ||
|
|
c96b0913ed | ||
|
|
b033823611 | ||
|
|
3044b51e7c | ||
|
|
8e8edd7706 | ||
|
|
57ab99d13e | ||
|
|
998c8d2702 | ||
|
|
cd6832249b | ||
|
|
b5f3094619 | ||
|
|
07f284c391 | ||
|
|
7b08cf74ed | ||
|
|
d8b8f0dffd | ||
|
|
8e25c55bfb | ||
|
|
e4b70c2521 | ||
|
|
f7c27cc81f | ||
|
|
66680c58b7 | ||
|
|
2a1ae896e4 | ||
|
|
bd0b1fe9d5 | ||
|
|
964a8f5fde | ||
|
|
40cc0e04a6 | ||
|
|
2f837c4780 | ||
|
|
36c469e34e | ||
|
|
a2005cb2a6 | ||
|
|
e9c45144cf | ||
|
|
5ecee9e64d | ||
|
|
01b3125815 | ||
|
|
7b8df39cf7 | ||
|
|
6787dccace | ||
|
|
d1fecdface | ||
|
|
38e14a867b | ||
|
|
eda4a000cc | ||
|
|
bf71fe0039 | ||
|
|
53048ffffb | ||
|
|
6a3a97333f | ||
|
|
8ec9ff4a12 | ||
|
|
5463038760 | ||
|
|
828cbabb55 | ||
|
|
5919f519fd | ||
|
|
63f641a6c9 | ||
|
|
3c3a78d08e | ||
|
|
3300874cb5 | ||
|
|
bddd7960c1 | ||
|
|
a9d12a0792 | ||
|
|
564fc56698 | ||
|
|
11fdb68281 | ||
|
|
35ad7d08a5 | ||
|
|
0d8588e3ae | ||
|
|
cc58272219 | ||
|
|
5be4ee9892 | ||
|
|
22e74221cb | ||
|
|
cc80641be1 | ||
|
|
927c906379 | ||
|
|
0721e3c1e9 |
@@ -11,4 +11,4 @@ jobs:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@3c4e3dcb1aa7874d2c16be7d79418e9b7efd6261
|
||||
uses: actions/dependency-review-action@05fe4576374b728f0c523d6a13d64c25081e0803
|
||||
|
||||
@@ -119,7 +119,7 @@ jobs:
|
||||
echo "✅ All template rendering tests passed!"
|
||||
|
||||
- name: Create kind cluster
|
||||
uses: helm/kind-action@v1.13.0
|
||||
uses: helm/kind-action@v1.14.0
|
||||
|
||||
- name: Run chart-testing (install)
|
||||
run: ct install --target-branch ${{ github.event.repository.default_branch }} --all --chart-dirs k8s/charts
|
||||
|
||||
@@ -111,7 +111,7 @@ jobs:
|
||||
|
||||
# Wait for S3 API
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8333/ > /dev/null 2>&1; then
|
||||
if curl -s http://localhost:8333/healthz > /dev/null 2>&1; then
|
||||
echo "✓ S3 API is ready"
|
||||
break
|
||||
fi
|
||||
|
||||
@@ -39,6 +39,14 @@ jobs:
|
||||
echo "=== Running S3 Integration Tests ==="
|
||||
go test -v -timeout=60s -run TestS3Integration ./...
|
||||
|
||||
- name: Run S3 DeleteBucketNotEmpty Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/normal
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running S3 DeleteBucketNotEmpty Tests ==="
|
||||
go test -v -timeout=60s -run TestS3DeleteBucketNotEmpty ./...
|
||||
|
||||
- name: Run IAM Integration Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/normal
|
||||
|
||||
@@ -461,6 +461,8 @@ jobs:
|
||||
export S3_ENDPOINT="http://localhost:8006"
|
||||
export S3_ACCESS_KEY="0555b35654ad1656d804"
|
||||
export S3_SECRET_KEY="h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=="
|
||||
export AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY"
|
||||
export AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY"
|
||||
|
||||
# Run the specific test that is equivalent to AWS S3 tagging behavior
|
||||
make test-with-server || {
|
||||
|
||||
@@ -97,7 +97,7 @@ jobs:
|
||||
# Verify service accessibility
|
||||
echo "=== Verifying Service Accessibility ==="
|
||||
curl -f http://localhost:8080/realms/master
|
||||
curl -s http://localhost:8333
|
||||
curl -s http://localhost:8333/healthz
|
||||
echo "✅ SeaweedFS S3 API is responding (IAM-protected endpoint)"
|
||||
|
||||
# Run Keycloak-specific tests
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
set -e
|
||||
mkdir -p /tmp/data
|
||||
./weed -v=3 server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json -master.peers=none > weed.log 2>&1 &
|
||||
until curl -s http://localhost:8333/ > /dev/null; do sleep 1; done
|
||||
until curl -s http://localhost:8333/healthz > /dev/null; do sleep 1; done
|
||||
|
||||
- name: Setup Caddy
|
||||
run: |
|
||||
@@ -54,7 +54,7 @@ jobs:
|
||||
- name: Start Caddy
|
||||
run: |
|
||||
./caddy start
|
||||
until curl -fsS --insecure https://localhost:8443 > /dev/null; do sleep 1; done
|
||||
until curl -fsS --insecure https://localhost:8443/healthz > /dev/null; do sleep 1; done
|
||||
|
||||
- name: Create Bucket
|
||||
run: |
|
||||
|
||||
@@ -9,7 +9,6 @@ on:
|
||||
- 'weed/storage/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- 'rust/volume_server/**'
|
||||
- '.github/workflows/volume-server-integration-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
@@ -19,7 +18,6 @@ on:
|
||||
- 'weed/storage/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- 'rust/volume_server/**'
|
||||
- '.github/workflows/volume-server-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
@@ -122,55 +120,3 @@ jobs:
|
||||
echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
volume-server-rust-smoke:
|
||||
name: Volume Server Rust Smoke (${{ matrix.mode }})
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 40
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
mode: [exec, proxy, native]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go ${{ env.GO_VERSION }}
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
|
||||
- name: Set up Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Build SeaweedFS binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Run Rust-mode HTTP smoke tests
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
VOLUME_SERVER_IMPL: rust
|
||||
VOLUME_SERVER_RUST_MODE: ${{ matrix.mode }}
|
||||
run: |
|
||||
go test -v -count=1 -timeout=25m ./test/volume_server/http/... -run "TestAdminStatusAndHealthz|TestUploadReadRangeHeadDeleteRoundTrip"
|
||||
|
||||
- name: Run Rust-mode gRPC smoke tests
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
VOLUME_SERVER_IMPL: rust
|
||||
VOLUME_SERVER_RUST_MODE: ${{ matrix.mode }}
|
||||
run: |
|
||||
go test -v -count=1 -timeout=25m ./test/volume_server/grpc/... -run "TestStateAndStatusRPCs|TestVolumeSyncStatusAndReadVolumeFileStatus"
|
||||
|
||||
- name: Rust smoke summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## Volume Server Rust Smoke Summary (${{ matrix.mode }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Mode: Go master + Rust volume server launcher (VOLUME_SERVER_IMPL=rust, VOLUME_SERVER_RUST_MODE=${{ matrix.mode }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- HTTP Command: go test -v -count=1 -timeout=25m ./test/volume_server/http/... -run \"TestAdminStatusAndHealthz|TestUploadReadRangeHeadDeleteRoundTrip\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- gRPC Command: go test -v -count=1 -timeout=25m ./test/volume_server/grpc/... -run \"TestStateAndStatusRPCs|TestVolumeSyncStatusAndReadVolumeFileStatus\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
.PHONY: test admin-generate admin-build admin-clean admin-dev admin-run admin-test admin-fmt admin-help
|
||||
.PHONY: test admin-generate admin-build admin-clean admin-dev admin-run admin-test admin-fmt admin-help weed-commands
|
||||
|
||||
BINARY = weed
|
||||
ADMIN_DIR = weed/admin
|
||||
@@ -11,6 +11,9 @@ all: install
|
||||
install: admin-generate
|
||||
cd weed; go install
|
||||
|
||||
weed-commands:
|
||||
cd weed && $(MAKE) weed-db weed-sql
|
||||
|
||||
warp_install:
|
||||
go install github.com/minio/warp@v0.7.6
|
||||
|
||||
@@ -40,7 +43,7 @@ test: admin-generate
|
||||
# Admin component targets
|
||||
admin-generate:
|
||||
@echo "Generating admin component templates..."
|
||||
@templ generate
|
||||
@cd $(ADMIN_DIR) && templ generate ./view
|
||||
|
||||
admin-build: admin-generate
|
||||
@echo "Building admin component..."
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
package command
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
@@ -13,43 +14,12 @@ import (
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/server/postgres"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
|
||||
)
|
||||
|
||||
var (
|
||||
dbOptions DBOptions
|
||||
)
|
||||
const usageLine = "weed-db -port=5432 -master=<master_server>"
|
||||
|
||||
type DBOptions struct {
|
||||
host *string
|
||||
port *int
|
||||
masterAddr *string
|
||||
authMethod *string
|
||||
users *string
|
||||
database *string
|
||||
maxConns *int
|
||||
idleTimeout *string
|
||||
tlsCert *string
|
||||
tlsKey *string
|
||||
}
|
||||
|
||||
func init() {
|
||||
cmdDB.Run = runDB // break init cycle
|
||||
dbOptions.host = cmdDB.Flag.String("host", "localhost", "Database server host")
|
||||
dbOptions.port = cmdDB.Flag.Int("port", 5432, "Database server port")
|
||||
dbOptions.masterAddr = cmdDB.Flag.String("master", "localhost:9333", "SeaweedFS master server address")
|
||||
dbOptions.authMethod = cmdDB.Flag.String("auth", "trust", "Authentication method: trust, password, md5")
|
||||
dbOptions.users = cmdDB.Flag.String("users", "", "User credentials for auth (JSON format '{\"user1\":\"pass1\",\"user2\":\"pass2\"}' or file '@/path/to/users.json')")
|
||||
dbOptions.database = cmdDB.Flag.String("database", "default", "Default database name")
|
||||
dbOptions.maxConns = cmdDB.Flag.Int("max-connections", 100, "Maximum concurrent connections per server")
|
||||
dbOptions.idleTimeout = cmdDB.Flag.String("idle-timeout", "1h", "Connection idle timeout")
|
||||
dbOptions.tlsCert = cmdDB.Flag.String("tls-cert", "", "TLS certificate file path")
|
||||
dbOptions.tlsKey = cmdDB.Flag.String("tls-key", "", "TLS private key file path")
|
||||
}
|
||||
|
||||
var cmdDB = &Command{
|
||||
UsageLine: "db -port=5432 -master=<master_server>",
|
||||
Short: "start a PostgreSQL-compatible database server for SQL queries",
|
||||
Long: `Start a PostgreSQL wire protocol compatible database server that provides SQL query access to SeaweedFS.
|
||||
const longHelp = `Start a PostgreSQL wire protocol compatible database server that provides SQL query access to SeaweedFS.
|
||||
|
||||
This database server enables any PostgreSQL client, tool, or application to connect to SeaweedFS
|
||||
and execute SQL queries against MQ topics. It implements the PostgreSQL wire protocol for maximum
|
||||
@@ -58,25 +28,25 @@ compatibility with the existing PostgreSQL ecosystem.
|
||||
Examples:
|
||||
|
||||
# Start database server on default port 5432
|
||||
weed db
|
||||
weed-db
|
||||
|
||||
# Start with MD5 authentication using JSON format (recommended)
|
||||
weed db -auth=md5 -users='{"admin":"secret","readonly":"view123"}'
|
||||
weed-db -auth=md5 -users='{"admin":"secret","readonly":"view123"}'
|
||||
|
||||
# Start with complex passwords using JSON format
|
||||
weed db -auth=md5 -users='{"admin":"pass;with;semicolons","user":"password:with:colons"}'
|
||||
weed-db -auth=md5 -users='{"admin":"pass;with;semicolons","user":"password:with:colons"}'
|
||||
|
||||
# Start with credentials from JSON file (most secure)
|
||||
weed db -auth=md5 -users="@/etc/seaweedfs/users.json"
|
||||
weed-db -auth=md5 -users="@/etc/seaweedfs/users.json"
|
||||
|
||||
# Start with custom port and master
|
||||
weed db -port=5433 -master=master1:9333
|
||||
weed-db -port=5433 -master=master1:9333
|
||||
|
||||
# Allow connections from any host
|
||||
weed db -host=0.0.0.0 -port=5432
|
||||
weed-db -host=0.0.0.0 -port=5432
|
||||
|
||||
# Start with TLS encryption
|
||||
weed db -tls-cert=server.crt -tls-key=server.key
|
||||
weed-db -tls-cert=server.crt -tls-key=server.key
|
||||
|
||||
Client Connection Examples:
|
||||
|
||||
@@ -95,7 +65,7 @@ Programming Language Examples:
|
||||
# Python (psycopg2)
|
||||
import psycopg2
|
||||
conn = psycopg2.connect(
|
||||
host="localhost", port=5432,
|
||||
host="localhost", port=5432,
|
||||
user="seaweedfs", database="default"
|
||||
)
|
||||
|
||||
@@ -116,7 +86,7 @@ Supported SQL Operations:
|
||||
- SELECT queries on MQ topics
|
||||
- DESCRIBE/DESC table_name commands
|
||||
- EXPLAIN query execution plans
|
||||
- SHOW DATABASES/TABLES commands
|
||||
- SHOW DATABASES/TABLES commands
|
||||
- Aggregation functions (COUNT, SUM, AVG, MIN, MAX)
|
||||
- WHERE clauses with filtering
|
||||
- System columns (_timestamp_ns, _key, _source)
|
||||
@@ -149,50 +119,95 @@ Performance Features:
|
||||
- PostgreSQL wire protocol
|
||||
- Query result streaming
|
||||
|
||||
`,
|
||||
`
|
||||
|
||||
type Options struct {
|
||||
Host string
|
||||
Port int
|
||||
MasterAddr string
|
||||
AuthMethod string
|
||||
Users string
|
||||
Database string
|
||||
MaxConns int
|
||||
IdleTimeout string
|
||||
TLSCert string
|
||||
TLSKey string
|
||||
}
|
||||
|
||||
func runDB(cmd *Command, args []string) bool {
|
||||
// Run executes the weed-db CLI.
|
||||
func Run(args []string) int {
|
||||
fs := flag.NewFlagSet("weed-db", flag.ContinueOnError)
|
||||
usageWriter := io.Writer(os.Stderr)
|
||||
fs.SetOutput(usageWriter)
|
||||
|
||||
var opts Options
|
||||
fs.StringVar(&opts.Host, "host", "localhost", "Database server host")
|
||||
fs.IntVar(&opts.Port, "port", 5432, "Database server port")
|
||||
fs.StringVar(&opts.MasterAddr, "master", "localhost:9333", "SeaweedFS master server address")
|
||||
fs.StringVar(&opts.AuthMethod, "auth", "trust", "Authentication method: trust, password, md5")
|
||||
fs.StringVar(&opts.Users, "users", "", "User credentials for auth (JSON format '{\"user1\":\"pass1\",\"user2\":\"pass2\"}' or file '@/path/to/users.json')")
|
||||
fs.StringVar(&opts.Database, "database", "default", "Default database name")
|
||||
fs.IntVar(&opts.MaxConns, "max-connections", 100, "Maximum concurrent connections per server")
|
||||
fs.StringVar(&opts.IdleTimeout, "idle-timeout", "1h", "Connection idle timeout")
|
||||
fs.StringVar(&opts.TLSCert, "tls-cert", "", "TLS certificate file path")
|
||||
fs.StringVar(&opts.TLSKey, "tls-key", "", "TLS private key file path")
|
||||
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintf(usageWriter, "Usage: %s\n\n%s\n", usageLine, longHelp)
|
||||
fmt.Fprintln(usageWriter, "Default Parameters:")
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
|
||||
if !runWithOptions(&opts) {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func runWithOptions(opts *Options) bool {
|
||||
util.LoadConfiguration("security", false)
|
||||
|
||||
// Validate options
|
||||
if *dbOptions.masterAddr == "" {
|
||||
// Validate options.
|
||||
if opts.MasterAddr == "" {
|
||||
fmt.Fprintf(os.Stderr, "Error: master address is required\n")
|
||||
return false
|
||||
}
|
||||
|
||||
// Parse authentication method
|
||||
authMethod, err := parseAuthMethod(*dbOptions.authMethod)
|
||||
// Parse authentication method.
|
||||
authMethod, err := parseAuthMethod(opts.AuthMethod)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
return false
|
||||
}
|
||||
|
||||
// Parse user credentials
|
||||
users, err := parseUsers(*dbOptions.users, authMethod)
|
||||
// Parse user credentials.
|
||||
users, err := parseUsers(opts.Users, authMethod)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
return false
|
||||
}
|
||||
|
||||
// Parse idle timeout
|
||||
idleTimeout, err := time.ParseDuration(*dbOptions.idleTimeout)
|
||||
// Parse idle timeout.
|
||||
idleTimeout, err := time.ParseDuration(opts.IdleTimeout)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error parsing idle timeout: %v\n", err)
|
||||
return false
|
||||
}
|
||||
|
||||
// Validate port number
|
||||
if err := validatePortNumber(*dbOptions.port); err != nil {
|
||||
// Validate port number.
|
||||
if err := validatePortNumber(opts.Port); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
return false
|
||||
}
|
||||
|
||||
// Setup TLS if requested
|
||||
// Setup TLS if requested.
|
||||
var tlsConfig *tls.Config
|
||||
if *dbOptions.tlsCert != "" && *dbOptions.tlsKey != "" {
|
||||
cert, err := tls.LoadX509KeyPair(*dbOptions.tlsCert, *dbOptions.tlsKey)
|
||||
if opts.TLSCert != "" && opts.TLSKey != "" {
|
||||
cert, err := tls.LoadX509KeyPair(opts.TLSCert, opts.TLSKey)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error loading TLS certificates: %v\n", err)
|
||||
return false
|
||||
@@ -202,34 +217,34 @@ func runDB(cmd *Command, args []string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// Create server configuration
|
||||
// Create server configuration.
|
||||
config := &postgres.PostgreSQLServerConfig{
|
||||
Host: *dbOptions.host,
|
||||
Port: *dbOptions.port,
|
||||
Host: opts.Host,
|
||||
Port: opts.Port,
|
||||
AuthMethod: authMethod,
|
||||
Users: users,
|
||||
Database: *dbOptions.database,
|
||||
MaxConns: *dbOptions.maxConns,
|
||||
Database: opts.Database,
|
||||
MaxConns: opts.MaxConns,
|
||||
IdleTimeout: idleTimeout,
|
||||
TLSConfig: tlsConfig,
|
||||
}
|
||||
|
||||
// Create database server
|
||||
dbServer, err := postgres.NewPostgreSQLServer(config, *dbOptions.masterAddr)
|
||||
// Create database server.
|
||||
dbServer, err := postgres.NewPostgreSQLServer(config, opts.MasterAddr)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error creating database server: %v\n", err)
|
||||
return false
|
||||
}
|
||||
|
||||
// Print startup information
|
||||
// Print startup information.
|
||||
fmt.Printf("Starting SeaweedFS Database Server...\n")
|
||||
fmt.Printf("Host: %s\n", *dbOptions.host)
|
||||
fmt.Printf("Port: %d\n", *dbOptions.port)
|
||||
fmt.Printf("Master: %s\n", *dbOptions.masterAddr)
|
||||
fmt.Printf("Database: %s\n", *dbOptions.database)
|
||||
fmt.Printf("Auth Method: %s\n", *dbOptions.authMethod)
|
||||
fmt.Printf("Max Connections: %d\n", *dbOptions.maxConns)
|
||||
fmt.Printf("Idle Timeout: %s\n", *dbOptions.idleTimeout)
|
||||
fmt.Printf("Host: %s\n", opts.Host)
|
||||
fmt.Printf("Port: %d\n", opts.Port)
|
||||
fmt.Printf("Master: %s\n", opts.MasterAddr)
|
||||
fmt.Printf("Database: %s\n", opts.Database)
|
||||
fmt.Printf("Auth Method: %s\n", opts.AuthMethod)
|
||||
fmt.Printf("Max Connections: %d\n", opts.MaxConns)
|
||||
fmt.Printf("Idle Timeout: %s\n", opts.IdleTimeout)
|
||||
if tlsConfig != nil {
|
||||
fmt.Printf("TLS: Enabled\n")
|
||||
} else {
|
||||
@@ -240,15 +255,15 @@ func runDB(cmd *Command, args []string) bool {
|
||||
}
|
||||
|
||||
fmt.Printf("\nDatabase Connection Examples:\n")
|
||||
fmt.Printf(" psql -h %s -p %d -U seaweedfs -d %s\n", *dbOptions.host, *dbOptions.port, *dbOptions.database)
|
||||
fmt.Printf(" psql -h %s -p %d -U seaweedfs -d %s\n", opts.Host, opts.Port, opts.Database)
|
||||
if len(users) > 0 {
|
||||
// Show first user as example
|
||||
// Show first user as example.
|
||||
for username := range users {
|
||||
fmt.Printf(" psql -h %s -p %d -U %s -d %s\n", *dbOptions.host, *dbOptions.port, username, *dbOptions.database)
|
||||
fmt.Printf(" psql -h %s -p %d -U %s -d %s\n", opts.Host, opts.Port, username, opts.Database)
|
||||
break
|
||||
}
|
||||
}
|
||||
fmt.Printf(" postgresql://%s:%d/%s\n", *dbOptions.host, *dbOptions.port, *dbOptions.database)
|
||||
fmt.Printf(" postgresql://%s:%d/%s\n", opts.Host, opts.Port, opts.Database)
|
||||
|
||||
fmt.Printf("\nSupported Operations:\n")
|
||||
fmt.Printf(" - SELECT queries on MQ topics\n")
|
||||
@@ -261,26 +276,26 @@ func runDB(cmd *Command, args []string) bool {
|
||||
|
||||
fmt.Printf("\nReady for database connections!\n\n")
|
||||
|
||||
// Start the server
|
||||
// Start the server.
|
||||
err = dbServer.Start()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error starting database server: %v\n", err)
|
||||
return false
|
||||
}
|
||||
|
||||
// Set up signal handling for graceful shutdown
|
||||
// Set up signal handling for graceful shutdown.
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
// Wait for shutdown signal
|
||||
// Wait for shutdown signal.
|
||||
<-sigChan
|
||||
fmt.Printf("\nReceived shutdown signal, stopping database server...\n")
|
||||
|
||||
// Create context with timeout for graceful shutdown
|
||||
// Create context with timeout for graceful shutdown.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Stop the server with timeout
|
||||
// Stop the server with timeout.
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- dbServer.Stop()
|
||||
@@ -301,7 +316,7 @@ func runDB(cmd *Command, args []string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// parseAuthMethod parses the authentication method string
|
||||
// parseAuthMethod parses the authentication method string.
|
||||
func parseAuthMethod(method string) (postgres.AuthMethod, error) {
|
||||
switch strings.ToLower(method) {
|
||||
case "trust":
|
||||
@@ -315,7 +330,7 @@ func parseAuthMethod(method string) (postgres.AuthMethod, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// parseUsers parses the user credentials string with support for secure formats only
|
||||
// parseUsers parses the user credentials string with support for secure formats only.
|
||||
// Supported formats:
|
||||
// 1. JSON format: {"username":"password","username2":"password2"}
|
||||
// 2. File format: /path/to/users.json or @/path/to/users.json
|
||||
@@ -323,41 +338,41 @@ func parseUsers(usersStr string, authMethod postgres.AuthMethod) (map[string]str
|
||||
users := make(map[string]string)
|
||||
|
||||
if usersStr == "" {
|
||||
// No users specified
|
||||
// No users specified.
|
||||
if authMethod != postgres.AuthTrust {
|
||||
return nil, fmt.Errorf("users must be specified when auth method is not 'trust'")
|
||||
}
|
||||
return users, nil
|
||||
}
|
||||
|
||||
// Trim whitespace
|
||||
// Trim whitespace.
|
||||
usersStr = strings.TrimSpace(usersStr)
|
||||
|
||||
// Determine format and parse accordingly
|
||||
// Determine format and parse accordingly.
|
||||
if strings.HasPrefix(usersStr, "{") && strings.HasSuffix(usersStr, "}") {
|
||||
// JSON format
|
||||
// JSON format.
|
||||
return parseUsersJSON(usersStr, authMethod)
|
||||
}
|
||||
|
||||
// Check if it's a file path (with or without @ prefix) before declaring invalid format
|
||||
// Check if it's a file path (with or without @ prefix) before declaring invalid format.
|
||||
filePath := strings.TrimPrefix(usersStr, "@")
|
||||
if _, err := os.Stat(filePath); err == nil {
|
||||
// File format
|
||||
return parseUsersFile(usersStr, authMethod) // Pass original string to preserve @ handling
|
||||
// File format.
|
||||
return parseUsersFile(usersStr, authMethod) // Pass original string to preserve @ handling.
|
||||
}
|
||||
|
||||
// Invalid format
|
||||
// Invalid format.
|
||||
return nil, fmt.Errorf("invalid user credentials format. Use JSON format '{\"user\":\"pass\"}' or file format '@/path/to/users.json' or 'path/to/users.json'. Legacy semicolon-separated format is no longer supported")
|
||||
}
|
||||
|
||||
// parseUsersJSON parses user credentials from JSON format
|
||||
// parseUsersJSON parses user credentials from JSON format.
|
||||
func parseUsersJSON(jsonStr string, authMethod postgres.AuthMethod) (map[string]string, error) {
|
||||
var users map[string]string
|
||||
if err := json.Unmarshal([]byte(jsonStr), &users); err != nil {
|
||||
return nil, fmt.Errorf("invalid JSON format for users: %v", err)
|
||||
}
|
||||
|
||||
// Validate users
|
||||
// Validate users.
|
||||
for username, password := range users {
|
||||
if username == "" {
|
||||
return nil, fmt.Errorf("empty username in JSON user specification")
|
||||
@@ -370,12 +385,12 @@ func parseUsersJSON(jsonStr string, authMethod postgres.AuthMethod) (map[string]
|
||||
return users, nil
|
||||
}
|
||||
|
||||
// parseUsersFile parses user credentials from a JSON file
|
||||
// parseUsersFile parses user credentials from a JSON file.
|
||||
func parseUsersFile(filePath string, authMethod postgres.AuthMethod) (map[string]string, error) {
|
||||
// Remove @ prefix if present
|
||||
// Remove @ prefix if present.
|
||||
filePath = strings.TrimPrefix(filePath, "@")
|
||||
|
||||
// Read file content
|
||||
// Read file content.
|
||||
content, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read users file '%s': %v", filePath, err)
|
||||
@@ -383,16 +398,16 @@ func parseUsersFile(filePath string, authMethod postgres.AuthMethod) (map[string
|
||||
|
||||
contentStr := strings.TrimSpace(string(content))
|
||||
|
||||
// File must contain JSON format
|
||||
// File must contain JSON format.
|
||||
if !strings.HasPrefix(contentStr, "{") || !strings.HasSuffix(contentStr, "}") {
|
||||
return nil, fmt.Errorf("users file '%s' must contain JSON format: {\"user\":\"pass\"}. Legacy formats are no longer supported", filePath)
|
||||
}
|
||||
|
||||
// Parse as JSON
|
||||
// Parse as JSON.
|
||||
return parseUsersJSON(contentStr, authMethod)
|
||||
}
|
||||
|
||||
// validatePortNumber validates that the port number is reasonable
|
||||
// validatePortNumber validates that the port number is reasonable.
|
||||
func validatePortNumber(port int) error {
|
||||
if port < 1 || port > 65535 {
|
||||
return fmt.Errorf("port number must be between 1 and 65535, got %d", port)
|
||||
@@ -0,0 +1,7 @@
|
||||
package main
|
||||
|
||||
import "os"
|
||||
|
||||
func main() {
|
||||
os.Exit(Run(os.Args[1:]))
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package main
|
||||
|
||||
import "os"
|
||||
|
||||
func main() {
|
||||
os.Exit(Run(os.Args[1:]))
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package command
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -13,28 +13,24 @@ import (
|
||||
|
||||
"github.com/peterh/liner"
|
||||
"github.com/seaweedfs/seaweedfs/weed/query/engine"
|
||||
flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/grace"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/sqlutil"
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmdSql.Run = runSql
|
||||
}
|
||||
const usageLine = "weed-sql [-master=localhost:9333] [-interactive] [-file=query.sql] [-output=table|json|csv] [-database=dbname] [-query=\"SQL\"]"
|
||||
|
||||
var cmdSql = &Command{
|
||||
UsageLine: "sql [-master=localhost:9333] [-interactive] [-file=query.sql] [-output=table|json|csv] [-database=dbname] [-query=\"SQL\"]",
|
||||
Short: "advanced SQL query interface for SeaweedFS MQ topics with multiple execution modes",
|
||||
Long: `Enhanced SQL interface for SeaweedFS Message Queue topics with multiple execution modes.
|
||||
const longHelp = `Enhanced SQL interface for SeaweedFS Message Queue topics with multiple execution modes.
|
||||
|
||||
Execution Modes:
|
||||
- Interactive shell (default): weed sql -interactive
|
||||
- Single query: weed sql -query "SELECT * FROM user_events"
|
||||
- Batch from file: weed sql -file queries.sql
|
||||
- Context switching: weed sql -database analytics -interactive
|
||||
- Interactive shell (default): weed-sql -interactive
|
||||
- Single query: weed-sql -query "SELECT * FROM user_events"
|
||||
- Batch from file: weed-sql -file queries.sql
|
||||
- Context switching: weed-sql -database analytics -interactive
|
||||
|
||||
Output Formats:
|
||||
- table: ASCII table format (default for interactive)
|
||||
- json: JSON format (default for non-interactive)
|
||||
- json: JSON format (default for non-interactive)
|
||||
- csv: Comma-separated values
|
||||
|
||||
Features:
|
||||
@@ -45,24 +41,23 @@ Features:
|
||||
- Database context switching
|
||||
|
||||
Examples:
|
||||
weed sql -interactive
|
||||
weed sql -query "SHOW DATABASES" -output json
|
||||
weed sql -file batch_queries.sql -output csv
|
||||
weed sql -database analytics -query "SELECT COUNT(*) FROM metrics"
|
||||
weed sql -master broker1:9333 -interactive
|
||||
`,
|
||||
weed-sql -interactive
|
||||
weed-sql -query "SHOW DATABASES" -output json
|
||||
weed-sql -file batch_queries.sql -output csv
|
||||
weed-sql -database analytics -query "SELECT COUNT(*) FROM metrics"
|
||||
weed-sql -master broker1:9333 -interactive
|
||||
`
|
||||
|
||||
type Options struct {
|
||||
Master string
|
||||
Interactive bool
|
||||
File string
|
||||
Output string
|
||||
Database string
|
||||
Query string
|
||||
}
|
||||
|
||||
var (
|
||||
sqlMaster = cmdSql.Flag.String("master", "localhost:9333", "SeaweedFS master server HTTP address")
|
||||
sqlInteractive = cmdSql.Flag.Bool("interactive", false, "start interactive shell mode")
|
||||
sqlFile = cmdSql.Flag.String("file", "", "execute SQL queries from file")
|
||||
sqlOutput = cmdSql.Flag.String("output", "", "output format: table, json, csv (auto-detected if not specified)")
|
||||
sqlDatabase = cmdSql.Flag.String("database", "", "default database context")
|
||||
sqlQuery = cmdSql.Flag.String("query", "", "execute single SQL query")
|
||||
)
|
||||
|
||||
// OutputFormat represents different output formatting options
|
||||
// OutputFormat represents different output formatting options.
|
||||
type OutputFormat string
|
||||
|
||||
const (
|
||||
@@ -71,50 +66,82 @@ const (
|
||||
OutputCSV OutputFormat = "csv"
|
||||
)
|
||||
|
||||
// SQLContext holds the execution context for SQL operations
|
||||
// SQLContext holds the execution context for SQL operations.
|
||||
type SQLContext struct {
|
||||
engine *engine.SQLEngine
|
||||
currentDatabase string
|
||||
outputFormat OutputFormat
|
||||
interactive bool
|
||||
master string
|
||||
}
|
||||
|
||||
func runSql(command *Command, args []string) bool {
|
||||
// Initialize SQL engine with master address for service discovery
|
||||
sqlEngine := engine.NewSQLEngine(*sqlMaster)
|
||||
// Run executes the weed-sql CLI.
|
||||
func Run(args []string) int {
|
||||
fs := flag.NewFlagSet("weed-sql", flag.ContinueOnError)
|
||||
usageWriter := io.Writer(os.Stderr)
|
||||
fs.SetOutput(usageWriter)
|
||||
|
||||
// Determine execution mode and output format
|
||||
interactive := *sqlInteractive || (*sqlQuery == "" && *sqlFile == "")
|
||||
outputFormat := determineOutputFormat(*sqlOutput, interactive)
|
||||
var opts Options
|
||||
fs.StringVar(&opts.Master, "master", "localhost:9333", "SeaweedFS master server HTTP address")
|
||||
fs.BoolVar(&opts.Interactive, "interactive", false, "start interactive shell mode")
|
||||
fs.StringVar(&opts.File, "file", "", "execute SQL queries from file")
|
||||
fs.StringVar(&opts.Output, "output", "", "output format: table, json, csv (auto-detected if not specified)")
|
||||
fs.StringVar(&opts.Database, "database", "", "default database context")
|
||||
fs.StringVar(&opts.Query, "query", "", "execute single SQL query")
|
||||
|
||||
// Create SQL context
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintf(usageWriter, "Usage: %s\n\n%s\n", usageLine, longHelp)
|
||||
fmt.Fprintln(usageWriter, "Default Parameters:")
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
|
||||
if !runWithOptions(&opts) {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func runWithOptions(opts *Options) bool {
|
||||
// Initialize SQL engine with master address for service discovery.
|
||||
sqlEngine := engine.NewSQLEngine(opts.Master)
|
||||
|
||||
// Determine execution mode and output format.
|
||||
interactive := opts.Interactive || (opts.Query == "" && opts.File == "")
|
||||
outputFormat := determineOutputFormat(opts.Output, interactive)
|
||||
|
||||
// Create SQL context.
|
||||
ctx := &SQLContext{
|
||||
engine: sqlEngine,
|
||||
currentDatabase: *sqlDatabase,
|
||||
currentDatabase: opts.Database,
|
||||
outputFormat: outputFormat,
|
||||
interactive: interactive,
|
||||
master: opts.Master,
|
||||
}
|
||||
|
||||
// Set current database in SQL engine if specified via command line
|
||||
if *sqlDatabase != "" {
|
||||
ctx.engine.GetCatalog().SetCurrentDatabase(*sqlDatabase)
|
||||
// Set current database in SQL engine if specified via command line.
|
||||
if opts.Database != "" {
|
||||
ctx.engine.GetCatalog().SetCurrentDatabase(opts.Database)
|
||||
}
|
||||
|
||||
// Execute based on mode
|
||||
// Execute based on mode.
|
||||
switch {
|
||||
case *sqlQuery != "":
|
||||
// Single query mode
|
||||
return executeSingleQuery(ctx, *sqlQuery)
|
||||
case *sqlFile != "":
|
||||
// Batch file mode
|
||||
return executeFileQueries(ctx, *sqlFile)
|
||||
case opts.Query != "":
|
||||
// Single query mode.
|
||||
return executeSingleQuery(ctx, opts.Query)
|
||||
case opts.File != "":
|
||||
// Batch file mode.
|
||||
return executeFileQueries(ctx, opts.File)
|
||||
default:
|
||||
// Interactive mode
|
||||
// Interactive mode.
|
||||
return runInteractiveShell(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// determineOutputFormat selects the appropriate output format
|
||||
// determineOutputFormat selects the appropriate output format.
|
||||
func determineOutputFormat(specified string, interactive bool) OutputFormat {
|
||||
switch strings.ToLower(specified) {
|
||||
case "table":
|
||||
@@ -124,7 +151,7 @@ func determineOutputFormat(specified string, interactive bool) OutputFormat {
|
||||
case "csv":
|
||||
return OutputCSV
|
||||
default:
|
||||
// Auto-detect based on mode
|
||||
// Auto-detect based on mode.
|
||||
if interactive {
|
||||
return OutputTable
|
||||
}
|
||||
@@ -132,18 +159,18 @@ func determineOutputFormat(specified string, interactive bool) OutputFormat {
|
||||
}
|
||||
}
|
||||
|
||||
// executeSingleQuery executes a single query and outputs the result
|
||||
// executeSingleQuery executes a single query and outputs the result.
|
||||
func executeSingleQuery(ctx *SQLContext, query string) bool {
|
||||
if ctx.outputFormat != OutputTable {
|
||||
// Suppress banner for non-interactive output
|
||||
// Suppress banner for non-interactive output.
|
||||
return executeAndDisplay(ctx, query, false)
|
||||
}
|
||||
|
||||
fmt.Printf("Executing query against %s...\n", *sqlMaster)
|
||||
fmt.Printf("Executing query against %s...\n", ctx.master)
|
||||
return executeAndDisplay(ctx, query, true)
|
||||
}
|
||||
|
||||
// executeFileQueries processes SQL queries from a file
|
||||
// executeFileQueries processes SQL queries from a file.
|
||||
func executeFileQueries(ctx *SQLContext, filename string) bool {
|
||||
content, err := os.ReadFile(filename)
|
||||
if err != nil {
|
||||
@@ -152,10 +179,10 @@ func executeFileQueries(ctx *SQLContext, filename string) bool {
|
||||
}
|
||||
|
||||
if ctx.outputFormat == OutputTable && ctx.interactive {
|
||||
fmt.Printf("Executing queries from %s against %s...\n", filename, *sqlMaster)
|
||||
fmt.Printf("Executing queries from %s against %s...\n", filename, ctx.master)
|
||||
}
|
||||
|
||||
// Split file content into individual queries (robust approach)
|
||||
// Split file content into individual queries (robust approach).
|
||||
queries := sqlutil.SplitStatements(string(content))
|
||||
|
||||
for i, query := range queries {
|
||||
@@ -176,11 +203,11 @@ func executeFileQueries(ctx *SQLContext, filename string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// runInteractiveShell starts the enhanced interactive shell with readline support
|
||||
// runInteractiveShell starts the enhanced interactive shell with readline support.
|
||||
func runInteractiveShell(ctx *SQLContext) bool {
|
||||
fmt.Println("SeaweedFS Enhanced SQL Interface")
|
||||
fmt.Println("Type 'help;' for help, 'exit;' to quit")
|
||||
fmt.Printf("Connected to master: %s\n", *sqlMaster)
|
||||
fmt.Printf("Connected to master: %s\n", ctx.master)
|
||||
if ctx.currentDatabase != "" {
|
||||
fmt.Printf("Current database: %s\n", ctx.currentDatabase)
|
||||
}
|
||||
@@ -188,24 +215,24 @@ func runInteractiveShell(ctx *SQLContext) bool {
|
||||
fmt.Println("Use up/down arrows for command history")
|
||||
fmt.Println()
|
||||
|
||||
// Initialize liner for readline functionality
|
||||
// Initialize liner for readline functionality.
|
||||
line := liner.NewLiner()
|
||||
defer line.Close()
|
||||
|
||||
// Handle Ctrl+C gracefully
|
||||
// Handle Ctrl+C gracefully.
|
||||
line.SetCtrlCAborts(true)
|
||||
grace.OnInterrupt(func() {
|
||||
line.Close()
|
||||
})
|
||||
|
||||
// Load command history
|
||||
// Load command history.
|
||||
historyPath := path.Join(os.TempDir(), "weed-sql-history")
|
||||
if f, err := os.Open(historyPath); err == nil {
|
||||
line.ReadHistory(f)
|
||||
f.Close()
|
||||
}
|
||||
|
||||
// Save history on exit
|
||||
// Save history on exit.
|
||||
defer func() {
|
||||
if f, err := os.Create(historyPath); err == nil {
|
||||
line.WriteHistory(f)
|
||||
@@ -216,7 +243,7 @@ func runInteractiveShell(ctx *SQLContext) bool {
|
||||
var queryBuffer strings.Builder
|
||||
|
||||
for {
|
||||
// Show prompt with current database context
|
||||
// Show prompt with current database context.
|
||||
var prompt string
|
||||
if queryBuffer.Len() == 0 {
|
||||
if ctx.currentDatabase != "" {
|
||||
@@ -225,10 +252,10 @@ func runInteractiveShell(ctx *SQLContext) bool {
|
||||
prompt = "seaweedfs> "
|
||||
}
|
||||
} else {
|
||||
prompt = " -> " // Continuation prompt
|
||||
prompt = " -> " // Continuation prompt.
|
||||
}
|
||||
|
||||
// Read line with readline support
|
||||
// Read line with readline support.
|
||||
input, err := line.Prompt(prompt)
|
||||
if err != nil {
|
||||
if err == liner.ErrPromptAborted {
|
||||
@@ -244,30 +271,30 @@ func runInteractiveShell(ctx *SQLContext) bool {
|
||||
|
||||
lineStr := strings.TrimSpace(input)
|
||||
|
||||
// Handle empty lines
|
||||
// Handle empty lines.
|
||||
if lineStr == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Accumulate lines in query buffer
|
||||
// Accumulate lines in query buffer.
|
||||
if queryBuffer.Len() > 0 {
|
||||
queryBuffer.WriteString(" ")
|
||||
}
|
||||
queryBuffer.WriteString(lineStr)
|
||||
|
||||
// Check if we have a complete statement (ends with semicolon or special command)
|
||||
// Check if we have a complete statement (ends with semicolon or special command).
|
||||
fullQuery := strings.TrimSpace(queryBuffer.String())
|
||||
isComplete := strings.HasSuffix(lineStr, ";") ||
|
||||
isSpecialCommand(fullQuery)
|
||||
|
||||
if !isComplete {
|
||||
continue // Continue reading more lines
|
||||
continue // Continue reading more lines.
|
||||
}
|
||||
|
||||
// Add completed command to history
|
||||
// Add completed command to history.
|
||||
line.AppendHistory(fullQuery)
|
||||
|
||||
// Handle special commands (with or without semicolon)
|
||||
// Handle special commands (with or without semicolon).
|
||||
cleanQuery := strings.TrimSuffix(fullQuery, ";")
|
||||
cleanQuery = strings.TrimSpace(cleanQuery)
|
||||
|
||||
@@ -282,19 +309,19 @@ func runInteractiveShell(ctx *SQLContext) bool {
|
||||
continue
|
||||
}
|
||||
|
||||
// Handle database switching - use proper SQL parser instead of manual parsing
|
||||
// Handle database switching - use proper SQL parser instead of manual parsing.
|
||||
if strings.HasPrefix(strings.ToUpper(cleanQuery), "USE ") {
|
||||
// Execute USE statement through the SQL engine for proper parsing
|
||||
// Execute USE statement through the SQL engine for proper parsing.
|
||||
result, err := ctx.engine.ExecuteSQL(context.Background(), cleanQuery)
|
||||
if err != nil {
|
||||
fmt.Printf("Error: %v\n\n", err)
|
||||
} else if result.Error != nil {
|
||||
fmt.Printf("Error: %v\n\n", result.Error)
|
||||
} else {
|
||||
// Extract the database name from the result message for CLI context
|
||||
// Extract the database name from the result message for CLI context.
|
||||
if len(result.Rows) > 0 && len(result.Rows[0]) > 0 {
|
||||
message := result.Rows[0][0].ToString()
|
||||
// Extract database name from "Database changed to: dbname"
|
||||
// Extract database name from "Database changed to: dbname".
|
||||
if strings.HasPrefix(message, "Database changed to: ") {
|
||||
ctx.currentDatabase = strings.TrimPrefix(message, "Database changed to: ")
|
||||
}
|
||||
@@ -305,7 +332,7 @@ func runInteractiveShell(ctx *SQLContext) bool {
|
||||
continue
|
||||
}
|
||||
|
||||
// Handle output format switching
|
||||
// Handle output format switching.
|
||||
if strings.HasPrefix(strings.ToUpper(cleanQuery), "\\FORMAT ") {
|
||||
format := strings.TrimSpace(strings.TrimPrefix(strings.ToUpper(cleanQuery), "\\FORMAT "))
|
||||
switch format {
|
||||
@@ -325,22 +352,22 @@ func runInteractiveShell(ctx *SQLContext) bool {
|
||||
continue
|
||||
}
|
||||
|
||||
// Execute SQL query (without semicolon)
|
||||
// Execute SQL query (without semicolon).
|
||||
executeAndDisplay(ctx, cleanQuery, true)
|
||||
|
||||
// Reset buffer for next query
|
||||
// Reset buffer for next query.
|
||||
queryBuffer.Reset()
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// isSpecialCommand checks if a command is a special command that doesn't require semicolon
|
||||
// isSpecialCommand checks if a command is a special command that doesn't require semicolon.
|
||||
func isSpecialCommand(query string) bool {
|
||||
cleanQuery := strings.TrimSuffix(strings.TrimSpace(query), ";")
|
||||
cleanQuery = strings.ToLower(cleanQuery)
|
||||
|
||||
// Special commands that work with or without semicolon
|
||||
// Special commands that work with or without semicolon.
|
||||
specialCommands := []string{
|
||||
"exit", "quit", "\\q", "help",
|
||||
}
|
||||
@@ -351,7 +378,7 @@ func isSpecialCommand(query string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// Commands that are exactly specific commands (not just prefixes)
|
||||
// Commands that are exactly specific commands (not just prefixes).
|
||||
parts := strings.Fields(strings.ToUpper(cleanQuery))
|
||||
if len(parts) == 0 {
|
||||
return false
|
||||
@@ -360,11 +387,11 @@ func isSpecialCommand(query string) bool {
|
||||
strings.HasPrefix(strings.ToUpper(cleanQuery), "\\FORMAT ")
|
||||
}
|
||||
|
||||
// executeAndDisplay executes a query and displays the result in the specified format
|
||||
// executeAndDisplay executes a query and displays the result in the specified format.
|
||||
func executeAndDisplay(ctx *SQLContext, query string, showTiming bool) bool {
|
||||
startTime := time.Now()
|
||||
|
||||
// Execute the query
|
||||
// Execute the query.
|
||||
execCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
@@ -397,7 +424,7 @@ func executeAndDisplay(ctx *SQLContext, query string, showTiming bool) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Display results in the specified format
|
||||
// Display results in the specified format.
|
||||
switch ctx.outputFormat {
|
||||
case OutputTable:
|
||||
displayTableResult(result)
|
||||
@@ -407,8 +434,8 @@ func executeAndDisplay(ctx *SQLContext, query string, showTiming bool) bool {
|
||||
displayCSVResult(result)
|
||||
}
|
||||
|
||||
// Show execution time for interactive/table mode
|
||||
// Only show timing if there are columns or if result is truly empty
|
||||
// Show execution time for interactive/table mode.
|
||||
// Only show timing if there are columns or if result is truly empty.
|
||||
if showTiming && ctx.outputFormat == OutputTable && (len(result.Columns) > 0 || len(result.Rows) == 0) {
|
||||
elapsed := time.Since(startTime)
|
||||
fmt.Printf("\n(%d rows in set, %.3f sec)\n\n", len(result.Rows), elapsed.Seconds())
|
||||
@@ -417,20 +444,20 @@ func executeAndDisplay(ctx *SQLContext, query string, showTiming bool) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// displayTableResult formats and displays query results in ASCII table format
|
||||
// displayTableResult formats and displays query results in ASCII table format.
|
||||
func displayTableResult(result *engine.QueryResult) {
|
||||
if len(result.Columns) == 0 {
|
||||
fmt.Println("Empty result set")
|
||||
return
|
||||
}
|
||||
|
||||
// Calculate column widths for formatting
|
||||
// Calculate column widths for formatting.
|
||||
colWidths := make([]int, len(result.Columns))
|
||||
for i, col := range result.Columns {
|
||||
colWidths[i] = len(col)
|
||||
}
|
||||
|
||||
// Check data for wider columns
|
||||
// Check data for wider columns.
|
||||
for _, row := range result.Rows {
|
||||
for i, val := range row {
|
||||
if i < len(colWidths) {
|
||||
@@ -442,28 +469,28 @@ func displayTableResult(result *engine.QueryResult) {
|
||||
}
|
||||
}
|
||||
|
||||
// Print header separator
|
||||
// Print header separator.
|
||||
fmt.Print("+")
|
||||
for _, width := range colWidths {
|
||||
fmt.Print(strings.Repeat("-", width+2) + "+")
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
// Print column headers
|
||||
// Print column headers.
|
||||
fmt.Print("|")
|
||||
for i, col := range result.Columns {
|
||||
fmt.Printf(" %-*s |", colWidths[i], col)
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
// Print separator
|
||||
// Print separator.
|
||||
fmt.Print("+")
|
||||
for _, width := range colWidths {
|
||||
fmt.Print(strings.Repeat("-", width+2) + "+")
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
// Print data rows
|
||||
// Print data rows.
|
||||
for _, row := range result.Rows {
|
||||
fmt.Print("|")
|
||||
for i, val := range row {
|
||||
@@ -474,7 +501,7 @@ func displayTableResult(result *engine.QueryResult) {
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
// Print bottom separator
|
||||
// Print bottom separator.
|
||||
fmt.Print("+")
|
||||
for _, width := range colWidths {
|
||||
fmt.Print(strings.Repeat("-", width+2) + "+")
|
||||
@@ -482,16 +509,16 @@ func displayTableResult(result *engine.QueryResult) {
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
// displayJSONResult outputs query results in JSON format
|
||||
// displayJSONResult outputs query results in JSON format.
|
||||
func displayJSONResult(result *engine.QueryResult) {
|
||||
// Convert result to JSON-friendly format
|
||||
// Convert result to JSON-friendly format.
|
||||
jsonResult := map[string]interface{}{
|
||||
"columns": result.Columns,
|
||||
"rows": make([]map[string]interface{}, len(result.Rows)),
|
||||
"count": len(result.Rows),
|
||||
}
|
||||
|
||||
// Convert rows to JSON objects
|
||||
// Convert rows to JSON objects.
|
||||
for i, row := range result.Rows {
|
||||
rowObj := make(map[string]interface{})
|
||||
for j, val := range row {
|
||||
@@ -502,7 +529,7 @@ func displayJSONResult(result *engine.QueryResult) {
|
||||
jsonResult["rows"].([]map[string]interface{})[i] = rowObj
|
||||
}
|
||||
|
||||
// Marshal and print JSON
|
||||
// Marshal and print JSON.
|
||||
jsonBytes, err := json.MarshalIndent(jsonResult, "", " ")
|
||||
if err != nil {
|
||||
fmt.Printf("Error formatting JSON: %v\n", err)
|
||||
@@ -512,11 +539,11 @@ func displayJSONResult(result *engine.QueryResult) {
|
||||
fmt.Println(string(jsonBytes))
|
||||
}
|
||||
|
||||
// displayCSVResult outputs query results in CSV format
|
||||
// displayCSVResult outputs query results in CSV format.
|
||||
func displayCSVResult(result *engine.QueryResult) {
|
||||
// Handle execution plan results specially to avoid CSV quoting issues
|
||||
// Handle execution plan results specially to avoid CSV quoting issues.
|
||||
if len(result.Columns) == 1 && result.Columns[0] == "Query Execution Plan" {
|
||||
// For execution plans, output directly without CSV encoding to avoid quotes
|
||||
// For execution plans, output directly without CSV encoding to avoid quotes.
|
||||
for _, row := range result.Rows {
|
||||
if len(row) > 0 {
|
||||
fmt.Println(row[0].ToString())
|
||||
@@ -525,17 +552,17 @@ func displayCSVResult(result *engine.QueryResult) {
|
||||
return
|
||||
}
|
||||
|
||||
// Standard CSV output for regular query results
|
||||
// Standard CSV output for regular query results.
|
||||
writer := csv.NewWriter(os.Stdout)
|
||||
defer writer.Flush()
|
||||
|
||||
// Write headers
|
||||
// Write headers.
|
||||
if err := writer.Write(result.Columns); err != nil {
|
||||
fmt.Printf("Error writing CSV headers: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Write data rows
|
||||
// Write data rows.
|
||||
for _, row := range result.Rows {
|
||||
csvRow := make([]string, len(row))
|
||||
for i, val := range row {
|
||||
@@ -553,7 +580,7 @@ func showEnhancedHelp() {
|
||||
|
||||
METADATA OPERATIONS:
|
||||
SHOW DATABASES; - List all MQ namespaces
|
||||
SHOW TABLES; - List all topics in current namespace
|
||||
SHOW TABLES; - List all topics in current namespace
|
||||
SHOW TABLES FROM database; - List topics in specific namespace
|
||||
DESCRIBE table_name; - Show table schema
|
||||
|
||||
@@ -581,7 +608,7 @@ SPECIAL COMMANDS:
|
||||
|
||||
EXTENDED WHERE OPERATORS:
|
||||
=, <, >, <=, >= - Comparison operators
|
||||
!=, <> - Not equal operators
|
||||
!=, <> - Not equal operators
|
||||
LIKE 'pattern%' - Pattern matching (% = any chars, _ = single char)
|
||||
IN (value1, value2, ...) - Multi-value matching
|
||||
AND, OR - Logical operators
|
||||
@@ -20,13 +20,17 @@ if [ "$(id -u)" = "0" ]; then
|
||||
|
||||
DATA_UID=$(stat -c '%u' /data 2>/dev/null)
|
||||
DATA_GID=$(stat -c '%g' /data 2>/dev/null)
|
||||
|
||||
# Only run chown -R if ownership doesn't match (much faster for subsequent starts)
|
||||
|
||||
# Only run chown -R if ownership doesn't already match (avoids expensive
|
||||
# recursive chown on subsequent starts, and is a no-op on OpenShift when
|
||||
# fsGroup has already set correct ownership on the PVC).
|
||||
if [ "$DATA_UID" != "$SEAWEED_UID" ] || [ "$DATA_GID" != "$SEAWEED_GID" ]; then
|
||||
echo "Fixing /data ownership for seaweed user (uid=$SEAWEED_UID, gid=$SEAWEED_GID)"
|
||||
if ! chown -R seaweed:seaweed /data; then
|
||||
echo "Warning: Failed to change ownership of /data. This may cause permission errors." >&2
|
||||
echo "If /data is read-only or has mount issues, the application may fail to start." >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
# Use su-exec to drop privileges and run as seaweed user
|
||||
exec su-exec seaweed "$0" "$@"
|
||||
|
||||
@@ -42,3 +42,12 @@ services:
|
||||
- master
|
||||
- volume
|
||||
- filer
|
||||
admin:
|
||||
image: chrislusf/seaweedfs:dev # use a remote dev image
|
||||
ports:
|
||||
- 23646:23646
|
||||
command: 'admin -master=master:9333'
|
||||
depends_on:
|
||||
- master
|
||||
- volume
|
||||
- filer
|
||||
|
||||
@@ -46,7 +46,7 @@ require (
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/klauspost/compress v1.18.3
|
||||
github.com/klauspost/compress v1.18.4
|
||||
github.com/klauspost/reedsolomon v1.13.0
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/linxGnu/grocksdb v1.10.7
|
||||
@@ -94,7 +94,7 @@ require (
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.44.0
|
||||
golang.org/x/crypto v0.48.0
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546
|
||||
golang.org/x/image v0.35.0
|
||||
golang.org/x/image v0.36.0
|
||||
golang.org/x/net v0.49.0
|
||||
golang.org/x/oauth2 v0.34.0
|
||||
golang.org/x/sys v0.41.0
|
||||
@@ -109,7 +109,7 @@ require (
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/mathutil v1.7.1
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
modernc.org/sqlite v1.44.3
|
||||
modernc.org/sqlite v1.46.1
|
||||
modernc.org/strutil v1.2.1
|
||||
)
|
||||
|
||||
@@ -130,9 +130,7 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
|
||||
github.com/cognusion/imaging v1.0.2
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
github.com/getsentry/sentry-go v0.40.0
|
||||
github.com/gin-contrib/sessions v1.0.4
|
||||
github.com/gin-gonic/gin v1.11.0
|
||||
github.com/getsentry/sentry-go v0.42.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.12
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
@@ -142,15 +140,15 @@ require (
|
||||
github.com/jhump/protoreflect v1.18.0
|
||||
github.com/lib/pq v1.11.1
|
||||
github.com/linkedin/goavro/v2 v2.14.1
|
||||
github.com/mattn/go-sqlite3 v1.14.33
|
||||
github.com/mattn/go-sqlite3 v1.14.34
|
||||
github.com/minio/crc64nvme v1.1.1
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
github.com/parquet-go/parquet-go v0.26.4
|
||||
github.com/pkg/sftp v1.13.10
|
||||
github.com/rabbitmq/amqp091-go v1.10.0
|
||||
github.com/rclone/rclone v1.72.1
|
||||
github.com/rclone/rclone v1.73.1
|
||||
github.com/rdleal/intervalst v1.5.0
|
||||
github.com/redis/go-redis/v9 v9.17.2
|
||||
github.com/redis/go-redis/v9 v9.18.0
|
||||
github.com/schollz/progressbar/v3 v3.19.0
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.1
|
||||
github.com/shirou/gopsutil/v4 v4.26.1
|
||||
@@ -180,6 +178,7 @@ require (
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
|
||||
github.com/Azure/go-autorest/autorest/to v0.4.1 // indirect
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.37 // indirect
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 // indirect
|
||||
github.com/anchore/go-lzo v0.1.0 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
@@ -212,6 +211,8 @@ require (
|
||||
github.com/docker/docker v28.5.0+incompatible // indirect
|
||||
github.com/docker/go-connections v0.6.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/dromara/dongle v1.0.1 // indirect
|
||||
github.com/gin-gonic/gin v1.11.0 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.6.2 // indirect
|
||||
github.com/goccy/go-yaml v1.18.0 // indirect
|
||||
@@ -226,6 +227,7 @@ require (
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
|
||||
github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260213125353-6f59c89fcb7c // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
@@ -257,8 +259,9 @@ require (
|
||||
github.com/pierrre/geohash v1.0.0 // indirect
|
||||
github.com/pquerna/otp v1.5.0 // indirect
|
||||
github.com/pterm/pterm v0.12.81 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/quic-go/quic-go v0.57.0 // indirect
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11 // indirect
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
github.com/sasha-s/go-deadlock v0.3.1 // indirect
|
||||
@@ -269,6 +272,7 @@ require (
|
||||
github.com/substrait-io/substrait-protobuf/go v0.71.0 // indirect
|
||||
github.com/twpayne/go-geom v1.4.1 // indirect
|
||||
github.com/twpayne/go-kml v1.5.2 // indirect
|
||||
github.com/tyler-smith/go-bip39 v1.1.0 // indirect
|
||||
github.com/ulikunitz/xz v0.5.15 // indirect
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
|
||||
@@ -279,6 +283,7 @@ require (
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.7.0 // indirect
|
||||
go.uber.org/mock v0.5.2 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/mod v0.32.0 // indirect
|
||||
@@ -293,11 +298,11 @@ require (
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
cloud.google.com/go/iam v1.5.3 // indirect
|
||||
cloud.google.com/go/monitoring v1.24.2 // indirect
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
filippo.io/edwards25519 v1.1.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.1.0 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
|
||||
@@ -341,15 +346,12 @@ require (
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||
github.com/buengese/sgzip v0.1.1 // indirect
|
||||
github.com/bytedance/sonic v1.14.0 // indirect
|
||||
github.com/bytedance/sonic/loader v0.3.0 // indirect
|
||||
github.com/calebcase/tmpfile v1.0.3 // indirect
|
||||
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
|
||||
github.com/cloudflare/circl v1.6.1 // indirect
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.13.0 // indirect
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
|
||||
github.com/cloudwego/base64x v0.1.6 // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
|
||||
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
|
||||
github.com/creasty/defaults v1.8.0 // indirect
|
||||
@@ -369,8 +371,7 @@ require (
|
||||
github.com/flynn/noise v1.1.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.11 // indirect
|
||||
github.com/geoffgarside/ber v1.2.0 // indirect
|
||||
github.com/gin-contrib/sse v1.1.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.2.3 // indirect
|
||||
github.com/go-chi/chi/v5 v5.2.5 // indirect
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
@@ -389,10 +390,9 @@ require (
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
|
||||
github.com/gorilla/context v1.1.2 // indirect
|
||||
github.com/gorilla/schema v1.4.1 // indirect
|
||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||
github.com/gorilla/sessions v1.4.0 // indirect
|
||||
github.com/gorilla/sessions v1.4.0
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
@@ -402,8 +402,6 @@ require (
|
||||
github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect
|
||||
github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
|
||||
github.com/hashicorp/golang-lru v0.6.0 // indirect
|
||||
github.com/henrybear327/Proton-API-Bridge v1.0.0 // indirect
|
||||
github.com/henrybear327/go-proton-api v1.0.0 // indirect
|
||||
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
|
||||
@@ -445,7 +443,7 @@ require (
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.22
|
||||
github.com/pierrec/lz4/v4 v4.1.25
|
||||
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
|
||||
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
|
||||
github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
|
||||
@@ -476,9 +474,7 @@ require (
|
||||
github.com/tinylib/msgp v1.5.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/twmb/murmur3 v1.1.8 // indirect
|
||||
github.com/ugorji/go/codec v1.3.0 // indirect
|
||||
github.com/unknwon/goconfig v1.0.0 // indirect
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
@@ -504,7 +500,6 @@ require (
|
||||
go.opentelemetry.io/otel/trace v1.38.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
golang.org/x/arch v0.20.0 // indirect
|
||||
golang.org/x/term v0.40.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251124214823-79d6a2a48846 // indirect
|
||||
|
||||
@@ -547,8 +547,8 @@ cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcP
|
||||
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
||||
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw=
|
||||
filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8=
|
||||
git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
|
||||
@@ -569,8 +569,8 @@ github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGlu
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 h1:ZJJNFaQ86GVKQ9ehwqyAFE6pIfyicpuJ8IkVaPBc6/4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATVKqHBH9/0nOiNKk0+YcwfQ3WkK5PqHKxc8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 h1:sxgSqOB9CDToiaVFpxuvb5wGgGqWa3lCShcm5o0n3bE=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3/go.mod h1:XdED8i399lEVblYHTZM8eXaP07gv4Z58IL6ueMlVlrg=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
|
||||
@@ -596,6 +596,8 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3
|
||||
github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
|
||||
github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e h1:rd4bOvKmDIx0WeTv9Qz+hghsgyjikFiPrseXHlKepO0=
|
||||
github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e/go.mod h1:blbwPQh4DTlCZEfk1BLU4oMIhLda2U+A840Uag9DsZw=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.37 h1:W8S9TrAyZ4//3PXsU6+Bi+fe/6uIL986GyS7PVzIDL4=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.37/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264 h1:lMHTplAYI9FtmCo/QOcpRxmPA5REVAct1r2riQmDQKw=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264/go.mod h1:wGqkOzRu/ClJibvDgcfuJNAqI2nLhe8g91tPlDKRCdE=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c=
|
||||
@@ -962,6 +964,8 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
|
||||
github.com/dromara/dongle v1.0.1 h1:si/7UP/EXxnFVZok1cNos70GiMGxInAYMilHQFP5dJs=
|
||||
github.com/dromara/dongle v1.0.1/go.mod h1:ebFhTaDgxaDIKppycENTWlBsxz8mWCPWOLnsEgDpMv4=
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 h1:FT+t0UEDykcor4y3dMVKXIiWJETBpRgERYTGlmMd7HU=
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5/go.mod h1:rSS3kM9XMzSQ6pw91Qgd6yB5jdt70N4OdtrAf74As5M=
|
||||
github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI=
|
||||
@@ -989,6 +993,7 @@ github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff h1:4N8wnS3f1hNHS
|
||||
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM=
|
||||
github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
|
||||
github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
|
||||
github.com/emmansun/gmsm v0.15.5/go.mod h1:2m4jygryohSWkaSduFErgCwQKab5BNjURoFrn2DNwyU=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
@@ -1053,19 +1058,17 @@ github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj
|
||||
github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
|
||||
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
||||
github.com/getsentry/sentry-go v0.40.0 h1:VTJMN9zbTvqDqPwheRVLcp0qcUcM+8eFivvGocAaSbo=
|
||||
github.com/getsentry/sentry-go v0.40.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
|
||||
github.com/getsentry/sentry-go v0.42.0 h1:eeFMACuZTbUQf90RE8dE4tXeSe4CZyfvR1MBL7RLEt8=
|
||||
github.com/getsentry/sentry-go v0.42.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gin-contrib/sessions v1.0.4 h1:ha6CNdpYiTOK/hTp05miJLbpTSNfOnFg5Jm2kbcqy8U=
|
||||
github.com/gin-contrib/sessions v1.0.4/go.mod h1:ccmkrb2z6iU2osiAHZG3x3J4suJK+OU27oqzlWOqQgs=
|
||||
github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
|
||||
github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
|
||||
github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk=
|
||||
github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ4S3TGls2FvczZtj5Re/2ZzkV9VwqPHH/3Bo=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE=
|
||||
github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
|
||||
github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
|
||||
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68=
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo=
|
||||
github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=
|
||||
@@ -1309,8 +1312,6 @@ github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
|
||||
github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w=
|
||||
github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
|
||||
github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
|
||||
github.com/gorilla/context v1.1.2 h1:WRkNAv2uoa03QNIc1A6u4O7DAGMUVoopZhkiXWA2V1o=
|
||||
github.com/gorilla/context v1.1.2/go.mod h1:KDPwT9i/MeWHiLl90fuTgrt4/wPcv75vFAZLaOOcbxM=
|
||||
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
||||
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
|
||||
github.com/gorilla/schema v1.4.1 h1:jUg5hUjCSDZpNGLuXQOgIWGdlgrIdYvgQ0wZtdK1M3E=
|
||||
@@ -1390,10 +1391,6 @@ github.com/hashicorp/raft-boltdb/v2 v2.3.1 h1:ackhdCNPKblmOhjEU9+4lHSJYFkJd6Jqyv
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1/go.mod h1:n4S+g43dXF1tqDT+yzcXHhXM6y7MrlUd3TTwGRcUvQE=
|
||||
github.com/hashicorp/vault/api v1.22.0 h1:+HYFquE35/B74fHoIeXlZIP2YADVboaPjaSicHEZiH0=
|
||||
github.com/hashicorp/vault/api v1.22.0/go.mod h1:IUZA2cDvr4Ok3+NtK2Oq/r+lJeXkeCrHRmqdyWfpmGM=
|
||||
github.com/henrybear327/Proton-API-Bridge v1.0.0 h1:gjKAaWfKu++77WsZTHg6FUyPC5W0LTKWQciUm8PMZb0=
|
||||
github.com/henrybear327/Proton-API-Bridge v1.0.0/go.mod h1:gunH16hf6U74W2b9CGDaWRadiLICsoJ6KRkSt53zLts=
|
||||
github.com/henrybear327/go-proton-api v1.0.0 h1:zYi/IbjLwFAW7ltCeqXneUGJey0TN//Xo851a/BgLXw=
|
||||
github.com/henrybear327/go-proton-api v1.0.0/go.mod h1:w63MZuzufKcIZ93pwRgiOtxMXYafI8H74D77AxytOBc=
|
||||
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
|
||||
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
@@ -1410,6 +1407,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s=
|
||||
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf/go.mod h1:yrqSXGoD/4EKfF26AOGzscPOgTTJcyAwM2rpixWT+t4=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260213125353-6f59c89fcb7c h1:r+KtxPyrhsYeNbsfeqTfEM8xRdwgV6LuNhLZxpXecb4=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260213125353-6f59c89fcb7c/go.mod h1:vdPya4AIcDjvng4ViaAzqjegJf0VHYpYHQguFx5xBp0=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
@@ -1487,8 +1486,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
|
||||
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
|
||||
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw=
|
||||
github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
|
||||
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
@@ -1564,8 +1563,8 @@ github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhg
|
||||
github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
|
||||
github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
github.com/mattn/go-sqlite3 v1.14.33 h1:A5blZ5ulQo2AtayQ9/limgHEkFreKj1Dv226a1K73s0=
|
||||
github.com/mattn/go-sqlite3 v1.14.33/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
|
||||
github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
|
||||
@@ -1714,8 +1713,8 @@ github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2
|
||||
github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
|
||||
github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
|
||||
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
|
||||
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0=
|
||||
github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
|
||||
github.com/pierrre/compare v1.0.2 h1:k4IUsHgh+dbcAOIWCfxVa/7G6STjADH2qmhomv+1quc=
|
||||
github.com/pierrre/compare v1.0.2/go.mod h1:8UvyRHH+9HS8Pczdd2z5x/wvv67krDwVxoOndaIIDVU=
|
||||
github.com/pierrre/geohash v1.0.0 h1:f/zfjdV4rVofTCz1FhP07T+EMQAvcMM2ioGZVt+zqjI=
|
||||
@@ -1808,14 +1807,18 @@ github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc h1:zAsgcP8MhzAbhMnB1QQ2
|
||||
github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc/go.mod h1:S8xSOnV3CgpNrWd0GQ/OoQfMtlg2uPRSuTzcSGrzwK8=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/rclone/rclone v1.72.1 h1:Cc/NshKd3/TP3CC0cx9Jg9nTLG8YQ8yLYMTm6Z/LdHk=
|
||||
github.com/rclone/rclone v1.72.1/go.mod h1:QjmSgz98CjQZZJhROIeYHYjpN5kN7rTA+jtChj/+3Do=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11 h1:4MI2alxM/Ye2gIRBlYf28JGWTipZ4Zz7yAziPKrttjs=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11/go.mod h1:3HLX7dwZgvB7nt+Yl/xdzVPcargQ1yBmJEUg3n+jMKM=
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18 h1:Lc+d3ISfQaMJKWZOE7z4ZSY4RVmdzbn1B0IM8xN18qM=
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18/go.mod h1:LB2kCEaZMzNn3ocdz+qYfxXmuLxxN0ka62KJd2x53Bc=
|
||||
github.com/rclone/rclone v1.73.1 h1:FCGZUd7sXO8+nIpbuXGPkd+f23o/ZACefrHtRGDAiDM=
|
||||
github.com/rclone/rclone v1.73.1/go.mod h1:T9HeSXv/SI9qJFkYLqsjY/5ljIFWJTVmTrG7H/4JrPY=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
|
||||
github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
|
||||
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
|
||||
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
|
||||
github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
|
||||
github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
|
||||
github.com/redis/rueidis v1.0.69 h1:WlUefRhuDekji5LsD387ys3UCJtSFeBVf0e5yI0B8b4=
|
||||
github.com/redis/rueidis v1.0.69/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
|
||||
github.com/redis/rueidis/rueidiscompat v1.0.69 h1:IWVYY9lXdjNO3do2VpJT7aDFi8zbCUuQxZB6E2Grahs=
|
||||
@@ -2011,6 +2014,8 @@ github.com/twpayne/go-kml v1.5.2 h1:rFMw2/EwgkVssGS2MT6YfWSPZz6BgcJkLxQ53jnE8rQ=
|
||||
github.com/twpayne/go-kml v1.5.2/go.mod h1:kz8jAiIz6FIdU2Zjce9qGlVtgFYES9vt7BTPBHf5jl4=
|
||||
github.com/twpayne/go-polyline v1.0.0/go.mod h1:ICh24bcLYBX8CknfvNPKqoTbe+eg+MX1NPyJmSBo7pU=
|
||||
github.com/twpayne/go-waypoint v0.0.0-20200706203930-b263a7f6e4e8/go.mod h1:qj5pHncxKhu9gxtZEYWypA/z097sxhFlbTyOyt9gcnU=
|
||||
github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8=
|
||||
github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U=
|
||||
github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 h1:QEePdg0ty2r0t1+qwfZmQ4OOl/MB2UXIeJSpIZv56lg=
|
||||
github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43/go.mod h1:OYRfF6eb5wY9VRFkXJH8FFBi3plw2v+giaIu7P054pM=
|
||||
github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA=
|
||||
@@ -2221,6 +2226,7 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
|
||||
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
|
||||
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
@@ -2260,8 +2266,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
|
||||
golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.35.0 h1:LKjiHdgMtO8z7Fh18nGY6KDcoEtVfsgLDPeLyguqb7I=
|
||||
golang.org/x/image v0.35.0/go.mod h1:MwPLTVgvxSASsxdLzKrl8BRFuyqMyGhLwmC+TO1Sybk=
|
||||
golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
|
||||
golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -3047,8 +3053,8 @@ modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||
modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4=
|
||||
modernc.org/sqlite v1.44.3 h1:+39JvV/HWMcYslAwRxHb8067w+2zowvFOUrOWIy9PjY=
|
||||
modernc.org/sqlite v1.44.3/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA=
|
||||
modernc.org/sqlite v1.46.1 h1:eFJ2ShBLIEnUWlLy12raN0Z1plqmFX9Qe3rjQTKt6sU=
|
||||
modernc.org/sqlite v1.46.1/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA=
|
||||
modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
|
||||
modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
|
||||
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.12"
|
||||
appVersion: "4.13"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.0.412
|
||||
version: 4.0.413
|
||||
|
||||
@@ -212,8 +212,9 @@ To enable workers, add the following to your values.yaml:
|
||||
worker:
|
||||
enabled: true
|
||||
replicas: 2 # Scale based on workload
|
||||
capabilities: "vacuum,balance,erasure_coding" # Tasks this worker can handle
|
||||
maxConcurrent: 3 # Maximum concurrent tasks per worker
|
||||
jobType: "vacuum,volume_balance,erasure_coding" # Job types this worker can handle
|
||||
maxDetect: 1 # Maximum concurrent detection requests
|
||||
maxExecute: 4 # Maximum concurrent execution jobs per worker
|
||||
|
||||
# Working directory for task execution
|
||||
# Default: "/tmp/seaweedfs-worker"
|
||||
@@ -248,14 +249,14 @@ worker:
|
||||
memory: "2Gi"
|
||||
```
|
||||
|
||||
### Worker Capabilities
|
||||
### Worker Job Types
|
||||
|
||||
Workers can be configured with different capabilities:
|
||||
Workers can be configured with different job types:
|
||||
- **vacuum**: Reclaim deleted file space
|
||||
- **balance**: Balance volumes across volume servers
|
||||
- **volume_balance**: Balance volumes across volume servers
|
||||
- **erasure_coding**: Handle erasure coding operations
|
||||
|
||||
You can configure workers with all capabilities or create specialized worker pools with specific capabilities.
|
||||
You can configure workers with all job types or create specialized worker pools with specific job types.
|
||||
|
||||
### Worker Deployment Strategy
|
||||
|
||||
@@ -264,11 +265,11 @@ For production deployments, consider:
|
||||
1. **Multiple Workers**: Deploy 2+ worker replicas for high availability
|
||||
2. **Resource Allocation**: Workers need sufficient CPU/memory for maintenance tasks
|
||||
3. **Storage**: Workers need temporary storage for vacuum and balance operations (size depends on volume size)
|
||||
4. **Specialized Workers**: Create separate worker deployments for different capabilities if needed
|
||||
4. **Specialized Workers**: Create separate worker deployments for different job types if needed
|
||||
|
||||
Example specialized worker configuration:
|
||||
|
||||
For specialized worker pools, deploy separate Helm releases with different capabilities:
|
||||
For specialized worker pools, deploy separate Helm releases with different job types:
|
||||
|
||||
**values-worker-vacuum.yaml** (for vacuum operations):
|
||||
```yaml
|
||||
@@ -287,8 +288,8 @@ admin:
|
||||
worker:
|
||||
enabled: true
|
||||
replicas: 2
|
||||
capabilities: "vacuum"
|
||||
maxConcurrent: 2
|
||||
jobType: "vacuum"
|
||||
maxExecute: 2
|
||||
# REQUIRED: Point to the admin service of your main SeaweedFS release
|
||||
# Replace <namespace> with the namespace where your main seaweedfs is deployed
|
||||
# Example: If deploying in namespace "production":
|
||||
@@ -313,8 +314,8 @@ admin:
|
||||
worker:
|
||||
enabled: true
|
||||
replicas: 1
|
||||
capabilities: "balance"
|
||||
maxConcurrent: 1
|
||||
jobType: "volume_balance"
|
||||
maxExecute: 1
|
||||
# REQUIRED: Point to the admin service of your main SeaweedFS release
|
||||
# Replace <namespace> with the namespace where your main seaweedfs is deployed
|
||||
# Example: If deploying in namespace "production":
|
||||
@@ -323,6 +324,7 @@ worker:
|
||||
```
|
||||
|
||||
Deploy the specialized workers as separate releases:
|
||||
### Specialized Worker Deployment
|
||||
```bash
|
||||
# Deploy vacuum workers
|
||||
helm install seaweedfs-worker-vacuum seaweedfs/seaweedfs -f values-worker-vacuum.yaml
|
||||
@@ -331,6 +333,22 @@ helm install seaweedfs-worker-vacuum seaweedfs/seaweedfs -f values-worker-vacuum
|
||||
helm install seaweedfs-worker-balance seaweedfs/seaweedfs -f values-worker-balance.yaml
|
||||
```
|
||||
|
||||
## OpenShift Support
|
||||
|
||||
SeaweedFS can be deployed on OpenShift or any cluster enforcing the Kubernetes "restricted" Pod Security Standard. By default, OpenShift blocks containers that run as root or use `hostPath` volumes.
|
||||
|
||||
To deploy on OpenShift, use the provided `openshift-values.yaml` which overrides the default configuration to:
|
||||
1. Use `PersistentVolumeClaims` instead of `hostPath`.
|
||||
2. Enable `runAsNonRoot` and omit hardcoded UIDs to allow OpenShift to assign valid UIDs automatically.
|
||||
3. Apply appropriate `seccompProfile` and drop capabilities.
|
||||
|
||||
Usage:
|
||||
```bash
|
||||
helm install seaweedfs seaweedfs/seaweedfs \
|
||||
-n seaweedfs --create-namespace \
|
||||
-f openshift-values.yaml
|
||||
```
|
||||
|
||||
## Enterprise
|
||||
|
||||
For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition,
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
# openshift-values.yaml
|
||||
#
|
||||
# Example overrides for deploying SeaweedFS on OpenShift (or any cluster
|
||||
# enforcing the Kubernetes "restricted" Pod Security Standard).
|
||||
#
|
||||
# OpenShift's default "restricted" SCC blocks containers that:
|
||||
# - Run as UID 0 (root)
|
||||
# - Request privilege escalation
|
||||
# - Use hostPath volumes
|
||||
# - Omit a seccompProfile
|
||||
#
|
||||
# These overrides satisfy all four requirements by:
|
||||
# 1. Replacing hostPath volumes with PersistentVolumeClaims (or emptyDir for logs)
|
||||
# 2. Enabling runAsNonRoot: true. By omitting runAsUser, OpenShift will
|
||||
# automatically assign a valid UID from the namespace's allocated range.
|
||||
# 3. Dropping all Linux capabilities and setting allowPrivilegeEscalation: false
|
||||
# 4. Enabling RuntimeDefault seccompProfile
|
||||
#
|
||||
# Usage:
|
||||
# helm install seaweedfs seaweedfs/seaweedfs \
|
||||
# -n seaweedfs --create-namespace \
|
||||
# -f openshift-values.yaml
|
||||
#
|
||||
# Adjust storageClass and sizes to match your cluster's available StorageClasses.
|
||||
# On OpenShift you can discover them with: oc get storageclass
|
||||
|
||||
master:
|
||||
data:
|
||||
type: "persistentVolumeClaim"
|
||||
size: "10Gi"
|
||||
storageClass: "" # leave empty to use the cluster default StorageClass
|
||||
|
||||
logs:
|
||||
type: "emptyDir" # avoids hostPath; use persistentVolumeClaim if you need log persistence
|
||||
|
||||
podSecurityContext:
|
||||
enabled: true
|
||||
# On OpenShift, we omit runAsUser/runAsGroup/fsGroup to let the admission
|
||||
# controller assign them automatically based on the namespace's SCC.
|
||||
runAsNonRoot: true
|
||||
|
||||
containerSecurityContext:
|
||||
enabled: true
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
|
||||
volume:
|
||||
dataDirs:
|
||||
- name: data1
|
||||
type: "persistentVolumeClaim"
|
||||
size: "100Gi"
|
||||
storageClass: "" # leave empty to use the cluster default StorageClass
|
||||
maxVolumes: 0
|
||||
|
||||
logs:
|
||||
type: "emptyDir"
|
||||
|
||||
podSecurityContext:
|
||||
enabled: true
|
||||
# On OpenShift, we omit runAsUser/runAsGroup/fsGroup to let the admission
|
||||
# controller assign them automatically based on the namespace's SCC.
|
||||
runAsNonRoot: true
|
||||
|
||||
containerSecurityContext:
|
||||
enabled: true
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
|
||||
filer:
|
||||
data:
|
||||
type: "persistentVolumeClaim"
|
||||
size: "25Gi"
|
||||
storageClass: "" # leave empty to use the cluster default StorageClass
|
||||
|
||||
logs:
|
||||
type: "emptyDir"
|
||||
|
||||
podSecurityContext:
|
||||
enabled: true
|
||||
# On OpenShift, we omit runAsUser/runAsGroup/fsGroup to let the admission
|
||||
# controller assign them automatically based on the namespace's SCC.
|
||||
runAsNonRoot: true
|
||||
|
||||
containerSecurityContext:
|
||||
enabled: true
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
|
||||
# S3 gateway (if enabled)
|
||||
s3:
|
||||
podSecurityContext:
|
||||
enabled: true
|
||||
# On OpenShift, we omit runAsUser/runAsGroup/fsGroup to let the admission
|
||||
# controller assign them automatically based on the namespace's SCC.
|
||||
runAsNonRoot: true
|
||||
|
||||
containerSecurityContext:
|
||||
enabled: true
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
@@ -10,10 +10,21 @@
|
||||
{{- if and .Values.s3.reuseLegacySecret $existingSecret }}
|
||||
{{- $reuse = true }}
|
||||
{{- end }}
|
||||
{{- $access_key_admin := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_admin := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $access_key_read := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_read := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $creds := .Values.s3.credentials | default dict -}}
|
||||
{{- $adminCreds := $creds.admin | default dict -}}
|
||||
{{- $access_key_admin := $adminCreds.accessKey -}}
|
||||
{{- $secret_key_admin := $adminCreds.secretKey -}}
|
||||
{{- if not (and $access_key_admin $secret_key_admin) -}}
|
||||
{{- $access_key_admin = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_admin = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- end -}}
|
||||
{{- $readCreds := $creds.read | default dict -}}
|
||||
{{- $access_key_read := $readCreds.accessKey -}}
|
||||
{{- $secret_key_read := $readCreds.secretKey -}}
|
||||
{{- if not (and $access_key_read $secret_key_read) -}}
|
||||
{{- $access_key_read = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_read = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- end -}}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
type: Opaque
|
||||
@@ -43,4 +54,4 @@ stringData:
|
||||
s3_auditLogConfig.json: |
|
||||
{{ toJson .Values.s3.auditLogConfig | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -6,6 +6,7 @@ metadata:
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: sftp
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
|
||||
@@ -135,7 +135,9 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- $repositoryName := default .Values.image.repository .Values.global.repository | toString -}}
|
||||
{{- $name := .Values.global.imageName | toString -}}
|
||||
{{- $tag := default .Chart.AppVersion .Values.image.tag | toString -}}
|
||||
{{- if $repositoryName -}}
|
||||
{{- if .Values.image.repository -}}
|
||||
{{- $name = $repositoryName -}}
|
||||
{{- else if $repositoryName -}}
|
||||
{{- $name = printf "%s/%s" (trimSuffix "/" $repositoryName) (base $name) -}}
|
||||
{{- end -}}
|
||||
{{- if $registryName -}}
|
||||
|
||||
@@ -3,6 +3,10 @@ kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "seaweedfs.serviceAccountName" . }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- with .Values.global.serviceAccountAnnotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
|
||||
@@ -12,6 +12,7 @@ metadata:
|
||||
namespace: {{ $.Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" $ }}
|
||||
app.kubernetes.io/instance: {{ $.Release.Name }}
|
||||
app.kubernetes.io/component: {{ $volumeName }}
|
||||
helm.sh/chart: {{ $.Chart.Name }}-{{ $.Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ $.Release.Service }}
|
||||
|
||||
@@ -136,11 +136,15 @@ spec:
|
||||
{{- else }}
|
||||
-admin={{ template "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}:{{ .Values.admin.port }}{{ if .Values.admin.grpcPort }}.{{ .Values.admin.grpcPort }}{{ end }} \
|
||||
{{- end }}
|
||||
-capabilities={{ .Values.worker.capabilities }} \
|
||||
-maxConcurrent={{ .Values.worker.maxConcurrent }} \
|
||||
-workingDir={{ .Values.worker.workingDir }}{{- if or .Values.worker.metricsPort .Values.worker.extraArgs }} \{{ end }}
|
||||
-jobType={{ .Values.worker.jobType }} \
|
||||
-maxDetect={{ .Values.worker.maxDetect }} \
|
||||
-maxExecute={{ .Values.worker.maxExecute }} \
|
||||
-workingDir={{ .Values.worker.workingDir }}{{- if or .Values.worker.metricsPort .Values.worker.metricsIp .Values.worker.extraArgs }} \{{ end }}
|
||||
{{- if .Values.worker.metricsPort }}
|
||||
-metricsPort={{ .Values.worker.metricsPort }}{{- if .Values.worker.extraArgs }} \{{ end }}
|
||||
-metricsPort={{ .Values.worker.metricsPort }}{{- if or .Values.worker.metricsIp .Values.worker.extraArgs }} \{{ end }}
|
||||
{{- end }}
|
||||
{{- if .Values.worker.metricsIp }}
|
||||
-metricsIp={{ .Values.worker.metricsIp }}{{- if .Values.worker.extraArgs }} \{{ end }}
|
||||
{{- end }}
|
||||
{{- range $index, $arg := .Values.worker.extraArgs }}
|
||||
{{ $arg }}{{- if lt $index (sub (len $.Values.worker.extraArgs) 1) }} \{{ end }}
|
||||
|
||||
@@ -20,6 +20,7 @@ global:
|
||||
filerRead: false
|
||||
# we will use this serviceAccountName for all ClusterRoles/ClusterRoleBindings
|
||||
serviceAccountName: "seaweedfs"
|
||||
serviceAccountAnnotations: {}
|
||||
automountServiceAccountToken: true
|
||||
certificates:
|
||||
duration: 87600h
|
||||
@@ -890,6 +891,10 @@ filer:
|
||||
# set to the name of an existing kubernetes Secret with the s3 json config file
|
||||
# should have a secret key called seaweedfs_s3_config with an inline json configure
|
||||
existingConfigSecret: null
|
||||
# To provide explicit credentials for the S3 gateway, set them under
|
||||
# the top-level s3.credentials key (not filer.s3.credentials).
|
||||
# The s3-secret.yaml template only reads from .Values.s3.credentials.
|
||||
# See: s3.credentials.admin.accessKey, s3.credentials.read.accessKey
|
||||
auditLogConfig: {}
|
||||
# You may specify buckets to be created during the install or upgrade process.
|
||||
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
|
||||
@@ -918,6 +923,16 @@ s3:
|
||||
# set to the name of an existing kubernetes Secret with the s3 json config file
|
||||
# should have a secret key called seaweedfs_s3_config with an inline json config
|
||||
existingConfigSecret: null
|
||||
# Optionally provide explicit credentials for the S3 gateway.
|
||||
# When set, these are used in the generated s3 secret instead of
|
||||
# auto-generating random credentials.
|
||||
# credentials:
|
||||
# admin:
|
||||
# accessKey: ""
|
||||
# secretKey: ""
|
||||
# read:
|
||||
# accessKey: ""
|
||||
# secretKey: ""
|
||||
auditLogConfig: {}
|
||||
# You may specify buckets to be created during the install or upgrade process.
|
||||
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
|
||||
@@ -1270,17 +1285,20 @@ worker:
|
||||
replicas: 1
|
||||
loggingOverrideLevel: null
|
||||
metricsPort: 9327
|
||||
metricsIp: "" # If empty, defaults to 0.0.0.0
|
||||
|
||||
# Admin server to connect to
|
||||
adminServer: ""
|
||||
|
||||
# Worker capabilities - comma-separated list
|
||||
# Available: vacuum, balance, erasure_coding
|
||||
# Default: "vacuum,balance,erasure_coding" (all capabilities)
|
||||
capabilities: "vacuum,balance,erasure_coding"
|
||||
# Worker job types - comma-separated list
|
||||
# Available: vacuum, volume_balance, erasure_coding
|
||||
jobType: "vacuum,volume_balance,erasure_coding"
|
||||
|
||||
# Maximum number of concurrent tasks
|
||||
maxConcurrent: 3
|
||||
# Maximum number of concurrent detection requests
|
||||
maxDetect: 1
|
||||
|
||||
# Maximum number of concurrent execution jobs
|
||||
maxExecute: 4
|
||||
|
||||
# Working directory for task execution
|
||||
workingDir: "/tmp/seaweedfs-worker"
|
||||
@@ -1418,6 +1436,10 @@ allInOne:
|
||||
# Set to the name of an existing kubernetes Secret with the s3 json config file
|
||||
# should have a secret key called seaweedfs_s3_config with an inline json config
|
||||
existingConfigSecret: null
|
||||
# To provide explicit credentials for the S3 gateway, set them under
|
||||
# the top-level s3.credentials key (not allInOne.s3.credentials).
|
||||
# The s3-secret.yaml template only reads from .Values.s3.credentials.
|
||||
# See: s3.credentials.admin.accessKey, s3.credentials.read.accessKey
|
||||
auditLogConfig: null # S3 audit log configuration (null inherits from s3.auditLogConfig)
|
||||
# You may specify buckets to be created during the install process.
|
||||
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
|
||||
|
||||
Generated
-7
@@ -1,7 +0,0 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "weed-volume-rs"
|
||||
version = "0.1.0"
|
||||
@@ -1,6 +0,0 @@
|
||||
[package]
|
||||
name = "weed-volume-rs"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
@@ -1,245 +0,0 @@
|
||||
# Rust Volume Server Parity Implementation Plan
|
||||
|
||||
## Objective
|
||||
Implement a native Rust volume server that replicates Go volume-server behavior for HTTP and gRPC APIs, so it can become a drop-in replacement validated by existing integration suites.
|
||||
|
||||
## Current Focus (2026-02-16)
|
||||
- Program focus is now Rust implementation parity, not broad test expansion.
|
||||
- `test/volume_server` is treated as the parity gate.
|
||||
- Existing Rust launcher modes (`exec`, `proxy`) are transition tools; they are not the final target.
|
||||
|
||||
## Current Status
|
||||
- Rust crate and launcher are in place.
|
||||
- Integration harness can run:
|
||||
- Go master + Go volume (default)
|
||||
- Go master + Rust launcher (`VOLUME_SERVER_IMPL=rust`)
|
||||
- Rust launcher `proxy` mode has full-suite integration pass while delegating backend handlers to Go.
|
||||
- Rust launcher `native` mode is wired as the default Rust entrypoint and currently bootstraps via supervised Go backend delegation.
|
||||
- Native Rust HTTP handlers now serve control/surface paths in `native` mode:
|
||||
- `/status`, `/healthz`
|
||||
- `OPTIONS` admin/public behavior (method allow-list + CORS preflight headers)
|
||||
- `/ui/index.html` with config-driven JWT/access-ui gating
|
||||
- `/favicon.ico` and `/seaweedfsstatic/*`
|
||||
- public non-read methods (`POST`/`PUT`/`DELETE`/unsupported verbs) as `200` no-op passthrough parity
|
||||
- admin unsupported verbs as `400` parity
|
||||
- Native Rust control-surface parity now also includes:
|
||||
- `/healthz` status mirroring from backend state transitions (e.g. leave/stopping => `503`)
|
||||
- absolute-form HTTP request-target normalization before native route matching
|
||||
- Native Rust HTTP data-path prevalidation now includes:
|
||||
- early malformed vid/fid rejection (`400`) for GET/HEAD/POST/PUT fid-route shapes before delegation
|
||||
- slash-form fid-route parsing for `/{vid}/{fid}` and `/{vid}/{fid}/{filename}` with reserved-path exclusions (`/status`, `/healthz`, `/ui/index.html`, `/stats/*`, static assets)
|
||||
- write-path error prevalidation for fid routes:
|
||||
- malformed multipart form-data requests without a boundary => `400`
|
||||
- requests containing `Content-MD5` header => `400` mismatch parity branch
|
||||
- request bodies exceeding configured `-fileSizeLimitMB` (by `Content-Length`) => `400`
|
||||
- Native Rust API/storage handlers are not implemented yet.
|
||||
|
||||
## Parity Exit Criteria
|
||||
1. Native mode passes:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
2. CI runs native Rust mode integration coverage (at least smoke, then expanded shards).
|
||||
3. Rust mode defaults to native behavior for integration harness.
|
||||
4. Go-backend delegation is removed (or retained only as explicit fallback mode).
|
||||
|
||||
## Architecture Workstreams
|
||||
|
||||
### A. Runtime and Configuration Parity
|
||||
- [x] Add `native` runtime mode in `weed-volume-rs` (bootstrap delegation path).
|
||||
- [ ] Parse and honor volume-server CLI/config flags used by integration harness:
|
||||
- [ ] network/bind ports (`-ip`, `-port`, `-port.grpc`, `-port.public`)
|
||||
- [ ] master target/config dir/read mode/throttling/JWT-related config
|
||||
- [ ] size/timeout controls and maintenance state defaults
|
||||
- [ ] Implement graceful lifecycle behavior (signals, shutdown, readiness).
|
||||
|
||||
### B. Native HTTP Surface
|
||||
- [ ] Admin/control endpoints:
|
||||
- [x] `GET /status` (native Rust in `native` mode)
|
||||
- [x] `GET /healthz` (native Rust in `native` mode)
|
||||
- [x] `OPTIONS` admin/public method+CORS control behavior (native Rust in `native` mode)
|
||||
- [x] static/UI endpoints currently exercised
|
||||
- [ ] Data read path parity:
|
||||
- [ ] fid parsing/path variants
|
||||
- [ ] conditional headers (`If-Modified-Since`, `If-None-Match`)
|
||||
- [ ] range handling (single/multi/invalid)
|
||||
- [ ] deleted reads, auth checks, read-mode branches
|
||||
- [ ] chunk-manifest and compression/image transformation branches
|
||||
- [ ] Data write/delete parity:
|
||||
- [ ] write success/unchanged/error paths
|
||||
- [ ] replication and file-size-limit paths
|
||||
- [ ] delete and chunk-manifest delete branches
|
||||
- [ ] Method/CORS/public-port parity for split admin/public behavior.
|
||||
|
||||
### C. Native gRPC Surface
|
||||
- [ ] Control-plane RPCs:
|
||||
- [ ] `GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `VolumeServerLeave`
|
||||
- [ ] admin lifecycle: allocate/mount/unmount/delete/configure/readonly/writable
|
||||
- [ ] Data RPCs:
|
||||
- [ ] `ReadNeedleBlob`, `ReadNeedleMeta`, `WriteNeedleBlob`
|
||||
- [ ] `BatchDelete`, `ReadAllNeedles`
|
||||
- [ ] sync/copy/receive and status endpoints
|
||||
- [ ] Stream RPCs:
|
||||
- [ ] tail sender/receiver
|
||||
- [ ] vacuum streams
|
||||
- [ ] query streams
|
||||
- [ ] Advanced families:
|
||||
- [ ] erasure coding RPC set
|
||||
- [ ] tiering/remote fetch
|
||||
- [ ] scrub/query mode matrix
|
||||
|
||||
### D. Storage Compatibility Layer
|
||||
- [ ] Implement volume data/index handling compatible with Go on-disk format.
|
||||
- [ ] Preserve cookie/checksum/timestamp semantics used by tests.
|
||||
- [ ] Match read/write/delete consistency and error mapping behavior.
|
||||
- [ ] Ensure EC metadata/data-path compatibility with existing files.
|
||||
|
||||
### E. Operational Hardening
|
||||
- [ ] Deterministic startup/readiness and shutdown semantics.
|
||||
- [ ] Log/error parity sufficient for debugging and CI triage.
|
||||
- [ ] Concurrency/timeout behavior alignment for throttling and streams.
|
||||
- [ ] Performance baseline checks vs Go for key flows.
|
||||
|
||||
## Milestone Plan
|
||||
|
||||
### M0 (Completed): Harness + Launcher Transition
|
||||
- [x] Rust launcher integrated into harness.
|
||||
- [x] Proxy mode full-suite validation with Go backend delegation.
|
||||
|
||||
### M1: Native Skeleton (Control Plane First)
|
||||
- [ ] `native` mode boots and serves:
|
||||
- [x] `/status`, `/healthz`
|
||||
- [ ] `GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `VolumeServerLeave`
|
||||
- Gate:
|
||||
- [x] targeted HTTP/grpc control tests pass in `native` mode (delegated backend path).
|
||||
|
||||
### M2: Native Core Data Paths
|
||||
- [ ] Native HTTP read/write/delete baseline parity.
|
||||
- [ ] Native gRPC data baseline parity (`Read/WriteNeedle*`, `BatchDelete`, `ReadAllNeedles`).
|
||||
- Gate:
|
||||
- core HTTP and gRPC data suites pass in `native` mode.
|
||||
|
||||
### M3: Native Stream + Copy/Sync
|
||||
- [ ] Tail/copy/receive/sync paths in native mode.
|
||||
- Gate:
|
||||
- stream/copy families pass in `native` mode.
|
||||
|
||||
### M4: Native Advanced Feature Families
|
||||
- [ ] EC, tiering, scrub/query advanced branches.
|
||||
- Gate:
|
||||
- full `/test/volume_server/http` and `/test/volume_server/grpc` pass in `native` mode.
|
||||
|
||||
### M5: CI/Cutover
|
||||
- [x] Add/expand native-mode CI jobs (smoke matrix includes `native`).
|
||||
- [x] Make native mode default for Rust integration runs.
|
||||
- [ ] Keep `exec`/`proxy` only as explicit fallback modes during rollout.
|
||||
|
||||
## Immediate Next Steps
|
||||
1. Implement minimal native gRPC state/ping RPC handlers (`GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `VolumeServerLeave`).
|
||||
2. Expand native HTTP parity from control/static surface into data-path handlers (read/write/delete variants) while preserving current native control behavior.
|
||||
3. Keep rerunning native-mode integration suites as each delegated branch is replaced.
|
||||
4. Add mismatch triage notes for each API moved from delegation to native implementation.
|
||||
|
||||
## Risk Register
|
||||
- On-disk format mismatch risk:
|
||||
- Mitigation: implement format-level compatibility tests early (idx/dat/needle encoding).
|
||||
- Behavioral drift in edge branches:
|
||||
- Mitigation: use integration suite failures as primary truth; only add tests for newly discovered untracked branches.
|
||||
- Stream/concurrency semantic mismatch:
|
||||
- Mitigation: stabilize with focused interruption/timeout parity tests.
|
||||
|
||||
## Progress Log
|
||||
- Date: 2026-02-15
|
||||
- Change: Added Rust launcher integration (`exec`) and harness wiring.
|
||||
- Validation: Rust launcher mode passed smoke and full integration suites while delegating to Go backend.
|
||||
- Commits: `7beab85c2`, `880c2e1da`, `63d08e8a9`, `d402573ea`, `3bd20e6a1`, `6ce4d7ede`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added Rust proxy supervisor mode and validated full integration suite.
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=proxy go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=proxy go test -count=1 ./test/volume_server/grpc`
|
||||
- Commits: `a7f50d23b`, `548b3d9a3`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Re-focused plan from test expansion to native Rust implementation parity.
|
||||
- Validation basis: latest Rust proxy full-suite pass keeps regression baseline stable while native implementation starts.
|
||||
- Commits: `14c863dbf`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added native Rust launcher mode bootstrap, set Rust launcher default mode to `native`, and expanded CI Rust smoke matrix to include `native`.
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
- `env VOLUME_SERVER_IMPL=rust go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust go test -count=1 ./test/volume_server/grpc -run '^TestStateAndStatusRPCs$'`
|
||||
- Commits: `70ddbee37`, `61befd10f`, `2e65966c0`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Implemented first native Rust HTTP handlers in `native` mode for `/status` and `/healthz` on the admin listener while preserving proxy delegation for other APIs.
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
- Commits: `7e6e0261a`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Implemented native Rust `OPTIONS` handling for admin and public listeners in `native` mode, including method allow-list and origin-driven CORS response parity used by integration tests.
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestOptionsMethodsByPort$|^TestOptionsWithOriginIncludesCorsHeaders$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
- Commits: `fbff2cb39`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added native Rust UI/static/public-surface handlers in `native` mode:
|
||||
- `/ui/index.html` with config-driven JWT/access-ui gating parity
|
||||
- `/favicon.ico` and `/seaweedfsstatic/*` static endpoint parity
|
||||
- public non-read methods as `200` no-op passthrough parity
|
||||
- admin unsupported methods as `400` parity
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$|^TestUiIndexNotExposedWhenJwtSigningEnabled$|^TestUiIndexExposedWhenJwtSigningEnabledAndAccessUITrue$|^TestStaticAssetEndpoints$|^TestStaticAssetEndpointsOnPublicPort$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestOptionsMethodsByPort$|^TestOptionsWithOriginIncludesCorsHeaders$|^TestPublicPortReadOnlyMethodBehavior$|^TestCorsAndUnsupportedMethodBehavior$|^TestUnsupportedMethodTraceParity$|^TestUnsupportedMethodPropfindParity$|^TestUnsupportedMethodConnectParity$|^TestUnsupportedMethodMkcolParity$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
- Commits: `23e4497b2`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Hardened native Rust HTTP control-path parity:
|
||||
- `/healthz` now mirrors backend health status (`200`/`503`) in native mode
|
||||
- absolute-form request-target parsing (e.g. `GET http://host/path HTTP/1.1`) now normalizes to route path parity
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
|
||||
- Commits: `d6ff6ed6d`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added native malformed fid-route validation in `native` mode:
|
||||
- GET/HEAD/POST/PUT requests matching fid URL shapes now return Rust-native `400 Bad Request` for invalid vid/fid tokens
|
||||
- valid fid-route requests still delegate to backend handlers for storage/data execution paths
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestInvalidReadPathReturnsBadRequest$|^TestWriteInvalidVidAndFidReturnBadRequest$|^TestMalformedVidFidPathReturnsBadRequest$|^TestDownloadLimitInvalidVidWhileOverLimitReturnsBadRequest$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
|
||||
- Commits: `1ce0174b2`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added native Rust write-path prevalidation branches for fid routes in `native` mode:
|
||||
- multipart form uploads without boundary now return native `400`
|
||||
- `Content-MD5` requests now return native `400` mismatch branch
|
||||
- `Content-Length` over configured `-fileSizeLimitMB` now returns native `400` limit branch
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestUploadReadRangeHeadDeleteRoundTrip$|^TestWriteMalformedMultipartAndMD5Mismatch$|^TestWriteRejectsPayloadOverFileSizeLimit$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
|
||||
- Commits: `94cefd6f4`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Broadened native fid-route parser to cover slash-form URLs:
|
||||
- `/{vid}/{fid}` and `/{vid}/{fid}/{filename}` now flow through native malformed vid/fid validation
|
||||
- added explicit non-fid exclusions for control/stats/static paths to preserve route parity
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$|^TestReadPathShapesAndIfModifiedSince$|^TestMalformedVidFidPathReturnsBadRequest$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
|
||||
- Commits: `e66983e9b`
|
||||
File diff suppressed because it is too large
Load Diff
@@ -2,13 +2,11 @@ package admin_dockertest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
crand "crypto/rand"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -161,129 +159,95 @@ func TestEcEndToEnd(t *testing.T) {
|
||||
|
||||
client := &http.Client{}
|
||||
|
||||
// 1. Configure Global Maintenance (Scan Interval = 1s) via API
|
||||
t.Log("Configuring Global Maintenance via API...")
|
||||
// 1. Configure plugin job types for fast EC detection/execution.
|
||||
t.Log("Configuring plugin job types via API...")
|
||||
|
||||
// 1.1 Fetch current config
|
||||
req, _ := http.NewRequest("GET", AdminUrl+"/api/maintenance/config", nil)
|
||||
// Disable volume balance to reduce interference for this EC-focused test.
|
||||
balanceConfig := map[string]interface{}{
|
||||
"job_type": "volume_balance",
|
||||
"admin_runtime": map[string]interface{}{
|
||||
"enabled": false,
|
||||
},
|
||||
}
|
||||
jsonBody, err := json.Marshal(balanceConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to marshal volume_balance config: %v", err)
|
||||
}
|
||||
req, err := http.NewRequest("PUT", AdminUrl+"/api/plugin/job-types/volume_balance/config", bytes.NewBuffer(jsonBody))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create volume_balance config request: %v", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get global config: %v", err)
|
||||
t.Fatalf("Failed to update volume_balance config: %v", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("Failed to get global config (status %d): %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var globalConfig map[string]interface{}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&globalConfig); err != nil {
|
||||
t.Fatalf("Failed to decode global config: %v", err)
|
||||
t.Fatalf("Failed to update volume_balance config (status %d): %s", resp.StatusCode, string(body))
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
// 1.2 Modify config
|
||||
globalConfig["enabled"] = true
|
||||
globalConfig["scan_interval_seconds"] = 1
|
||||
|
||||
// Ensure policy structure exists
|
||||
if globalConfig["policy"] == nil {
|
||||
globalConfig["policy"] = map[string]interface{}{}
|
||||
ecConfig := map[string]interface{}{
|
||||
"job_type": "erasure_coding",
|
||||
"admin_runtime": map[string]interface{}{
|
||||
"enabled": true,
|
||||
"detection_interval_seconds": 1,
|
||||
"global_execution_concurrency": 4,
|
||||
"per_worker_execution_concurrency": 4,
|
||||
"max_jobs_per_detection": 100,
|
||||
},
|
||||
"worker_config_values": map[string]interface{}{
|
||||
"quiet_for_seconds": map[string]interface{}{
|
||||
"int64_value": "1",
|
||||
},
|
||||
"min_interval_seconds": map[string]interface{}{
|
||||
"int64_value": "1",
|
||||
},
|
||||
"min_size_mb": map[string]interface{}{
|
||||
"int64_value": "1",
|
||||
},
|
||||
"fullness_ratio": map[string]interface{}{
|
||||
"double_value": 0.0001,
|
||||
},
|
||||
},
|
||||
}
|
||||
policy, _ := globalConfig["policy"].(map[string]interface{})
|
||||
|
||||
// Ensure task_policies structure exists
|
||||
if policy["task_policies"] == nil {
|
||||
policy["task_policies"] = map[string]interface{}{}
|
||||
jsonBody, err = json.Marshal(ecConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to marshal erasure_coding config: %v", err)
|
||||
}
|
||||
taskPolicies, _ := policy["task_policies"].(map[string]interface{})
|
||||
|
||||
// Disable balance tasks to avoid interference with EC test
|
||||
if taskPolicies["balance"] == nil {
|
||||
taskPolicies["balance"] = map[string]interface{}{}
|
||||
req, err = http.NewRequest("PUT", AdminUrl+"/api/plugin/job-types/erasure_coding/config", bytes.NewBuffer(jsonBody))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create erasure_coding config request: %v", err)
|
||||
}
|
||||
balancePolicy, _ := taskPolicies["balance"].(map[string]interface{})
|
||||
balancePolicy["enabled"] = false
|
||||
|
||||
// Set global max concurrent
|
||||
policy["global_max_concurrent"] = 4
|
||||
globalConfig["policy"] = policy
|
||||
|
||||
// Explicitly set required fields
|
||||
requiredFields := map[string]float64{
|
||||
"worker_timeout_seconds": 300,
|
||||
"task_timeout_seconds": 7200,
|
||||
"retry_delay_seconds": 900,
|
||||
"cleanup_interval_seconds": 86400,
|
||||
"task_retention_seconds": 604800,
|
||||
"max_retries": 3,
|
||||
}
|
||||
for field, val := range requiredFields {
|
||||
if _, ok := globalConfig[field]; !ok || globalConfig[field] == 0 {
|
||||
globalConfig[field] = val
|
||||
}
|
||||
}
|
||||
|
||||
// 1.3 Update config
|
||||
jsonBody, _ := json.Marshal(globalConfig)
|
||||
req, _ = http.NewRequest("PUT", AdminUrl+"/api/maintenance/config", bytes.NewBuffer(jsonBody))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err = client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to update global config: %v", err)
|
||||
t.Fatalf("Failed to update erasure_coding config: %v", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("Failed to update global config (status %d): %s", resp.StatusCode, string(body))
|
||||
t.Fatalf("Failed to update erasure_coding config (status %d): %s", resp.StatusCode, string(body))
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
// 2. Configure EC Task (Short intervals) via Form API
|
||||
t.Log("Configuring EC Task via Form API...")
|
||||
formData := url.Values{}
|
||||
formData.Set("enabled", "true")
|
||||
formData.Set("scan_interval_seconds", "1")
|
||||
formData.Set("repeat_interval_seconds", "1")
|
||||
formData.Set("check_interval_seconds", "1")
|
||||
formData.Set("max_concurrent", "4")
|
||||
formData.Set("quiet_for_seconds_value", "1")
|
||||
formData.Set("quiet_for_seconds_unit", "seconds")
|
||||
formData.Set("min_size_mb", "1")
|
||||
formData.Set("fullness_ratio", "0.0001")
|
||||
|
||||
req, _ = http.NewRequest("POST", AdminUrl+"/maintenance/config/erasure_coding", strings.NewReader(formData.Encode()))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
resp, err = client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to update EC config: %v", err)
|
||||
}
|
||||
if resp.StatusCode != 200 && resp.StatusCode != 303 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("Failed to update EC config (status %d): %s", resp.StatusCode, string(body))
|
||||
}
|
||||
resp.Body.Close()
|
||||
t.Log("EC Task Configuration updated")
|
||||
|
||||
// 3. Restart Admin to pick up Global Config (Scan Interval)
|
||||
if len(runningCmds) > 0 {
|
||||
adminCmd := runningCmds[len(runningCmds)-1]
|
||||
t.Log("Restarting Admin Server to apply configuration...")
|
||||
stopWeed(t, adminCmd)
|
||||
time.Sleep(10 * time.Second)
|
||||
startWeed(t, "admin_restarted", "admin", "-master=localhost:9333", "-port=23646", "-port.grpc=33646", "-dataDir=./tmp/admin")
|
||||
waitForUrl(t, AdminUrl+"/health", 60)
|
||||
}
|
||||
|
||||
// 4. Upload a file
|
||||
// 2. Upload a file
|
||||
fileSize := 5 * 1024 * 1024
|
||||
data := make([]byte, fileSize)
|
||||
rand.Read(data)
|
||||
crand.Read(data)
|
||||
fileName := fmt.Sprintf("ec_test_file_%d", time.Now().Unix())
|
||||
t.Logf("Uploading %d bytes file %s to Filer...", fileSize, fileName)
|
||||
uploadUrl := FilerUrl + "/" + fileName
|
||||
|
||||
var uploadErr error
|
||||
for i := 0; i < 10; i++ {
|
||||
req, _ := http.NewRequest("PUT", uploadUrl, bytes.NewBuffer(data))
|
||||
req, err := http.NewRequest("PUT", uploadUrl, bytes.NewBuffer(data))
|
||||
if err != nil {
|
||||
uploadErr = err
|
||||
t.Logf("Upload attempt %d failed to create request: %v", i+1, err)
|
||||
time.Sleep(2 * time.Second)
|
||||
continue
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err == nil {
|
||||
if resp.StatusCode == 201 {
|
||||
@@ -306,17 +270,17 @@ func TestEcEndToEnd(t *testing.T) {
|
||||
}
|
||||
t.Log("Upload successful")
|
||||
|
||||
// 5. Verify EC Encoding
|
||||
// 3. Verify EC Encoding
|
||||
t.Log("Waiting for EC encoding (checking Master topology)...")
|
||||
startTime := time.Now()
|
||||
ecVerified := false
|
||||
var lastBody []byte
|
||||
|
||||
for time.Since(startTime) < 300*time.Second {
|
||||
// 5.1 Check Master Topology
|
||||
// 3.1 Check Master Topology
|
||||
resp, err := http.Get(MasterUrl + "/dir/status")
|
||||
if err == nil {
|
||||
lastBody, _ = ioutil.ReadAll(resp.Body)
|
||||
lastBody, _ = io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
|
||||
// Check total EC shards
|
||||
@@ -336,8 +300,8 @@ func TestEcEndToEnd(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 5.2 Debug: Check workers and tasks
|
||||
wResp, wErr := http.Get(AdminUrl + "/api/maintenance/workers")
|
||||
// 3.2 Debug: Check workers and jobs
|
||||
wResp, wErr := http.Get(AdminUrl + "/api/plugin/workers")
|
||||
workerCount := 0
|
||||
if wErr == nil {
|
||||
var workers []interface{}
|
||||
@@ -346,7 +310,7 @@ func TestEcEndToEnd(t *testing.T) {
|
||||
workerCount = len(workers)
|
||||
}
|
||||
|
||||
tResp, tErr := http.Get(AdminUrl + "/api/maintenance/tasks")
|
||||
tResp, tErr := http.Get(AdminUrl + "/api/plugin/jobs?limit=1000")
|
||||
taskCount := 0
|
||||
if tErr == nil {
|
||||
var tasks []interface{}
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
//go:build foundationdb
|
||||
// +build foundationdb
|
||||
|
||||
package foundationdb
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
//go:build foundationdb
|
||||
// +build foundationdb
|
||||
|
||||
package foundationdb
|
||||
|
||||
|
||||
@@ -19,9 +19,6 @@ services:
|
||||
start_period: 10s
|
||||
|
||||
seaweedfs-volume:
|
||||
build:
|
||||
context: ../../../docker
|
||||
dockerfile: Dockerfile.local
|
||||
image: seaweedfs:local
|
||||
container_name: seaweedfs-spark-volume
|
||||
ports:
|
||||
@@ -43,9 +40,6 @@ services:
|
||||
start_period: 10s
|
||||
|
||||
seaweedfs-filer:
|
||||
build:
|
||||
context: ../../../docker
|
||||
dockerfile: Dockerfile.local
|
||||
image: seaweedfs:local
|
||||
container_name: seaweedfs-spark-filer
|
||||
ports:
|
||||
|
||||
@@ -116,7 +116,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.avro</groupId>
|
||||
<artifactId>avro</artifactId>
|
||||
<version>1.11.4</version>
|
||||
<version>1.11.5</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Apache ZooKeeper - Fix CVEs -->
|
||||
|
||||
@@ -16,8 +16,9 @@ RUN go mod download
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Build the weed binary with Kafka gateway support
|
||||
# Build the weed binaries with Kafka gateway support
|
||||
RUN CGO_ENABLED=1 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o weed ./weed
|
||||
RUN CGO_ENABLED=1 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o weed-sql ./cmd/weed-sql
|
||||
|
||||
# Final stage
|
||||
FROM alpine:latest
|
||||
@@ -32,8 +33,9 @@ RUN addgroup -g 1000 seaweedfs && \
|
||||
# Set working directory
|
||||
WORKDIR /usr/bin
|
||||
|
||||
# Copy binary from builder
|
||||
# Copy binaries from builder
|
||||
COPY --from=builder /app/weed .
|
||||
COPY --from=builder /app/weed-sql .
|
||||
|
||||
# Create data directory
|
||||
RUN mkdir -p /data && chown seaweedfs:seaweedfs /data
|
||||
|
||||
Executable → Regular
+1
-2
@@ -14,8 +14,7 @@ sleep 2
|
||||
|
||||
echo "Querying messages..."
|
||||
cd /Users/chrislu/go/src/github.com/seaweedfs/seaweedfs/test/kafka/kafka-client-loadtest
|
||||
docker compose exec kafka-gateway /usr/local/bin/weed sql \
|
||||
docker compose exec kafka-gateway /usr/bin/weed-sql \
|
||||
-master=seaweedfs-master:9333 \
|
||||
-database=kafka \
|
||||
-query="SELECT id, timestamp, producer_id, counter, user_id, event_type FROM \"test-json-topic\" LIMIT 5;"
|
||||
|
||||
|
||||
@@ -13,8 +13,9 @@ RUN go mod download
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Build the weed binary without CGO
|
||||
# Build the weed binaries without CGO
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags "-s -w" -o weed ./weed/
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags "-s -w" -o weed-db ./cmd/weed-db
|
||||
|
||||
# Final stage - minimal runtime image
|
||||
FROM alpine:latest
|
||||
@@ -24,11 +25,12 @@ RUN apk --no-cache add ca-certificates netcat-openbsd curl
|
||||
|
||||
WORKDIR /root/
|
||||
|
||||
# Copy the weed binary from builder stage
|
||||
# Copy the binaries from builder stage
|
||||
COPY --from=builder /app/weed .
|
||||
COPY --from=builder /app/weed-db .
|
||||
|
||||
# Make it executable
|
||||
RUN chmod +x ./weed
|
||||
RUN chmod +x ./weed ./weed-db
|
||||
|
||||
# Expose ports
|
||||
EXPOSE 9333 8888 8333 8085 9533 5432
|
||||
|
||||
@@ -280,7 +280,7 @@ This test setup proves:
|
||||
- Comprehensive error handling
|
||||
|
||||
### ✅ Performance and Scalability
|
||||
- Direct SQL engine integration (same as `weed sql`)
|
||||
- Direct SQL engine integration (same as `weed-sql`)
|
||||
- No translation overhead for real queries
|
||||
- Efficient data access from stored formats
|
||||
- Scalable architecture with service discovery
|
||||
|
||||
@@ -54,7 +54,7 @@ services:
|
||||
seaweedfs:
|
||||
condition: service_healthy
|
||||
command: >
|
||||
./weed db
|
||||
./weed-db
|
||||
-host=0.0.0.0
|
||||
-port=5432
|
||||
-master=seaweedfs:9333
|
||||
|
||||
Executable → Regular
Executable → Regular
@@ -12,6 +12,7 @@ FILER_PORT := 8888
|
||||
TEST_TIMEOUT := 10m
|
||||
TEST_PATTERN := TestCORS
|
||||
SERVER_DIR := test-mini-data
|
||||
S3_CONFIG := s3_test_config.json
|
||||
|
||||
# Default target
|
||||
help:
|
||||
@@ -80,13 +81,15 @@ start-server: check-deps
|
||||
@echo "🔍 DEBUG: Creating volume directory..."
|
||||
@mkdir -p $(SERVER_DIR)
|
||||
@echo "🔍 DEBUG: Launching SeaweedFS S3 server in background..."
|
||||
@echo "🔍 DEBUG: Command: $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT) -s3.config=$(S3_CONFIG)"
|
||||
@$(WEED_BINARY) mini \
|
||||
@echo "🔍 DEBUG: Command: AWS_ACCESS_KEY_ID=some_access_key1 AWS_SECRET_ACCESS_KEY=some_secret_key1 $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT) -s3.config=$(S3_CONFIG)"
|
||||
@env AWS_ACCESS_KEY_ID=some_access_key1 \
|
||||
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
|
||||
$(WEED_BINARY) mini \
|
||||
-dir=$(SERVER_DIR) \
|
||||
-s3.port=$(S3_PORT) \
|
||||
-s3.config=$(S3_CONFIG) \
|
||||
> weed-test.log 2>&1 & \
|
||||
echo $$! > weed-test.pid
|
||||
echo $$! > weed-server.pid
|
||||
|
||||
@echo "Waiting for S3 server to be ready..."
|
||||
@for i in $$(seq 1 30); do \
|
||||
@@ -97,7 +100,7 @@ start-server: check-deps
|
||||
sleep 1; \
|
||||
done; \
|
||||
echo "S3 server failed to start"; \
|
||||
exit 1 > weed-server.pid
|
||||
exit 1
|
||||
@echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')"
|
||||
@echo "🔍 DEBUG: Checking if PID is still running..."
|
||||
@sleep 2
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "anonymous",
|
||||
"actions": [
|
||||
"Read",
|
||||
"List"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "some_access_key1",
|
||||
"secretKey": "some_secret_key1"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Admin",
|
||||
"Read",
|
||||
"List",
|
||||
"Tagging",
|
||||
"Write"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/service/iam"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -202,6 +203,116 @@ func TestIAMPolicyManagement(t *testing.T) {
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("managed_policy_crud_lifecycle", func(t *testing.T) {
|
||||
policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::*"}]}`
|
||||
|
||||
policyNames := []string{"test-managed-policy-lifecycle-a", "test-managed-policy-lifecycle-b"}
|
||||
policyArns := make([]*string, 0, len(policyNames))
|
||||
for _, policyName := range policyNames {
|
||||
createResp, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
|
||||
PolicyName: aws.String(policyName),
|
||||
PolicyDocument: aws.String(policyDoc),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
policyArns = append(policyArns, createResp.Policy.Arn)
|
||||
}
|
||||
|
||||
t.Cleanup(func() {
|
||||
for _, policyArn := range policyArns {
|
||||
_, _ = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: policyArn})
|
||||
}
|
||||
})
|
||||
|
||||
listResp, err := iamClient.ListPolicies(&iam.ListPoliciesInput{})
|
||||
require.NoError(t, err)
|
||||
|
||||
foundByName := map[string]bool{}
|
||||
for _, policy := range listResp.Policies {
|
||||
if policy.PolicyName != nil {
|
||||
foundByName[*policy.PolicyName] = true
|
||||
}
|
||||
}
|
||||
for _, policyName := range policyNames {
|
||||
assert.True(t, foundByName[policyName], "policy %s should be listed", policyName)
|
||||
}
|
||||
|
||||
getResp, err := iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: policyArns[0]})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, getResp.Policy)
|
||||
assert.Equal(t, policyNames[0], aws.StringValue(getResp.Policy.PolicyName))
|
||||
assert.Equal(t, aws.StringValue(policyArns[0]), aws.StringValue(getResp.Policy.Arn))
|
||||
|
||||
_, err = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: policyArns[0]})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: policyArns[0]})
|
||||
require.Error(t, err)
|
||||
awsErr, ok := err.(awserr.Error)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
|
||||
listAfterDeleteResp, err := iamClient.ListPolicies(&iam.ListPoliciesInput{})
|
||||
require.NoError(t, err)
|
||||
deletedPolicyFound := false
|
||||
remainingPolicyFound := false
|
||||
for _, policy := range listAfterDeleteResp.Policies {
|
||||
if policy.PolicyName == nil {
|
||||
continue
|
||||
}
|
||||
if *policy.PolicyName == policyNames[0] {
|
||||
deletedPolicyFound = true
|
||||
}
|
||||
if *policy.PolicyName == policyNames[1] {
|
||||
remainingPolicyFound = true
|
||||
}
|
||||
}
|
||||
assert.False(t, deletedPolicyFound, "deleted policy should no longer be listed")
|
||||
assert.True(t, remainingPolicyFound, "remaining policy should still be listed")
|
||||
|
||||
policyArns[0] = nil
|
||||
})
|
||||
|
||||
t.Run("managed_policy_versions", func(t *testing.T) {
|
||||
policyName := "test-managed-policy-version"
|
||||
policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:ListBucket","Resource":"*"}]}`
|
||||
|
||||
createResp, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
|
||||
PolicyName: aws.String(policyName),
|
||||
PolicyDocument: aws.String(policyDoc),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Cleanup(func() {
|
||||
_, _ = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: createResp.Policy.Arn})
|
||||
})
|
||||
|
||||
listVersionsResp, err := iamClient.ListPolicyVersions(&iam.ListPolicyVersionsInput{
|
||||
PolicyArn: createResp.Policy.Arn,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, listVersionsResp.Versions)
|
||||
assert.Equal(t, "v1", aws.StringValue(listVersionsResp.Versions[0].VersionId))
|
||||
assert.Equal(t, true, aws.BoolValue(listVersionsResp.Versions[0].IsDefaultVersion))
|
||||
|
||||
getVersionResp, err := iamClient.GetPolicyVersion(&iam.GetPolicyVersionInput{
|
||||
PolicyArn: createResp.Policy.Arn,
|
||||
VersionId: aws.String("v1"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, getVersionResp.PolicyVersion)
|
||||
assert.Equal(t, "v1", aws.StringValue(getVersionResp.PolicyVersion.VersionId))
|
||||
assert.Contains(t, aws.StringValue(getVersionResp.PolicyVersion.Document), "s3:ListBucket")
|
||||
|
||||
_, err = iamClient.GetPolicyVersion(&iam.GetPolicyVersionInput{
|
||||
PolicyArn: createResp.Policy.Arn,
|
||||
VersionId: aws.String("v2"),
|
||||
})
|
||||
require.Error(t, err)
|
||||
awsErr, ok := err.(awserr.Error)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
})
|
||||
|
||||
t.Run("user_inline_policy", func(t *testing.T) {
|
||||
userName := "test-user-policy"
|
||||
_, err := iamClient.CreateUser(&iam.CreateUserInput{
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package example
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -22,10 +21,8 @@ func TestIAMOperations(t *testing.T) {
|
||||
// Set credentials before starting cluster
|
||||
accessKey := "testkey123"
|
||||
secretKey := "testsecret456"
|
||||
os.Setenv("AWS_ACCESS_KEY_ID", accessKey)
|
||||
os.Setenv("AWS_SECRET_ACCESS_KEY", secretKey)
|
||||
defer os.Unsetenv("AWS_ACCESS_KEY_ID")
|
||||
defer os.Unsetenv("AWS_SECRET_ACCESS_KEY")
|
||||
t.Setenv("AWS_ACCESS_KEY_ID", accessKey)
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", secretKey)
|
||||
|
||||
// Create and start test cluster
|
||||
cluster, err := startMiniCluster(t)
|
||||
|
||||
@@ -3,6 +3,8 @@ package example
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net"
|
||||
@@ -15,6 +17,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
@@ -68,6 +71,22 @@ func TestS3Integration(t *testing.T) {
|
||||
testPutObject(t, cluster)
|
||||
})
|
||||
|
||||
t.Run("UploadPart", func(t *testing.T) {
|
||||
testPutPartWithChecksum(t, cluster)
|
||||
})
|
||||
|
||||
t.Run("PutObjectWithChecksum", func(t *testing.T) {
|
||||
testPutObjectWithChecksum(t, cluster)
|
||||
})
|
||||
|
||||
t.Run("UploadPartWithChecksum", func(t *testing.T) {
|
||||
testUploadPartWithChecksum(t, cluster)
|
||||
})
|
||||
|
||||
t.Run("PutObjectWithChecksumAndSSEC", func(t *testing.T) {
|
||||
testPutObjectWithChecksumAndSSEC(t, cluster)
|
||||
})
|
||||
|
||||
t.Run("GetObject", func(t *testing.T) {
|
||||
testGetObject(t, cluster)
|
||||
})
|
||||
@@ -97,8 +116,9 @@ func findAvailablePort() (int, error) {
|
||||
return addr.Port, nil
|
||||
}
|
||||
|
||||
// startMiniCluster starts a weed mini instance directly without exec
|
||||
func startMiniCluster(t *testing.T) (*TestCluster, error) {
|
||||
// startMiniCluster starts a weed mini instance directly without exec.
|
||||
// Extra flags (e.g. "-s3.allowDeleteBucketNotEmpty=false") can be appended via extraArgs.
|
||||
func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) {
|
||||
// Find available ports
|
||||
masterPort, err := findAvailablePort()
|
||||
if err != nil {
|
||||
@@ -146,6 +166,14 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
|
||||
return nil, fmt.Errorf("failed to create security.toml: %v", err)
|
||||
}
|
||||
|
||||
// Set environment variables for admin credentials safely for this test
|
||||
if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
|
||||
t.Setenv("AWS_ACCESS_KEY_ID", "admin")
|
||||
}
|
||||
if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
|
||||
}
|
||||
|
||||
// Start weed mini in a goroutine by calling the command directly
|
||||
cluster.wg.Add(1)
|
||||
go func() {
|
||||
@@ -165,7 +193,7 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
|
||||
// Configure args for mini command
|
||||
// Note: When running via 'go test', os.Args[0] is the test binary
|
||||
// We need to make it look like we're running 'weed mini'
|
||||
os.Args = []string{
|
||||
os.Args = append([]string{
|
||||
"weed",
|
||||
"-dir=" + testDir,
|
||||
"-master.port=" + strconv.Itoa(masterPort),
|
||||
@@ -178,7 +206,7 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
|
||||
"-ip=127.0.0.1",
|
||||
"-master.peers=none", // Faster startup
|
||||
"-s3.iam.readOnly=false", // Enable IAM write operations for tests
|
||||
}
|
||||
}, extraArgs...)
|
||||
|
||||
// Suppress most logging during tests
|
||||
glog.MaxSize = 1024 * 1024
|
||||
@@ -336,6 +364,282 @@ func testPutObject(t *testing.T, cluster *TestCluster) {
|
||||
t.Logf("✓ Put object: %s/%s (%d bytes)", bucketName, objectKey, len(objectData))
|
||||
}
|
||||
|
||||
func createTestBucket(t *testing.T, cluster *TestCluster, prefix string) string {
|
||||
bucketName := prefix + randomString(8)
|
||||
_, err := cluster.s3Client.CreateBucket(&s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
return bucketName
|
||||
}
|
||||
|
||||
// generateSSECKey returns a 32-byte key as a raw string (what the SDK expects
|
||||
// for SSECustomerKey) and its base64-encoded MD5 (for SSECustomerKeyMD5).
|
||||
func generateSSECKey() (keyRaw, keyMD5B64 string) {
|
||||
rng := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
key := make([]byte, 32)
|
||||
for i := range key {
|
||||
key[i] = byte(rng.Intn(256))
|
||||
}
|
||||
keyRaw = string(key)
|
||||
keyHash := md5.Sum(key)
|
||||
keyMD5B64 = base64.StdEncoding.EncodeToString(keyHash[:])
|
||||
return
|
||||
}
|
||||
|
||||
func testPutObjectWithChecksum(t *testing.T, cluster *TestCluster) {
|
||||
bucketName := createTestBucket(t, cluster, "test-put-checksum-")
|
||||
objectKey := "test-checksummed-object.txt"
|
||||
objectData := "Hello, SeaweedFS S3!"
|
||||
|
||||
correctMD5 := calculateMd5(objectData)
|
||||
incorrectMD5 := calculateMd5(objectData + "incorrect")
|
||||
|
||||
// Put object with incorrect MD5 should be rejected
|
||||
_, err := cluster.s3Client.PutObject(&s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader([]byte(objectData)),
|
||||
ContentMD5: aws.String(incorrectMD5),
|
||||
})
|
||||
assertBadDigestError(t, err, "PutObject should fail with incorrect MD5")
|
||||
|
||||
t.Logf("✓ Put object with incorrect MD5 rejected: %s/%s", bucketName, objectKey)
|
||||
|
||||
// Put object with correct MD5 should succeed
|
||||
_, err = cluster.s3Client.PutObject(&s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader([]byte(objectData)),
|
||||
ContentMD5: aws.String(correctMD5),
|
||||
})
|
||||
require.NoError(t, err, "Failed to put object")
|
||||
|
||||
// Verify object exists
|
||||
headResp, err := cluster.s3Client.HeadObject(&s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, headResp.ContentLength)
|
||||
assert.Equal(t, int64(len(objectData)), aws.Int64Value(headResp.ContentLength))
|
||||
|
||||
t.Logf("✓ Put object with correct MD5: %s/%s (%d bytes)", bucketName, objectKey, len(objectData))
|
||||
}
|
||||
|
||||
// putObjectSSEC sends a PutObject request with SSE-C headers over HTTP.
|
||||
// The AWS SDK v1 refuses to send SSE-C keys over plain HTTP, so we use the
|
||||
// low-level Request API and clear the Validate handlers to bypass that check.
|
||||
// We use Clear() because the specific validator is internal and not easily removable by name.
|
||||
func putObjectSSEC(client *s3.S3, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
|
||||
req, output := client.PutObjectRequest(input)
|
||||
req.Handlers.Validate.Clear()
|
||||
err := req.Send()
|
||||
return output, err
|
||||
}
|
||||
|
||||
func headObjectSSEC(client *s3.S3, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
|
||||
req, output := client.HeadObjectRequest(input)
|
||||
req.Handlers.Validate.Clear()
|
||||
err := req.Send()
|
||||
return output, err
|
||||
}
|
||||
|
||||
func testPutObjectWithChecksumAndSSEC(t *testing.T, cluster *TestCluster) {
|
||||
bucketName := createTestBucket(t, cluster, "test-put-checksum-ssec-")
|
||||
objectKey := "test-checksummed-ssec-object.txt"
|
||||
objectData := "Hello, SeaweedFS S3 with SSE-C!"
|
||||
|
||||
correctMD5 := calculateMd5(objectData)
|
||||
incorrectMD5 := calculateMd5(objectData + "incorrect")
|
||||
keyRaw, keyMD5B64 := generateSSECKey()
|
||||
|
||||
// Put object with SSE-C and incorrect MD5 should be rejected
|
||||
_, err := putObjectSSEC(cluster.s3Client, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader([]byte(objectData)),
|
||||
ContentMD5: aws.String(incorrectMD5),
|
||||
SSECustomerAlgorithm: aws.String("AES256"),
|
||||
SSECustomerKey: aws.String(keyRaw),
|
||||
SSECustomerKeyMD5: aws.String(keyMD5B64),
|
||||
})
|
||||
assertBadDigestError(t, err, "PutObject with SSE-C should fail with incorrect MD5")
|
||||
|
||||
t.Logf("Put object with SSE-C and incorrect MD5 rejected: %s/%s", bucketName, objectKey)
|
||||
|
||||
// Put object with SSE-C and correct MD5 should succeed
|
||||
_, err = putObjectSSEC(cluster.s3Client, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader([]byte(objectData)),
|
||||
ContentMD5: aws.String(correctMD5),
|
||||
SSECustomerAlgorithm: aws.String("AES256"),
|
||||
SSECustomerKey: aws.String(keyRaw),
|
||||
SSECustomerKeyMD5: aws.String(keyMD5B64),
|
||||
})
|
||||
require.NoError(t, err, "Failed to put object with SSE-C and correct MD5")
|
||||
|
||||
// Verify object exists (SSE-C requires the key for HeadObject too)
|
||||
headResp, err := headObjectSSEC(cluster.s3Client, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
SSECustomerAlgorithm: aws.String("AES256"),
|
||||
SSECustomerKey: aws.String(keyRaw),
|
||||
SSECustomerKeyMD5: aws.String(keyMD5B64),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, headResp.ContentLength)
|
||||
assert.Equal(t, int64(len(objectData)), aws.Int64Value(headResp.ContentLength))
|
||||
|
||||
t.Logf("Put object with SSE-C and correct MD5: %s/%s (%d bytes)", bucketName, objectKey, len(objectData))
|
||||
}
|
||||
|
||||
func testUploadPartWithChecksum(t *testing.T, cluster *TestCluster) {
|
||||
bucketName := createTestBucket(t, cluster, "test-upload-part-checksum-")
|
||||
objectKey := "test-multipart-checksum.txt"
|
||||
objectData := "Hello, SeaweedFS S3 Multipart!"
|
||||
|
||||
// Initiate multipart upload
|
||||
initResp, err := cluster.s3Client.CreateMultipartUpload(&s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
uploadID := initResp.UploadId
|
||||
|
||||
correctMD5 := calculateMd5(objectData)
|
||||
incorrectMD5 := calculateMd5(objectData + "incorrect")
|
||||
|
||||
// Upload part with incorrect MD5
|
||||
_, err = cluster.s3Client.UploadPart(&s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
PartNumber: aws.Int64(1),
|
||||
UploadId: uploadID,
|
||||
Body: bytes.NewReader([]byte(objectData)),
|
||||
ContentMD5: aws.String(incorrectMD5),
|
||||
})
|
||||
assertBadDigestError(t, err, "UploadPart should fail with incorrect MD5")
|
||||
|
||||
// Upload part with correct MD5
|
||||
partResp, err := cluster.s3Client.UploadPart(&s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
PartNumber: aws.Int64(1),
|
||||
UploadId: uploadID,
|
||||
Body: bytes.NewReader([]byte(objectData)),
|
||||
ContentMD5: aws.String(correctMD5),
|
||||
})
|
||||
require.NoError(t, err, "Failed to upload part with correct MD5")
|
||||
|
||||
// Complete multipart upload
|
||||
_, err = cluster.s3Client.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: uploadID,
|
||||
MultipartUpload: &s3.CompletedMultipartUpload{
|
||||
Parts: []*s3.CompletedPart{
|
||||
{
|
||||
ETag: partResp.ETag,
|
||||
PartNumber: aws.Int64(1),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "Failed to complete multipart upload")
|
||||
|
||||
// Verify object exists
|
||||
headResp, err := cluster.s3Client.HeadObject(&s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(len(objectData)), aws.Int64Value(headResp.ContentLength))
|
||||
|
||||
t.Logf("✓ Multipart upload with checksum successful: %s/%s", bucketName, objectKey)
|
||||
}
|
||||
|
||||
func testPutPartWithChecksum(t *testing.T, cluster *TestCluster) {
|
||||
bucketName := createTestBucket(t, cluster, "test-put-checksum-")
|
||||
objectKey := "test-checksummed-part.txt"
|
||||
|
||||
partData := "Hello, SeaweedFS S3!"
|
||||
|
||||
correctMD5 := calculateMd5(partData)
|
||||
incorrectMD5 := calculateMd5(partData + "incorrect")
|
||||
|
||||
createResp, err := cluster.s3Client.CreateMultipartUpload(&s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
uploadID := createResp.UploadId
|
||||
|
||||
partBody := []byte(partData)
|
||||
|
||||
_, err = cluster.s3Client.UploadPart(&s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: uploadID,
|
||||
PartNumber: aws.Int64(1),
|
||||
Body: bytes.NewReader(partBody),
|
||||
ContentMD5: aws.String(incorrectMD5),
|
||||
})
|
||||
assertBadDigestError(t, err, "UploadPart should fail with incorrect MD5")
|
||||
|
||||
uploadResp, err := cluster.s3Client.UploadPart(&s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: uploadID,
|
||||
PartNumber: aws.Int64(1),
|
||||
Body: bytes.NewReader(partBody),
|
||||
ContentMD5: aws.String(correctMD5),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = cluster.s3Client.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: uploadID,
|
||||
MultipartUpload: &s3.CompletedMultipartUpload{
|
||||
Parts: []*s3.CompletedPart{
|
||||
{
|
||||
ETag: uploadResp.ETag,
|
||||
PartNumber: aws.Int64(1),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "Failed to complete multipart upload")
|
||||
|
||||
// Verify object exists
|
||||
headResp, err := cluster.s3Client.HeadObject(&s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(len(partData)), aws.Int64Value(headResp.ContentLength))
|
||||
|
||||
t.Logf("✓ UploadPart with MD5 validation: %s/%s", bucketName, objectKey)
|
||||
}
|
||||
|
||||
func calculateMd5(objectData string) string {
|
||||
dataBytes := []byte(objectData)
|
||||
hash := md5.Sum(dataBytes)
|
||||
return base64.StdEncoding.EncodeToString(hash[:])
|
||||
}
|
||||
|
||||
func assertBadDigestError(t *testing.T, err error, description string) {
|
||||
require.Error(t, err, description)
|
||||
|
||||
var awsErr awserr.Error
|
||||
require.ErrorAs(t, err, &awsErr)
|
||||
assert.Equal(t, "BadDigest", awsErr.Code())
|
||||
}
|
||||
|
||||
func testGetObject(t *testing.T, cluster *TestCluster) {
|
||||
bucketName := "test-get-" + randomString(8)
|
||||
objectKey := "test-data.txt"
|
||||
@@ -476,6 +780,49 @@ func testDeleteBucket(t *testing.T, cluster *TestCluster) {
|
||||
t.Logf("✓ Deleted bucket: %s", bucketName)
|
||||
}
|
||||
|
||||
func TestS3DeleteBucketNotEmpty(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster, err := startMiniCluster(t, "-s3.allowDeleteBucketNotEmpty=false")
|
||||
require.NoError(t, err)
|
||||
defer cluster.Stop()
|
||||
|
||||
t.Run("DeleteNonEmptyBucketFails", func(t *testing.T) {
|
||||
bucketName := createTestBucket(t, cluster, "test-notempty-")
|
||||
objectKey := "keep-me.txt"
|
||||
|
||||
// Put an object so the bucket is non-empty
|
||||
_, err := cluster.s3Client.PutObject(&s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader([]byte("data")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Attempt to delete the non-empty bucket — must fail with BucketNotEmpty (409)
|
||||
_, err = cluster.s3Client.DeleteBucket(&s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.Error(t, err, "deleting a non-empty bucket should fail")
|
||||
var awsErr awserr.Error
|
||||
require.ErrorAs(t, err, &awsErr)
|
||||
assert.Equal(t, "BucketNotEmpty", awsErr.Code(),
|
||||
"expected BucketNotEmpty error code, got %s: %s", awsErr.Code(), awsErr.Message())
|
||||
})
|
||||
|
||||
t.Run("DeleteEmptyBucketSucceeds", func(t *testing.T) {
|
||||
bucketName := createTestBucket(t, cluster, "test-empty-")
|
||||
|
||||
// Delete the empty bucket — should succeed even with the flag
|
||||
_, err := cluster.s3Client.DeleteBucket(&s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err, "deleting an empty bucket should succeed")
|
||||
})
|
||||
}
|
||||
|
||||
// randomString generates a random string for unique naming
|
||||
func randomString(length int) string {
|
||||
const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
|
||||
|
||||
@@ -2,6 +2,7 @@ package policy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -14,6 +15,11 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/iam"
|
||||
"github.com/seaweedfs/seaweedfs/weed/command"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
@@ -130,6 +136,151 @@ func TestS3PolicyShellRevised(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3IAMAttachDetachUserPolicy(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster, err := startMiniCluster(t)
|
||||
require.NoError(t, err)
|
||||
defer cluster.Stop()
|
||||
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
policyName := uniqueName("managed-policy")
|
||||
policyArn := fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)
|
||||
policyContent := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"*","Resource":"*"}]}`
|
||||
tmpPolicyFile, err := os.CreateTemp("", "test_policy_attach_*.json")
|
||||
require.NoError(t, err)
|
||||
defer os.Remove(tmpPolicyFile.Name())
|
||||
_, err = tmpPolicyFile.WriteString(policyContent)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, tmpPolicyFile.Close())
|
||||
|
||||
weedCmd := "weed"
|
||||
masterAddr := string(pb.NewServerAddress("127.0.0.1", cluster.masterPort, cluster.masterGrpcPort))
|
||||
filerAddr := string(pb.NewServerAddress("127.0.0.1", cluster.filerPort, cluster.filerGrpcPort))
|
||||
execShell(t, weedCmd, masterAddr, filerAddr, fmt.Sprintf("s3.policy -put -name=%s -file=%s", policyName, tmpPolicyFile.Name()))
|
||||
|
||||
iamClient := newIAMClient(t, cluster.s3Endpoint)
|
||||
|
||||
userName := uniqueName("iam-user")
|
||||
_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
|
||||
UserName: aws.String(userName),
|
||||
PolicyArn: aws.String(policyArn),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
listOut, err := iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.True(t, attachedPolicyContains(listOut.AttachedPolicies, policyName))
|
||||
|
||||
_, err = iamClient.DetachUserPolicy(&iam.DetachUserPolicyInput{
|
||||
UserName: aws.String(userName),
|
||||
PolicyArn: aws.String(policyArn),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
listOut, err = iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.False(t, attachedPolicyContains(listOut.AttachedPolicies, policyName))
|
||||
|
||||
_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
|
||||
UserName: aws.String(userName),
|
||||
PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
|
||||
})
|
||||
require.Error(t, err)
|
||||
if awsErr, ok := err.(awserr.Error); ok {
|
||||
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3IAMListPoliciesAndGetPolicy(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster, err := startMiniCluster(t)
|
||||
require.NoError(t, err)
|
||||
defer cluster.Stop()
|
||||
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
policyName := uniqueName("managed-policy")
|
||||
policyArn := fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)
|
||||
policyContent := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:ListAllMyBuckets","Resource":"*"}]}`
|
||||
|
||||
iamClient := newIAMClient(t, cluster.s3Endpoint)
|
||||
_, err = iamClient.CreatePolicy(&iam.CreatePolicyInput{
|
||||
PolicyName: aws.String(policyName),
|
||||
PolicyDocument: aws.String(policyContent),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
listOut, err := iamClient.ListPolicies(&iam.ListPoliciesInput{})
|
||||
require.NoError(t, err)
|
||||
require.True(t, managedPolicyContains(listOut.Policies, policyName))
|
||||
|
||||
getOut, err := iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: aws.String(policyArn)})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, getOut.Policy)
|
||||
require.NotNil(t, getOut.Policy.PolicyName)
|
||||
require.Equal(t, policyName, *getOut.Policy.PolicyName)
|
||||
|
||||
missingArn := fmt.Sprintf("arn:aws:iam:::policy/%s", uniqueName("missing"))
|
||||
_, err = iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: aws.String(missingArn)})
|
||||
require.Error(t, err)
|
||||
var awsErr awserr.Error
|
||||
require.True(t, errors.As(err, &awsErr))
|
||||
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
}
|
||||
|
||||
func TestS3IAMDeletePolicyInUse(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster, err := startMiniCluster(t)
|
||||
require.NoError(t, err)
|
||||
defer cluster.Stop()
|
||||
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
policyName := uniqueName("managed-delete-policy")
|
||||
policyArn := fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)
|
||||
policyContent := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}`
|
||||
|
||||
iamClient := newIAMClient(t, cluster.s3Endpoint)
|
||||
_, err = iamClient.CreatePolicy(&iam.CreatePolicyInput{
|
||||
PolicyName: aws.String(policyName),
|
||||
PolicyDocument: aws.String(policyContent),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
userName := uniqueName("iam-user-delete-policy")
|
||||
_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
|
||||
UserName: aws.String(userName),
|
||||
PolicyArn: aws.String(policyArn),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: aws.String(policyArn)})
|
||||
require.Error(t, err)
|
||||
var awsErr awserr.Error
|
||||
require.True(t, errors.As(err, &awsErr))
|
||||
require.Equal(t, iam.ErrCodeDeleteConflictException, awsErr.Code())
|
||||
}
|
||||
|
||||
func execShell(t *testing.T, weedCmd, master, filer, shellCmd string) string {
|
||||
// weed shell -master=... -filer=...
|
||||
args := []string{"shell", "-master=" + master, "-filer=" + filer}
|
||||
@@ -145,6 +296,52 @@ func execShell(t *testing.T, weedCmd, master, filer, shellCmd string) string {
|
||||
return string(out)
|
||||
}
|
||||
|
||||
func newIAMClient(t *testing.T, endpoint string) *iam.IAM {
|
||||
t.Helper()
|
||||
|
||||
accessKey := os.Getenv("AWS_ACCESS_KEY_ID")
|
||||
secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
|
||||
if accessKey == "" {
|
||||
accessKey = "admin"
|
||||
}
|
||||
if secretKey == "" {
|
||||
secretKey = "admin"
|
||||
}
|
||||
|
||||
sess, err := session.NewSession(&aws.Config{
|
||||
Region: aws.String("us-east-1"),
|
||||
Endpoint: aws.String(endpoint),
|
||||
DisableSSL: aws.Bool(true),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
return iam.New(sess)
|
||||
}
|
||||
|
||||
func attachedPolicyContains(policies []*iam.AttachedPolicy, policyName string) bool {
|
||||
for _, policy := range policies {
|
||||
if policy.PolicyName != nil && *policy.PolicyName == policyName {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func managedPolicyContains(policies []*iam.Policy, policyName string) bool {
|
||||
for _, policy := range policies {
|
||||
if policy.PolicyName != nil && *policy.PolicyName == policyName {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func uniqueName(prefix string) string {
|
||||
return fmt.Sprintf("%s-%s", prefix, strconv.FormatInt(time.Now().UnixNano(), 36))
|
||||
}
|
||||
|
||||
// --- Test setup helpers ---
|
||||
|
||||
func findAvailablePort() (int, error) {
|
||||
@@ -216,6 +413,14 @@ enabled = true
|
||||
err = os.WriteFile(credentialToml, []byte(credentialConfig), 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Set environment variables for admin credentials safely for this test
|
||||
if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
|
||||
t.Setenv("AWS_ACCESS_KEY_ID", "admin")
|
||||
}
|
||||
if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
|
||||
}
|
||||
|
||||
cluster.wg.Add(1)
|
||||
go func() {
|
||||
defer cluster.wg.Done()
|
||||
@@ -242,6 +447,7 @@ enabled = true
|
||||
"-master.volumeSizeLimitMB=32",
|
||||
"-ip=127.0.0.1",
|
||||
"-master.peers=none",
|
||||
"-s3.iam.readOnly=false",
|
||||
}
|
||||
glog.MaxSize = 1024 * 1024
|
||||
for _, cmd := range command.Commands {
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
//go:build integration
|
||||
// +build integration
|
||||
|
||||
package sse
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
@@ -18,6 +19,37 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// sharedEnv is the single TestEnvironment shared across all tests in this package.
|
||||
var sharedEnv *TestEnvironment
|
||||
|
||||
// TestMain starts one weed mini instance for the whole package and tears it down
|
||||
// after all tests have run.
|
||||
func TestMain(m *testing.M) {
|
||||
flag.Parse()
|
||||
if os.Getenv("SHORT") != "" || testing.Short() {
|
||||
// Let tests self-skip when run with -short.
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
env, err := newTestEnvironmentForMain()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "SKIP: setup failed: %v\n", err)
|
||||
os.Exit(0) // Skip all tests rather than fail
|
||||
}
|
||||
sharedEnv = env
|
||||
|
||||
if startErr := sharedEnv.startSeaweedFSForMain(); startErr != nil {
|
||||
fmt.Fprintf(os.Stderr, "SKIP: weed mini failed to start: %v\n", startErr)
|
||||
sharedEnv.cleanupForMain()
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
code := m.Run()
|
||||
|
||||
sharedEnv.cleanupForMain()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
// TestEnvironment contains the test environment configuration
|
||||
type TestEnvironment struct {
|
||||
seaweedDir string
|
||||
@@ -54,17 +86,15 @@ func getFreePort() (int, net.Listener, error) {
|
||||
return addr.Port, listener, nil
|
||||
}
|
||||
|
||||
// NewTestEnvironment creates a new test environment
|
||||
func NewTestEnvironment(t *testing.T) *TestEnvironment {
|
||||
t.Helper()
|
||||
|
||||
// newTestEnvironmentForMain creates a TestEnvironment without calling t.Fatalf so it
|
||||
// can be used from TestMain (which has no *testing.T).
|
||||
func newTestEnvironmentForMain() (*TestEnvironment, error) {
|
||||
// Find the SeaweedFS root directory
|
||||
wd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get working directory: %v", err)
|
||||
return nil, fmt.Errorf("get working directory: %w", err)
|
||||
}
|
||||
|
||||
// Navigate up to find the SeaweedFS root (contains go.mod)
|
||||
seaweedDir := wd
|
||||
for i := 0; i < 5; i++ {
|
||||
if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
|
||||
@@ -76,82 +106,93 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
|
||||
// Check for weed binary
|
||||
weedBinary := filepath.Join(seaweedDir, "weed", "weed")
|
||||
if _, err := os.Stat(weedBinary); os.IsNotExist(err) {
|
||||
// Try system PATH
|
||||
weedBinary = "weed"
|
||||
if _, err := exec.LookPath(weedBinary); err != nil {
|
||||
t.Skip("weed binary not found, skipping integration test")
|
||||
return nil, fmt.Errorf("weed binary not found")
|
||||
}
|
||||
}
|
||||
|
||||
// Create temporary data directory
|
||||
dataDir, err := os.MkdirTemp("", "seaweed-iceberg-test-*")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
return nil, fmt.Errorf("create temp dir: %w", err)
|
||||
}
|
||||
|
||||
// Allocate free ephemeral ports for each service
|
||||
var listeners []net.Listener
|
||||
defer func() {
|
||||
closeListeners := func() {
|
||||
for _, l := range listeners {
|
||||
l.Close()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
var l net.Listener
|
||||
s3Port, l, err := getFreePort()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port for S3: %v", err)
|
||||
closeListeners()
|
||||
return nil, fmt.Errorf("get free port for S3: %w", err)
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
|
||||
icebergPort, l, err := getFreePort()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port for Iceberg: %v", err)
|
||||
closeListeners()
|
||||
return nil, fmt.Errorf("get free port for Iceberg: %w", err)
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
|
||||
s3GrpcPort, l, err := getFreePort()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port for S3 gRPC: %v", err)
|
||||
closeListeners()
|
||||
return nil, fmt.Errorf("get free port for S3 gRPC: %w", err)
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
|
||||
masterPort, l, err := getFreePort()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port for Master: %v", err)
|
||||
closeListeners()
|
||||
return nil, fmt.Errorf("get free port for Master: %w", err)
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
|
||||
masterGrpcPort, l, err := getFreePort()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port for Master gRPC: %v", err)
|
||||
closeListeners()
|
||||
return nil, fmt.Errorf("get free port for Master gRPC: %w", err)
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
|
||||
filerPort, l, err := getFreePort()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port for Filer: %v", err)
|
||||
closeListeners()
|
||||
return nil, fmt.Errorf("get free port for Filer: %w", err)
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
|
||||
filerGrpcPort, l, err := getFreePort()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port for Filer gRPC: %v", err)
|
||||
closeListeners()
|
||||
return nil, fmt.Errorf("get free port for Filer gRPC: %w", err)
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
|
||||
volumePort, l, err := getFreePort()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port for Volume: %v", err)
|
||||
closeListeners()
|
||||
return nil, fmt.Errorf("get free port for Volume: %w", err)
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
|
||||
volumeGrpcPort, l, err := getFreePort()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port for Volume gRPC: %v", err)
|
||||
closeListeners()
|
||||
return nil, fmt.Errorf("get free port for Volume gRPC: %w", err)
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
|
||||
// Release the port reservations so weed mini can bind to them
|
||||
closeListeners()
|
||||
|
||||
return &TestEnvironment{
|
||||
seaweedDir: seaweedDir,
|
||||
weedBinary: weedBinary,
|
||||
@@ -166,13 +207,11 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
|
||||
volumePort: volumePort,
|
||||
volumeGrpcPort: volumeGrpcPort,
|
||||
dockerAvailable: hasDocker(),
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
// StartSeaweedFS starts a SeaweedFS mini cluster
|
||||
func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
// startSeaweedFSForMain starts weed mini without a *testing.T (for use in TestMain).
|
||||
func (env *TestEnvironment) startSeaweedFSForMain() error {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
env.weedCancel = cancel
|
||||
|
||||
@@ -182,7 +221,8 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
|
||||
|
||||
for _, dir := range []string{masterDir, filerDir, volumeDir} {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
t.Fatalf("Failed to create directory %s: %v", dir, err)
|
||||
cancel()
|
||||
return fmt.Errorf("create directory %s: %w", dir, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -203,13 +243,30 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatalf("Failed to start SeaweedFS: %v", err)
|
||||
cancel()
|
||||
return fmt.Errorf("start SeaweedFS: %w", err)
|
||||
}
|
||||
env.weedProcess = cmd
|
||||
|
||||
// Wait for services to be ready
|
||||
if !env.waitForService(fmt.Sprintf("http://127.0.0.1:%d/v1/config", env.icebergPort), 30*time.Second) {
|
||||
t.Fatalf("Iceberg REST API did not become ready")
|
||||
cancel()
|
||||
cmd.Wait()
|
||||
return fmt.Errorf("Iceberg REST API did not become ready")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanupForMain stops SeaweedFS and cleans up resources (no *testing.T needed).
|
||||
func (env *TestEnvironment) cleanupForMain() {
|
||||
if env.weedCancel != nil {
|
||||
env.weedCancel()
|
||||
}
|
||||
if env.weedProcess != nil {
|
||||
time.Sleep(2 * time.Second)
|
||||
env.weedProcess.Wait()
|
||||
}
|
||||
if env.dataDir != "" {
|
||||
os.RemoveAll(env.dataDir)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -230,25 +287,6 @@ func (env *TestEnvironment) waitForService(url string, timeout time.Duration) bo
|
||||
return false
|
||||
}
|
||||
|
||||
// Cleanup stops SeaweedFS and cleans up resources
|
||||
func (env *TestEnvironment) Cleanup(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
if env.weedCancel != nil {
|
||||
env.weedCancel()
|
||||
}
|
||||
|
||||
if env.weedProcess != nil {
|
||||
// Give process time to shut down gracefully
|
||||
time.Sleep(2 * time.Second)
|
||||
env.weedProcess.Wait()
|
||||
}
|
||||
|
||||
if env.dataDir != "" {
|
||||
os.RemoveAll(env.dataDir)
|
||||
}
|
||||
}
|
||||
|
||||
// IcebergURL returns the Iceberg REST Catalog URL
|
||||
func (env *TestEnvironment) IcebergURL() string {
|
||||
return fmt.Sprintf("http://127.0.0.1:%d", env.icebergPort)
|
||||
@@ -260,10 +298,7 @@ func TestIcebergConfig(t *testing.T) {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
env := NewTestEnvironment(t)
|
||||
defer env.Cleanup(t)
|
||||
|
||||
env.StartSeaweedFS(t)
|
||||
env := sharedEnv
|
||||
|
||||
// Test GET /v1/config
|
||||
resp, err := http.Get(env.IcebergURL() + "/v1/config")
|
||||
@@ -294,16 +329,14 @@ func TestIcebergNamespaces(t *testing.T) {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
env := NewTestEnvironment(t)
|
||||
defer env.Cleanup(t)
|
||||
|
||||
env.StartSeaweedFS(t)
|
||||
env := sharedEnv
|
||||
|
||||
// Create the default table bucket first via S3
|
||||
createTableBucket(t, env, "warehouse")
|
||||
bucketName := "warehouse-ns-" + randomSuffix()
|
||||
createTableBucket(t, env, bucketName)
|
||||
|
||||
// Test GET /v1/namespaces (should return empty list initially)
|
||||
resp, err := http.Get(env.IcebergURL() + "/v1/namespaces")
|
||||
resp, err := http.Get(env.IcebergURL() + icebergPath(bucketName, "/v1/namespaces"))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to list namespaces: %v", err)
|
||||
}
|
||||
@@ -321,16 +354,14 @@ func TestStageCreateAndFinalizeFlow(t *testing.T) {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
env := NewTestEnvironment(t)
|
||||
defer env.Cleanup(t)
|
||||
env := sharedEnv
|
||||
bucketName := "warehouse-stage-" + randomSuffix()
|
||||
createTableBucket(t, env, bucketName)
|
||||
|
||||
env.StartSeaweedFS(t)
|
||||
createTableBucket(t, env, "warehouse")
|
||||
|
||||
namespace := "stage_ns"
|
||||
namespace := "stage_ns_" + randomSuffix()
|
||||
tableName := "orders"
|
||||
|
||||
status, _, err := doIcebergJSONRequest(env, http.MethodPost, "/v1/namespaces", map[string]any{
|
||||
status, _, err := doIcebergJSONRequest(env, http.MethodPost, icebergPath(bucketName, "/v1/namespaces"), map[string]any{
|
||||
"namespace": []string{namespace},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -340,7 +371,7 @@ func TestStageCreateAndFinalizeFlow(t *testing.T) {
|
||||
t.Fatalf("Create namespace status = %d, want 200 or 409", status)
|
||||
}
|
||||
|
||||
status, badReqResp, err := doIcebergJSONRequest(env, http.MethodPost, fmt.Sprintf("/v1/namespaces/%s/tables", namespace), map[string]any{
|
||||
status, badReqResp, err := doIcebergJSONRequest(env, http.MethodPost, icebergPath(bucketName, fmt.Sprintf("/v1/namespaces/%s/tables", namespace)), map[string]any{
|
||||
"stage-create": true,
|
||||
})
|
||||
if err != nil {
|
||||
@@ -358,7 +389,7 @@ func TestStageCreateAndFinalizeFlow(t *testing.T) {
|
||||
t.Fatalf("error.message = %v, want it to include %q", errorObj["message"], "table name is required")
|
||||
}
|
||||
|
||||
status, stageResp, err := doIcebergJSONRequest(env, http.MethodPost, fmt.Sprintf("/v1/namespaces/%s/tables", namespace), map[string]any{
|
||||
status, stageResp, err := doIcebergJSONRequest(env, http.MethodPost, icebergPath(bucketName, fmt.Sprintf("/v1/namespaces/%s/tables", namespace)), map[string]any{
|
||||
"name": tableName,
|
||||
"stage-create": true,
|
||||
})
|
||||
@@ -373,7 +404,7 @@ func TestStageCreateAndFinalizeFlow(t *testing.T) {
|
||||
t.Fatalf("stage metadata-location = %q, want suffix /metadata/v1.metadata.json", stageLocation)
|
||||
}
|
||||
|
||||
status, _, err = doIcebergJSONRequest(env, http.MethodGet, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName), nil)
|
||||
status, _, err = doIcebergJSONRequest(env, http.MethodGet, icebergPath(bucketName, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName)), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Load staged table request failed: %v", err)
|
||||
}
|
||||
@@ -381,7 +412,7 @@ func TestStageCreateAndFinalizeFlow(t *testing.T) {
|
||||
t.Fatalf("Load staged table status = %d, want 404", status)
|
||||
}
|
||||
|
||||
status, commitResp, err := doIcebergJSONRequest(env, http.MethodPost, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName), map[string]any{
|
||||
status, commitResp, err := doIcebergJSONRequest(env, http.MethodPost, icebergPath(bucketName, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName)), map[string]any{
|
||||
"requirements": []map[string]any{
|
||||
{"type": "assert-create"},
|
||||
},
|
||||
@@ -398,7 +429,7 @@ func TestStageCreateAndFinalizeFlow(t *testing.T) {
|
||||
t.Fatalf("final metadata-location = %q, want suffix /metadata/v1.metadata.json", commitLocation)
|
||||
}
|
||||
|
||||
status, loadResp, err := doIcebergJSONRequest(env, http.MethodGet, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName), nil)
|
||||
status, loadResp, err := doIcebergJSONRequest(env, http.MethodGet, icebergPath(bucketName, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName)), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Load finalized table request failed: %v", err)
|
||||
}
|
||||
@@ -417,16 +448,14 @@ func TestCommitMissingTableWithoutAssertCreate(t *testing.T) {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
env := NewTestEnvironment(t)
|
||||
defer env.Cleanup(t)
|
||||
env := sharedEnv
|
||||
bucketName := "warehouse-missing-" + randomSuffix()
|
||||
createTableBucket(t, env, bucketName)
|
||||
|
||||
env.StartSeaweedFS(t)
|
||||
createTableBucket(t, env, "warehouse")
|
||||
|
||||
namespace := "stage_missing_assert_ns"
|
||||
namespace := "stage_missing_assert_ns_" + randomSuffix()
|
||||
tableName := "missing_table"
|
||||
|
||||
status, _, err := doIcebergJSONRequest(env, http.MethodPost, "/v1/namespaces", map[string]any{
|
||||
status, _, err := doIcebergJSONRequest(env, http.MethodPost, icebergPath(bucketName, "/v1/namespaces"), map[string]any{
|
||||
"namespace": []string{namespace},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -436,7 +465,7 @@ func TestCommitMissingTableWithoutAssertCreate(t *testing.T) {
|
||||
t.Fatalf("Create namespace status = %d, want 200 or 409", status)
|
||||
}
|
||||
|
||||
status, _, err = doIcebergJSONRequest(env, http.MethodPost, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName), map[string]any{
|
||||
status, _, err = doIcebergJSONRequest(env, http.MethodPost, icebergPath(bucketName, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName)), map[string]any{
|
||||
"requirements": []any{},
|
||||
"updates": []any{},
|
||||
})
|
||||
@@ -491,6 +520,21 @@ func doIcebergJSONRequest(env *TestEnvironment, method, path string, payload any
|
||||
return resp.StatusCode, decoded, nil
|
||||
}
|
||||
|
||||
// icebergPath inserts the table bucket prefix into Iceberg REST API paths.
|
||||
// For example, "/v1/namespaces" with prefix "my-bucket" becomes
|
||||
// "/v1/my-bucket/namespaces".
|
||||
func icebergPath(prefix, path string) string {
|
||||
if prefix == "" {
|
||||
return path
|
||||
}
|
||||
const base = "/v1/"
|
||||
if !strings.HasPrefix(path, base) {
|
||||
return path
|
||||
}
|
||||
withPrefix := base + prefix + "/" + strings.TrimPrefix(path, base)
|
||||
return withPrefix
|
||||
}
|
||||
|
||||
// createTableBucket creates a table bucket via the S3Tables REST API
|
||||
func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
|
||||
t.Helper()
|
||||
@@ -511,13 +555,20 @@ func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
t.Logf("Create table bucket %s response: status=%d, body=%s", bucketName, resp.StatusCode, string(body))
|
||||
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusConflict {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("Failed to create table bucket %s, status %d: %s", bucketName, resp.StatusCode, body)
|
||||
}
|
||||
t.Logf("Created table bucket %s", bucketName)
|
||||
}
|
||||
|
||||
// randomSuffix returns a short random hex suffix for unique resource naming.
|
||||
func randomSuffix() string {
|
||||
return fmt.Sprintf("%x", time.Now().UnixNano()&0xffffffff)
|
||||
}
|
||||
|
||||
// TestDuckDBIntegration tests Iceberg catalog operations using DuckDB
|
||||
// This test requires Docker to be available
|
||||
func TestDuckDBIntegration(t *testing.T) {
|
||||
@@ -525,15 +576,12 @@ func TestDuckDBIntegration(t *testing.T) {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
env := NewTestEnvironment(t)
|
||||
defer env.Cleanup(t)
|
||||
env := sharedEnv
|
||||
|
||||
if !env.dockerAvailable {
|
||||
t.Skip("Docker not available, skipping DuckDB integration test")
|
||||
}
|
||||
|
||||
env.StartSeaweedFS(t)
|
||||
|
||||
// Create a temporary SQL file for DuckDB to execute
|
||||
sqlFile := filepath.Join(env.dataDir, "test.sql")
|
||||
sqlContent := fmt.Sprintf(`
|
||||
|
||||
@@ -26,17 +26,14 @@ func TestPyIcebergRestCatalog(t *testing.T) {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
env := NewTestEnvironment(t)
|
||||
defer env.Cleanup(t)
|
||||
env := sharedEnv
|
||||
|
||||
if !env.dockerAvailable {
|
||||
t.Skip("Docker not available, skipping PyIceberg integration test")
|
||||
}
|
||||
|
||||
env.StartSeaweedFS(t)
|
||||
|
||||
// Create the test bucket first
|
||||
bucketName := "pyiceberg-compat-test"
|
||||
bucketName := "pyiceberg-compat-test-" + randomSuffix()
|
||||
createTableBucket(t, env, bucketName)
|
||||
|
||||
// Build the test working directory path
|
||||
@@ -56,8 +53,6 @@ func TestPyIcebergRestCatalog(t *testing.T) {
|
||||
|
||||
cmd := exec.Command("docker", "run", "--rm",
|
||||
"--add-host", "host.docker.internal:host-gateway",
|
||||
"-e", fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", "test"),
|
||||
"-e", fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", "test"),
|
||||
"-e", fmt.Sprintf("AWS_ENDPOINT_URL=%s", s3Endpoint),
|
||||
"-v", fmt.Sprintf("%s:/app:ro", testDir),
|
||||
"iceberg-rest-test",
|
||||
@@ -78,3 +73,65 @@ func TestPyIcebergRestCatalog(t *testing.T) {
|
||||
t.Errorf("PyIceberg test failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPyIcebergRestCatalogAuthenticated tests the Iceberg REST Catalog using PyIceberg with authentication.
|
||||
// This test uses the default admin credentials that SeaweedFS creates on startup.
|
||||
func TestPyIcebergRestCatalogAuthenticated(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
env := sharedEnv
|
||||
|
||||
if !env.dockerAvailable {
|
||||
t.Skip("Docker not available, skipping PyIceberg integration test")
|
||||
}
|
||||
|
||||
// Use default admin credentials
|
||||
testAccessKey := "admin"
|
||||
testSecretKey := "admin"
|
||||
|
||||
// Create the test bucket first (using unauthenticated request, which works with DefaultAllow)
|
||||
bucketName := "pyiceberg-auth-test-" + randomSuffix()
|
||||
createTableBucket(t, env, bucketName)
|
||||
|
||||
// Build the test working directory path
|
||||
testDir := filepath.Join(env.seaweedDir, "test", "s3tables", "catalog")
|
||||
|
||||
// Run PyIceberg test using Docker with authentication
|
||||
catalogURL := fmt.Sprintf("http://host.docker.internal:%d", env.icebergPort)
|
||||
s3Endpoint := fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
|
||||
warehouse := fmt.Sprintf("s3://%s/", bucketName)
|
||||
|
||||
// Build the test image first for faster repeated runs
|
||||
buildCmd := exec.Command("docker", "build", "-t", "iceberg-rest-test", "-f", "Dockerfile.pyiceberg", ".")
|
||||
buildCmd.Dir = testDir
|
||||
if out, err := buildCmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("Failed to build test image: %v\n%s", err, string(out))
|
||||
}
|
||||
|
||||
cmd := exec.Command("docker", "run", "--rm",
|
||||
"--add-host", "host.docker.internal:host-gateway",
|
||||
"-e", fmt.Sprintf("AWS_ENDPOINT_URL=%s", s3Endpoint),
|
||||
"-v", fmt.Sprintf("%s:/app:ro", testDir),
|
||||
"iceberg-rest-test",
|
||||
"python3", "/app/test_rest_catalog_auth.py",
|
||||
"--catalog-url", catalogURL,
|
||||
"--warehouse", warehouse,
|
||||
"--prefix", bucketName,
|
||||
"--access-key", testAccessKey,
|
||||
"--secret-key", testSecretKey,
|
||||
)
|
||||
cmd.Dir = testDir
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
t.Logf("Running PyIceberg REST catalog test with authentication...")
|
||||
t.Logf(" Catalog URL: %s", catalogURL)
|
||||
t.Logf(" Warehouse: %s", warehouse)
|
||||
t.Logf(" Access Key: %s", testAccessKey)
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
t.Errorf("PyIceberg authenticated test failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// verifyTableBucketMetadata verifies that a table bucket was created with proper metadata
|
||||
func verifyTableBucketMetadata(t *testing.T, env *TestEnvironment, bucketName string) {
|
||||
t.Helper()
|
||||
|
||||
// Use S3Tables REST API to get the bucket
|
||||
endpoint := fmt.Sprintf("http://localhost:%d/buckets/%s", env.s3Port, bucketName)
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create request: %v", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-amz-json-1.1")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get table bucket %s: %v", bucketName, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
t.Logf("Get table bucket %s response: status=%d, body=%s", bucketName, resp.StatusCode, string(body))
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("Failed to get table bucket %s, status %d: %s", bucketName, resp.StatusCode, body)
|
||||
}
|
||||
t.Logf("Verified table bucket %s exists with metadata", bucketName)
|
||||
}
|
||||
@@ -201,6 +201,7 @@ def main():
|
||||
"uri": args.catalog_url,
|
||||
"warehouse": args.warehouse,
|
||||
"prefix": args.prefix,
|
||||
"s3.anonymous": "true", # Disable AWS request signing for unauthenticated access
|
||||
}
|
||||
)
|
||||
print(f"Successfully connected to catalog on attempt {attempt + 1}")
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Iceberg REST Catalog Compatibility Test for SeaweedFS (Authenticated)
|
||||
|
||||
This script tests the Iceberg REST Catalog API compatibility with authentication.
|
||||
|
||||
Usage:
|
||||
python3 test_rest_catalog_auth.py --catalog-url http://localhost:8182 \\
|
||||
--access-key admin --secret-key admin
|
||||
|
||||
Requirements:
|
||||
pip install pyiceberg[s3fs]
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pyiceberg.catalog import load_catalog
|
||||
from pyiceberg.schema import Schema
|
||||
from pyiceberg.types import (
|
||||
IntegerType,
|
||||
LongType,
|
||||
StringType,
|
||||
NestedField,
|
||||
)
|
||||
from pyiceberg.exceptions import (
|
||||
NamespaceAlreadyExistsError,
|
||||
NoSuchNamespaceError,
|
||||
TableAlreadyExistsError,
|
||||
NoSuchTableError,
|
||||
)
|
||||
|
||||
|
||||
def test_config_endpoint(catalog):
|
||||
"""Test that the catalog config endpoint returns valid configuration."""
|
||||
print("Testing /v1/config endpoint...")
|
||||
# The catalog is already loaded which means config endpoint worked
|
||||
print(" /v1/config endpoint working")
|
||||
return True
|
||||
|
||||
|
||||
def test_namespace_operations(catalog, prefix):
|
||||
"""Test namespace CRUD operations."""
|
||||
print("Testing namespace operations...")
|
||||
namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
|
||||
|
||||
# List initial namespaces
|
||||
namespaces = catalog.list_namespaces()
|
||||
print(f" Initial namespaces: {namespaces}")
|
||||
|
||||
# Create namespace
|
||||
try:
|
||||
catalog.create_namespace(namespace)
|
||||
print(f" Created namespace: {namespace}")
|
||||
except NamespaceAlreadyExistsError:
|
||||
print(f" ! Namespace already exists: {namespace}")
|
||||
|
||||
# List namespaces (should include our new one)
|
||||
namespaces = catalog.list_namespaces()
|
||||
if namespace in namespaces:
|
||||
print(" Namespace appears in list")
|
||||
else:
|
||||
print(f" Namespace not found in list: {namespaces}")
|
||||
return False
|
||||
|
||||
# Get namespace properties
|
||||
try:
|
||||
props = catalog.load_namespace_properties(namespace)
|
||||
print(f" Loaded namespace properties: {props}")
|
||||
except NoSuchNamespaceError:
|
||||
print(f" Failed to load namespace properties")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def test_table_operations(catalog, prefix):
|
||||
"""Test table CRUD operations."""
|
||||
print("Testing table operations...")
|
||||
namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
|
||||
table_name = "auth_test_table"
|
||||
table_id = namespace + (table_name,)
|
||||
|
||||
# Define a simple schema
|
||||
schema = Schema(
|
||||
NestedField(field_id=1, name="id", field_type=LongType(), required=True),
|
||||
NestedField(field_id=2, name="name", field_type=StringType(), required=False),
|
||||
NestedField(field_id=3, name="age", field_type=IntegerType(), required=False),
|
||||
)
|
||||
|
||||
# Create table
|
||||
try:
|
||||
table = catalog.create_table(
|
||||
identifier=table_id,
|
||||
schema=schema,
|
||||
)
|
||||
print(f" Created table: {table_id}")
|
||||
except TableAlreadyExistsError:
|
||||
print(f" ! Table already exists: {table_id}")
|
||||
_ = catalog.load_table(table_id)
|
||||
|
||||
# List tables
|
||||
tables = catalog.list_tables(namespace)
|
||||
if table_name in [t[1] for t in tables]:
|
||||
print(" Table appears in list")
|
||||
else:
|
||||
print(f" Table not found in list: {tables}")
|
||||
return False
|
||||
|
||||
# Load table
|
||||
try:
|
||||
loaded_table = catalog.load_table(table_id)
|
||||
print(f" Loaded table: {loaded_table.name()}")
|
||||
print(f" Schema: {loaded_table.schema()}")
|
||||
print(f" Location: {loaded_table.location()}")
|
||||
except NoSuchTableError:
|
||||
print(f" Failed to load table")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def test_cleanup(catalog, prefix):
|
||||
"""Test table and namespace deletion."""
|
||||
print("Testing cleanup operations...")
|
||||
namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
|
||||
table_id = namespace + ("auth_test_table",)
|
||||
|
||||
# Drop table
|
||||
try:
|
||||
catalog.drop_table(table_id)
|
||||
print(f" Dropped table: {table_id}")
|
||||
except NoSuchTableError:
|
||||
print(f" ! Table already deleted: {table_id}")
|
||||
|
||||
# Drop namespace
|
||||
try:
|
||||
catalog.drop_namespace(namespace)
|
||||
print(f" Dropped namespace: {namespace}")
|
||||
except NoSuchNamespaceError:
|
||||
print(f" ! Namespace already deleted: {namespace}")
|
||||
except Exception as e:
|
||||
print(f" ? Namespace drop error (may be expected): {e}")
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Test Iceberg REST Catalog with authentication")
|
||||
parser.add_argument("--catalog-url", required=True, help="Iceberg REST Catalog URL")
|
||||
parser.add_argument("--warehouse", default="s3://iceberg-test/", help="Warehouse location")
|
||||
parser.add_argument("--prefix", required=True, help="Table bucket prefix")
|
||||
parser.add_argument("--access-key", required=True, help="AWS Access Key ID")
|
||||
parser.add_argument("--secret-key", required=True, help="AWS Secret Access Key")
|
||||
parser.add_argument("--skip-cleanup", action="store_true", help="Skip cleanup at the end")
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"Connecting to Iceberg REST Catalog at: {args.catalog_url}")
|
||||
print(f"Warehouse: {args.warehouse}")
|
||||
print(f"Prefix: {args.prefix}")
|
||||
print(f"Using authenticated access with key: {args.access_key}")
|
||||
print()
|
||||
|
||||
# Load the REST catalog with authentication
|
||||
import time
|
||||
max_retries = 10
|
||||
catalog = None
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
catalog = load_catalog(
|
||||
"rest",
|
||||
**{
|
||||
"type": "rest",
|
||||
"uri": args.catalog_url,
|
||||
"warehouse": args.warehouse,
|
||||
"prefix": args.prefix,
|
||||
"s3.access-key-id": args.access_key,
|
||||
"s3.secret-access-key": args.secret_key,
|
||||
}
|
||||
)
|
||||
print(f"Successfully connected to catalog on attempt {attempt + 1}")
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt < max_retries - 1:
|
||||
print(f" Attempt {attempt + 1} failed, retrying in 2s... ({e})")
|
||||
time.sleep(2)
|
||||
else:
|
||||
print(f" All {max_retries} attempts failed.")
|
||||
raise e
|
||||
|
||||
# Run tests
|
||||
tests = [
|
||||
("Config Endpoint", lambda: test_config_endpoint(catalog)),
|
||||
("Namespace Operations", lambda: test_namespace_operations(catalog, args.prefix)),
|
||||
("Table Operations", lambda: test_table_operations(catalog, args.prefix)),
|
||||
]
|
||||
|
||||
if not args.skip_cleanup:
|
||||
tests.append(("Cleanup", lambda: test_cleanup(catalog, args.prefix)))
|
||||
|
||||
passed = 0
|
||||
failed = 0
|
||||
|
||||
for name, test_fn in tests:
|
||||
print(f"\n{'='*50}")
|
||||
try:
|
||||
if test_fn():
|
||||
passed += 1
|
||||
print(f"PASSED: {name}")
|
||||
else:
|
||||
failed += 1
|
||||
print(f"FAILED: {name}")
|
||||
except Exception as e:
|
||||
failed += 1
|
||||
print(f"ERROR in {name}: {e}")
|
||||
|
||||
print(f"\n{'='*50}")
|
||||
print(f"Results: {passed} passed, {failed} failed")
|
||||
|
||||
return 0 if failed == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -40,6 +40,7 @@ type TestEnvironment struct {
|
||||
dockerAvailable bool
|
||||
accessKey string
|
||||
secretKey string
|
||||
closers []io.Closer
|
||||
}
|
||||
|
||||
func TestTrinoIcebergCatalog(t *testing.T) {
|
||||
@@ -118,30 +119,66 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
|
||||
|
||||
bindIP := testutil.FindBindIP()
|
||||
|
||||
masterPort, masterGrpcPort := mustFreePortPair(t, "Master")
|
||||
volumePort, volumeGrpcPort := mustFreePortPair(t, "Volume")
|
||||
filerPort, filerGrpcPort := mustFreePortPair(t, "Filer")
|
||||
s3Port, s3GrpcPort := mustFreePortPair(t, "S3")
|
||||
icebergPort := mustFreePort(t, "Iceberg")
|
||||
|
||||
return &TestEnvironment{
|
||||
seaweedDir: seaweedDir,
|
||||
weedBinary: weedBinary,
|
||||
dataDir: dataDir,
|
||||
bindIP: bindIP,
|
||||
s3Port: s3Port,
|
||||
s3GrpcPort: s3GrpcPort,
|
||||
icebergPort: icebergPort,
|
||||
masterPort: masterPort,
|
||||
masterGrpcPort: masterGrpcPort,
|
||||
filerPort: filerPort,
|
||||
filerGrpcPort: filerGrpcPort,
|
||||
volumePort: volumePort,
|
||||
volumeGrpcPort: volumeGrpcPort,
|
||||
dockerAvailable: hasDocker(),
|
||||
accessKey: "AKIAIOSFODNN7EXAMPLE",
|
||||
secretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
env := &TestEnvironment{
|
||||
seaweedDir: seaweedDir,
|
||||
weedBinary: weedBinary,
|
||||
dataDir: dataDir,
|
||||
bindIP: bindIP,
|
||||
closers: []io.Closer{},
|
||||
}
|
||||
|
||||
env.masterPort, env.masterGrpcPort = env.mustFreePortPair("Master")
|
||||
env.volumePort, env.volumeGrpcPort = env.mustFreePortPair("Volume")
|
||||
env.filerPort, env.filerGrpcPort = env.mustFreePortPair("Filer")
|
||||
env.s3Port, env.s3GrpcPort = env.mustFreePortPair("S3")
|
||||
env.icebergPort = env.mustFreePort("Iceberg")
|
||||
|
||||
env.dockerAvailable = hasDocker()
|
||||
env.accessKey = "AKIAIOSFODNN7EXAMPLE"
|
||||
env.secretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
|
||||
return env
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) mustFreePort(name string) int {
|
||||
port, closer, err := getFreePort()
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Failed to get free port for %s: %v", name, err))
|
||||
}
|
||||
env.closers = append(env.closers, closer)
|
||||
return port
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) mustFreePortPair(name string) (int, int) {
|
||||
httpPort, httpCloser, grpcPort, grpcCloser, err := findAvailablePortPair()
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Failed to get free port pair for %s: %v", name, err))
|
||||
}
|
||||
env.closers = append(env.closers, httpCloser, grpcCloser)
|
||||
return httpPort, grpcPort
|
||||
}
|
||||
|
||||
func mustFreePort(t *testing.T, name string) int {
|
||||
t.Helper()
|
||||
|
||||
port, closer, err := getFreePort()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port for %s: %v", name, err)
|
||||
}
|
||||
closer.Close()
|
||||
return port
|
||||
}
|
||||
|
||||
func mustFreePortPair(t *testing.T, name string) (int, int) {
|
||||
t.Helper()
|
||||
|
||||
httpPort, httpCloser, grpcPort, grpcCloser, err := findAvailablePortPair()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port pair for %s: %v", name, err)
|
||||
}
|
||||
httpCloser.Close()
|
||||
grpcCloser.Close()
|
||||
return httpPort, grpcPort
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
|
||||
@@ -161,6 +198,12 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
env.weedCancel = cancel
|
||||
|
||||
// Close all port listeners right before starting the weed process
|
||||
for _, closer := range env.closers {
|
||||
closer.Close()
|
||||
}
|
||||
env.closers = nil
|
||||
|
||||
cmd := exec.CommandContext(ctx, env.weedBinary, "mini",
|
||||
"-master.port", fmt.Sprintf("%d", env.masterPort),
|
||||
"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
|
||||
@@ -471,47 +514,27 @@ func hasDocker() bool {
|
||||
return cmd.Run() == nil
|
||||
}
|
||||
|
||||
func mustFreePort(t *testing.T, name string) int {
|
||||
t.Helper()
|
||||
|
||||
port, err := getFreePort()
|
||||
func findAvailablePortPair() (int, io.Closer, int, io.Closer, error) {
|
||||
httpPort, httpCloser, err := getFreePort()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port for %s: %v", name, err)
|
||||
return 0, nil, 0, nil, err
|
||||
}
|
||||
return port
|
||||
grpcPort, grpcCloser, err := getFreePort()
|
||||
if err != nil {
|
||||
httpCloser.Close()
|
||||
return 0, nil, 0, nil, err
|
||||
}
|
||||
return httpPort, httpCloser, grpcPort, grpcCloser, nil
|
||||
}
|
||||
|
||||
func mustFreePortPair(t *testing.T, name string) (int, int) {
|
||||
t.Helper()
|
||||
|
||||
httpPort, grpcPort, err := findAvailablePortPair()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port pair for %s: %v", name, err)
|
||||
}
|
||||
return httpPort, grpcPort
|
||||
}
|
||||
|
||||
func findAvailablePortPair() (int, int, error) {
|
||||
httpPort, err := getFreePort()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
grpcPort, err := getFreePort()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return httpPort, grpcPort, nil
|
||||
}
|
||||
|
||||
func getFreePort() (int, error) {
|
||||
func getFreePort() (int, io.Closer, error) {
|
||||
listener, err := net.Listen("tcp", "0.0.0.0:0")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
return 0, nil, err
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
addr := listener.Addr().(*net.TCPAddr)
|
||||
return addr.Port, nil
|
||||
return addr.Port, listener, nil
|
||||
}
|
||||
|
||||
func randomString(length int) string {
|
||||
|
||||
@@ -12,33 +12,57 @@ import (
|
||||
"time"
|
||||
|
||||
cryptorand "crypto/rand"
|
||||
"sync"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"flag"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/command"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
|
||||
)
|
||||
|
||||
var (
|
||||
miniClusterMutex sync.Mutex
|
||||
)
|
||||
// TestMain starts a single default weed mini cluster for the whole package and
|
||||
// tears it down after all tests have completed. Tests that require a different
|
||||
// cluster configuration (e.g. TestS3TablesCreateBucketIAMPolicy) start their
|
||||
// own cluster independently.
|
||||
func TestMain(m *testing.M) {
|
||||
flag.Parse()
|
||||
if testing.Short() {
|
||||
// Tests self-skip with t.Skip when -short is set; no cluster needed.
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
// Create a temporary T-less context so we can use t.TempDir-equivalent.
|
||||
testDir, err := os.MkdirTemp("", "seaweed-s3tables-shared-*")
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "SKIP: failed to create shared temp dir: %v\n", err)
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
cluster, err := startMiniClusterInDir(testDir, nil)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "SKIP: failed to start shared weed mini cluster: %v\n", err)
|
||||
os.RemoveAll(testDir)
|
||||
os.Exit(0)
|
||||
}
|
||||
sharedCluster = cluster
|
||||
|
||||
code := m.Run()
|
||||
|
||||
sharedCluster.Stop()
|
||||
os.RemoveAll(testDir)
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func TestS3TablesIntegration(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
// Create and start test cluster
|
||||
cluster, err := startMiniCluster(t)
|
||||
require.NoError(t, err)
|
||||
defer cluster.Stop()
|
||||
|
||||
// Create S3 Tables client
|
||||
client := NewS3TablesClient(cluster.s3Endpoint, testRegion, testAccessKey, testSecretKey)
|
||||
// Re-use the shared cluster started by TestMain.
|
||||
client := NewS3TablesClient(sharedCluster.s3Endpoint, testRegion, testAccessKey, testSecretKey)
|
||||
|
||||
// Run test suite
|
||||
t.Run("TableBucketLifecycle", func(t *testing.T) {
|
||||
@@ -70,6 +94,92 @@ func TestS3TablesIntegration(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestS3TablesCreateBucketIAMPolicy(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping IAM integration test in short mode")
|
||||
}
|
||||
|
||||
t.Setenv("AWS_ACCESS_KEY_ID", "env-admin")
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", "env-secret")
|
||||
|
||||
allowedBucket := "tables-allowed"
|
||||
deniedBucket := "tables-denied"
|
||||
iamConfigDir := t.TempDir()
|
||||
iamConfigPath := filepath.Join(iamConfigDir, "iam_config.json")
|
||||
iamConfig := fmt.Sprintf(`{
|
||||
"sts": {
|
||||
"tokenDuration": "1h",
|
||||
"maxSessionLength": "12h",
|
||||
"issuer": "seaweedfs-sts",
|
||||
"signingKey": "%s"
|
||||
},
|
||||
"accounts": [
|
||||
{
|
||||
"id": "%s",
|
||||
"displayName": "tables-integration"
|
||||
}
|
||||
],
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "%s",
|
||||
"secretKey": "%s"
|
||||
}
|
||||
],
|
||||
"account": {
|
||||
"id": "%s",
|
||||
"displayName": "tables-integration"
|
||||
},
|
||||
"policyNames": ["S3TablesBucketPolicy"]
|
||||
}
|
||||
],
|
||||
"policy": {
|
||||
"defaultEffect": "Deny",
|
||||
"storeType": "memory"
|
||||
},
|
||||
"policies": [
|
||||
{
|
||||
"name": "S3TablesBucketPolicy",
|
||||
"document": {
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3tables:CreateTableBucket"],
|
||||
"Resource": [
|
||||
"arn:aws:s3tables:*:*:bucket/%s",
|
||||
"arn:aws:s3:::%s"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}`, testIAMSigningKey, testAccountID, testAccessKey, testSecretKey, testAccountID, allowedBucket, allowedBucket)
|
||||
require.NoError(t, os.WriteFile(iamConfigPath, []byte(iamConfig), 0644))
|
||||
|
||||
cluster, err := startMiniClusterWithExtraArgs(t, []string{
|
||||
"-s3.config=" + iamConfigPath,
|
||||
"-s3.iam.config=" + iamConfigPath,
|
||||
})
|
||||
require.NoError(t, err, "failed to start cluster with IAM config")
|
||||
defer cluster.Stop()
|
||||
|
||||
client := NewS3TablesClient(cluster.s3Endpoint, testRegion, testAccessKey, testSecretKey)
|
||||
|
||||
_, err = client.CreateTableBucket(deniedBucket, nil)
|
||||
require.Error(t, err, "denied bucket creation should fail")
|
||||
assert.Contains(t, err.Error(), "AccessDenied")
|
||||
|
||||
allowedResp, err := client.CreateTableBucket(allowedBucket, nil)
|
||||
require.NoError(t, err, "allowed bucket creation should succeed")
|
||||
defer func() {
|
||||
_ = client.DeleteTableBucket(allowedResp.ARN)
|
||||
}()
|
||||
}
|
||||
|
||||
func testTableBucketLifecycle(t *testing.T, client *S3TablesClient) {
|
||||
bucketName := "test-bucket-" + randomString(8)
|
||||
|
||||
@@ -508,11 +618,12 @@ func findAvailablePorts(n int) ([]int, error) {
|
||||
return ports, nil
|
||||
}
|
||||
|
||||
// startMiniCluster starts a weed mini instance directly without exec
|
||||
func startMiniCluster(t *testing.T) (*TestCluster, error) {
|
||||
// Find available ports
|
||||
// We need 8 unique ports: Master(2), Volume(2), Filer(2), S3(2)
|
||||
ports, err := findAvailablePorts(8)
|
||||
// startMiniClusterInDir starts a weed mini instance using testDir as the data
|
||||
// directory. It does not require a *testing.T so it can be called from TestMain.
|
||||
// extraArgs are appended to the default mini command flags.
|
||||
func startMiniClusterInDir(testDir string, extraArgs []string) (*TestCluster, error) {
|
||||
// We need 10 unique ports: Master(2), Volume(2), Filer(2), S3(2), Admin(2)
|
||||
ports, err := findAvailablePorts(10)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find available ports: %v", err)
|
||||
}
|
||||
@@ -525,8 +636,8 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
|
||||
filerGrpcPort := ports[5]
|
||||
s3Port := ports[6]
|
||||
s3GrpcPort := ports[7]
|
||||
// Create temporary directory for test data
|
||||
testDir := t.TempDir()
|
||||
adminPort := ports[8]
|
||||
adminGrpcPort := ports[9]
|
||||
|
||||
// Ensure no configuration file from previous runs
|
||||
configFile := filepath.Join(testDir, "mini.options")
|
||||
@@ -537,7 +648,6 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
|
||||
|
||||
s3Endpoint := fmt.Sprintf("http://127.0.0.1:%d", s3Port)
|
||||
cluster := &TestCluster{
|
||||
t: t,
|
||||
dataDir: testDir,
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
@@ -550,22 +660,25 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
|
||||
|
||||
// Create empty security.toml to disable JWT authentication in tests
|
||||
securityToml := filepath.Join(testDir, "security.toml")
|
||||
err = os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644)
|
||||
if err != nil {
|
||||
if err = os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644); err != nil {
|
||||
cancel()
|
||||
return nil, fmt.Errorf("failed to create security.toml: %v", err)
|
||||
}
|
||||
|
||||
// Ensure AWS credentials are set (don't use t.Setenv here — we are in TestMain).
|
||||
if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
|
||||
os.Setenv("AWS_ACCESS_KEY_ID", "admin")
|
||||
}
|
||||
if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
|
||||
os.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
|
||||
}
|
||||
|
||||
// Start weed mini in a goroutine by calling the command directly
|
||||
cluster.wg.Add(1)
|
||||
go func() {
|
||||
defer cluster.wg.Done()
|
||||
|
||||
// Protect global state mutation with a mutex
|
||||
miniClusterMutex.Lock()
|
||||
defer miniClusterMutex.Unlock()
|
||||
|
||||
// Save current directory and args
|
||||
// Save current directory and args, restore on exit.
|
||||
oldDir, _ := os.Getwd()
|
||||
oldArgs := os.Args
|
||||
defer func() {
|
||||
@@ -573,21 +686,24 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
|
||||
os.Args = oldArgs
|
||||
}()
|
||||
|
||||
// Change to test directory so mini picks up security.toml
|
||||
// Change to test directory so mini picks up security.toml.
|
||||
os.Chdir(testDir)
|
||||
|
||||
// Configure args for mini command
|
||||
os.Args = []string{
|
||||
"weed",
|
||||
baseArgs := []string{
|
||||
"-dir=" + testDir,
|
||||
"-master.dir=" + testDir,
|
||||
"-master.port=" + strconv.Itoa(masterPort),
|
||||
"-master.port.grpc=" + strconv.Itoa(masterGrpcPort),
|
||||
"-volume.port=" + strconv.Itoa(volumePort),
|
||||
"-volume.port.grpc=" + strconv.Itoa(volumeGrpcPort),
|
||||
"-volume.port.public=" + strconv.Itoa(volumePort),
|
||||
"-volume.publicUrl=127.0.0.1:" + strconv.Itoa(volumePort),
|
||||
"-filer.port=" + strconv.Itoa(filerPort),
|
||||
"-filer.port.grpc=" + strconv.Itoa(filerGrpcPort),
|
||||
"-s3.port=" + strconv.Itoa(s3Port),
|
||||
"-s3.port.grpc=" + strconv.Itoa(s3GrpcPort),
|
||||
"-admin.port=" + strconv.Itoa(adminPort),
|
||||
"-admin.port.grpc=" + strconv.Itoa(adminGrpcPort),
|
||||
"-webdav.port=0", // Disable WebDAV
|
||||
"-admin.ui=false", // Disable admin UI
|
||||
"-master.volumeSizeLimitMB=32", // Small volumes for testing
|
||||
@@ -595,6 +711,10 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
|
||||
"-master.peers=none", // Faster startup
|
||||
"-s3.iam.readOnly=false", // Enable IAM write operations for tests
|
||||
}
|
||||
if len(extraArgs) > 0 {
|
||||
baseArgs = append(baseArgs, extraArgs...)
|
||||
}
|
||||
os.Args = append([]string{"weed"}, baseArgs...)
|
||||
|
||||
// Suppress most logging during tests
|
||||
glog.MaxSize = 1024 * 1024
|
||||
@@ -613,14 +733,34 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
|
||||
}()
|
||||
|
||||
// Wait for S3 service to be ready
|
||||
err = waitForS3Ready(cluster.s3Endpoint, 30*time.Second)
|
||||
if err != nil {
|
||||
if err = waitForS3Ready(cluster.s3Endpoint, 30*time.Second); err != nil {
|
||||
cancel()
|
||||
return nil, fmt.Errorf("S3 service failed to start: %v", err)
|
||||
}
|
||||
|
||||
cluster.isRunning = true
|
||||
return cluster, nil
|
||||
}
|
||||
|
||||
// startMiniClusterWithExtraArgs starts a weed mini instance for a single test.
|
||||
// It uses t.TempDir() for data isolation and t.Setenv for credential scoping.
|
||||
func startMiniClusterWithExtraArgs(t *testing.T, extraArgs []string) (*TestCluster, error) {
|
||||
t.Helper()
|
||||
testDir := t.TempDir()
|
||||
|
||||
// Scope credentials to the test so they are restored after test completion.
|
||||
if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
|
||||
t.Setenv("AWS_ACCESS_KEY_ID", "admin")
|
||||
}
|
||||
if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
|
||||
}
|
||||
|
||||
cluster, err := startMiniClusterInDir(testDir, extraArgs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cluster.t = t
|
||||
t.Logf("Test cluster started successfully at %s", cluster.s3Endpoint)
|
||||
return cluster, nil
|
||||
}
|
||||
@@ -646,22 +786,15 @@ func (c *TestCluster) Stop() {
|
||||
case <-done:
|
||||
// Goroutine finished
|
||||
case <-timer.C:
|
||||
// Timeout - goroutine doesn't respond to context cancel
|
||||
// This may indicate the mini cluster didn't shut down cleanly
|
||||
c.t.Log("Warning: Test cluster shutdown timed out after 2 seconds")
|
||||
}
|
||||
|
||||
// Reset the global cmdMini flags to prevent state leakage to other tests
|
||||
for _, cmd := range command.Commands {
|
||||
if cmd.Name() == "mini" {
|
||||
// Reset flags to defaults
|
||||
cmd.Flag.VisitAll(func(f *flag.Flag) {
|
||||
// Reset to default value
|
||||
f.Value.Set(f.DefValue)
|
||||
})
|
||||
break
|
||||
// Timeout - goroutine doesn't respond to context cancel.
|
||||
// This may indicate the mini cluster didn't shut down cleanly.
|
||||
if c.t != nil {
|
||||
c.t.Log("Warning: Test cluster shutdown timed out after 2 seconds")
|
||||
} else {
|
||||
fmt.Println("Warning: Test cluster shutdown timed out after 2 seconds")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// waitForS3Ready waits for the S3 service to be ready
|
||||
|
||||
@@ -8,6 +8,11 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// sharedCluster is the single default TestCluster shared across all tests
|
||||
// that do not require a specialised cluster configuration.
|
||||
// It is initialised by TestMain and must not be modified by individual tests.
|
||||
var sharedCluster *TestCluster
|
||||
|
||||
// TestCluster manages the weed mini instance for integration testing
|
||||
type TestCluster struct {
|
||||
t *testing.T
|
||||
@@ -46,8 +51,9 @@ func NewS3TablesClient(endpoint, region, accessKey, secretKey string) *S3TablesC
|
||||
|
||||
// Test configuration constants
|
||||
const (
|
||||
testRegion = "us-west-2"
|
||||
testAccessKey = "admin"
|
||||
testSecretKey = "admin"
|
||||
testAccountID = "111122223333"
|
||||
testRegion = "us-west-2"
|
||||
testAccessKey = "admin"
|
||||
testSecretKey = "admin"
|
||||
testAccountID = "111122223333"
|
||||
testIAMSigningKey = "dGVzdC1zaWduaW5nLWtleS1mb3Itc3RzLWludGVncmF0aW9uLXRlc3Rz"
|
||||
)
|
||||
|
||||
@@ -3,12 +3,6 @@
|
||||
## Goal
|
||||
Create a Go integration test suite under `test/volume_server` that validates **drop-in behavior parity** for the Volume Server HTTP and gRPC APIs, so a Rust rewrite can be verified against the current Go behavior.
|
||||
|
||||
## Current Program Focus (2026-02-16)
|
||||
- Primary execution focus has shifted to implementing native Rust volume-server parity.
|
||||
- This integration suite is now the parity gate for Rust implementation work.
|
||||
- New tests should be added only when native Rust implementation reveals uncovered Go behavior that is not yet captured.
|
||||
- Rust implementation roadmap lives in `rust/volume_server/DEV_PLAN.md`.
|
||||
|
||||
## Hard Requirements
|
||||
- Tests live under `test/volume_server`.
|
||||
- Tests are written in Go.
|
||||
@@ -1133,202 +1127,3 @@ Update this section during implementation:
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: `499` cancellation and transport-interruption branches remain pending.
|
||||
- Commit: `d1e5f390a`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added Rust volume-server migration bootstrap wiring.
|
||||
- APIs covered: Harness now supports Go master + selectable volume binary (`VOLUME_SERVER_IMPL=rust` or `VOLUME_SERVER_BINARY`), with Rust-mode smoke coverage for representative HTTP/gRPC integration tests.
|
||||
- Profiles covered: P1 smoke and default Go matrix unchanged.
|
||||
- Gaps introduced/remaining: native Rust handler implementations are tracked in `rust/volume_server/DEV_PLAN.md`; current Rust mode is a compatibility launcher phase.
|
||||
- Commit: `7beab85c2`, `880c2e1da`, `63d08e8a9`, `d402573ea`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Validated Rust-mode compatibility path with full integration suites.
|
||||
- APIs covered: full `/test/volume_server/http` and `/test/volume_server/grpc` suites executed successfully with `VOLUME_SERVER_IMPL=rust`.
|
||||
- Profiles covered: existing integration matrix in Rust launcher mode.
|
||||
- Gaps introduced/remaining: native Rust endpoint and storage/RPC implementation remains pending; current mode delegates execution to Go volume server.
|
||||
- Commit: `6ce4d7ede`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added Rust proxy-supervision launcher mode and validated all volume-server integration suites in proxy mode.
|
||||
- APIs covered: full `/test/volume_server/http` and `/test/volume_server/grpc` suites executed successfully with `VOLUME_SERVER_IMPL=rust` and `VOLUME_SERVER_RUST_MODE=proxy`.
|
||||
- Profiles covered: existing integration matrix in Rust proxy launcher mode.
|
||||
- Gaps introduced/remaining: native Rust endpoint/storage/RPC logic remains pending; current proxy mode still delegates backend handlers to Go.
|
||||
- Commit: `a7f50d23b`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Expanded GitHub CI Rust smoke job to run both launcher modes (`exec` and `proxy`).
|
||||
- APIs covered: representative HTTP/gRPC smoke flows now validated in CI for both Rust launcher execution paths.
|
||||
- Profiles covered: CI smoke profile for P1 representative calls.
|
||||
- Gaps introduced/remaining: CI still runs Rust mode as smoke scope; full-suite Rust-mode CI remains optional due runtime cost.
|
||||
- Commit: `548b3d9a3`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added JWT UI override integration coverage with explicit `access.ui=true` profile support.
|
||||
- APIs covered: `/ui/index.html` behavior under JWT profile now verifies both default auth-gated path (`401`) and explicit UI-enable override path (`200` with rendered UI).
|
||||
- Profiles covered: P3 and P3+`access.ui=true`.
|
||||
- Gaps introduced/remaining: UI exposure behavior under JWT profiles is now covered for both secured and override branches.
|
||||
- Commit: `de974c05d`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added oversized upload integration coverage using explicit `-fileSizeLimitMB` profile configuration.
|
||||
- APIs covered: HTTP write path now verifies payloads larger than configured file-size limit are rejected (`400`) with limit-related error context.
|
||||
- Profiles covered: P1-derived profile with `FileSizeLimitMB=1`.
|
||||
- Gaps introduced/remaining: file-size limit rejection branch is now covered; replicate-write failure and cancellation (`499`) branches remain pending.
|
||||
- Commit: `4d61cbdee`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added gRPC EC-only metadata-read unsupported-path coverage.
|
||||
- APIs covered: `ReadNeedleMeta` now verifies the explicit unsupported branch when a volume is unmounted and only EC shards are mounted.
|
||||
- Profiles covered: P1 with EC shard generation/mount lifecycle setup.
|
||||
- Gaps introduced/remaining: low-level read metadata fault-injection and transport interruption branches remain pending.
|
||||
- Commit: `37bf9b5eb`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added deterministic replicated-write failure coverage for unmet replication requirements.
|
||||
- APIs covered: HTTP write path now verifies replication error handling (`500`) and no-local-commit outcome (`404` on follow-up read) when volume replication is configured (`001`) but replica set cannot be satisfied.
|
||||
- Profiles covered: P1-derived single-node profile with per-volume replication override.
|
||||
- Gaps introduced/remaining: replicated-write failure branch is now covered; cancellation (`499`) and deeper transport fault-injection branches remain pending.
|
||||
- Commit: `4835d3443`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added EC-backed `BatchDelete` positive-path integration coverage.
|
||||
- APIs covered: `BatchDelete` now validates successful EC-needle deletion (`202`) with post-delete `VolumeEcShardRead` deleted-marker verification (`IsDeleted=true`).
|
||||
- Profiles covered: P1 with EC shard generate/mount lifecycle; includes test-local gRPC offset proxy for EC internal dial-path parity.
|
||||
- Gaps introduced/remaining: EC `BatchDelete` success branch is now covered; deeper distributed failure-injection permutations remain pending.
|
||||
- Commit: `1bb40b6bc`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Expanded HTTP conditional-header matrix for `HEAD` requests with combined header scenarios.
|
||||
- APIs covered: `HEAD` now verifies `If-Modified-Since` precedence over mismatched `If-None-Match`, and `If-None-Match` precedence when IMS is stale but ETag matches.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: combined conditional-header precedence paths for `HEAD`/`GET` are now explicitly covered.
|
||||
- Commit: `ed23e290f`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added multi-volume success-stream coverage for `ReadAllNeedles`.
|
||||
- APIs covered: `ReadAllNeedles` now verifies happy-path streaming across two existing volumes in one request, with per-volume payload validation.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: multi-volume stream boundary coverage is now present for successful reads; transport interruption branches remain pending.
|
||||
- Commit: `ab95a6ef1`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Expanded split-port unsupported HTTP method matrix with `MKCOL` coverage.
|
||||
- APIs covered: admin/public parity for `MKCOL` (`400` on admin, passthrough `200` on public) with post-call data-integrity verification.
|
||||
- Profiles covered: P2.
|
||||
- Gaps introduced/remaining: unsupported-method sampling now covers `PATCH`, `TRACE`, `PROPFIND`, `CONNECT`, and `MKCOL`; cancellation/transport fault paths remain the main unaddressed area.
|
||||
- Commit: `2ab30900d`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Hardened framework port allocation to keep volume ports within `10000..55535`.
|
||||
- APIs covered: test harness now guarantees valid `admin+10000` gRPC offset ranges, eliminating EC batch-delete flake scenarios that rely on offset dialing.
|
||||
- Profiles covered: all profiles using shared framework port allocation.
|
||||
- Gaps introduced/remaining: no new API gaps; transport cancellation/fault-injection branches remain the primary uncovered area.
|
||||
- Commit: `90e82b15c`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Re-validated full Rust proxy-mode integration suite after latest HTTP/gRPC coverage additions and framework hardening.
|
||||
- APIs covered: full `/test/volume_server/http` and `/test/volume_server/grpc` packages.
|
||||
- Profiles covered: existing matrix in Rust proxy launcher mode.
|
||||
- Gaps introduced/remaining: native Rust endpoint/storage/RPC implementation remains pending (current Rust mode still delegates to Go backend handlers).
|
||||
- Commit: `6bb9d8bac`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added tail sender stream-cancellation interruption coverage.
|
||||
- APIs covered: `VolumeTailSender` now verifies client-side context cancellation behavior (`codes.Canceled`) after stream start/heartbeat.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: tail transport interruption coverage is now partially closed; receiver-side interruption and deeper network fault injection remain pending.
|
||||
- Commit: `27a80f760`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added receiver-side tail stream interruption coverage for unavailable source node.
|
||||
- APIs covered: `VolumeTailReceiver` now verifies source connect/dial failure branch when `SourceVolumeServer` is unreachable.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: both sender and receiver transport interruption branches are now covered at integration level; deeper injected network faults remain pending.
|
||||
- Commit: `a5864c3eb`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Expanded query CSV matrix with explicit CSV-payload parity coverage.
|
||||
- APIs covered: `Query` now verifies current CSV-input behavior (no streamed rows / immediate EOF) even when source needle content is valid CSV text.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: CSV parsing/selection implementation still absent in current server behavior; integration parity now locks the current no-output semantics for both JSON and CSV payload shapes.
|
||||
- Commit: `a12dd5f8d`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Expanded ping unreachable-target matrix with volume-server target coverage.
|
||||
- APIs covered: `Ping` now explicitly verifies unreachable-target error wrapping for `target_type=volumeServer`, alongside existing master/filer variants.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: ping target-type matrix is covered for success and unreachable branches across master/filer/volume-server.
|
||||
- Commit: `b9fbb85af`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Expanded deleted-read HTTP matrix with `HEAD` parity coverage.
|
||||
- APIs covered: `HEAD` with `readDeleted=true` now validates current behavior (`200`, empty body, payload-size `Content-Length`) after delete.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: deleted-read parity now covers both `GET` and `HEAD` semantics on local-volume path.
|
||||
- Commit: `cc80ad364`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Shifted planning priority to native Rust implementation parity.
|
||||
- APIs covered: no new API additions in this plan entry; integration suite remains the validation gate.
|
||||
- Profiles covered: unchanged.
|
||||
- Gaps introduced/remaining: primary remaining gap is native Rust handler/storage/RPC implementation replacing Go backend delegation.
|
||||
- Commit: `14c863dbf`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added Rust launcher `native` mode bootstrap, made it the default Rust launcher mode, and expanded CI Rust smoke matrix coverage to include `native`.
|
||||
- APIs covered: full `/test/volume_server/http` and `/test/volume_server/grpc` packages re-validated in `VOLUME_SERVER_RUST_MODE=native`; default Rust launcher path (`VOLUME_SERVER_IMPL=rust`) smoke-validated for HTTP and gRPC control tests.
|
||||
- Profiles covered: existing HTTP/gRPC matrix in native launcher mode; default-mode smoke checks on P1 control flows.
|
||||
- Gaps introduced/remaining: native Rust API/storage/RPC handlers still pending; current `native` mode remains a delegation bootstrap while parity replacement proceeds.
|
||||
- Commit: `70ddbee37`, `61befd10f`, `2e65966c0`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Implemented first native Rust HTTP control handlers in `native` mode for `/status` and `/healthz` (admin listener path), with proxy fallback retained for remaining endpoints.
|
||||
- APIs covered: `/status` and `/healthz` now served by Rust-native code path in `VOLUME_SERVER_RUST_MODE=native`; full HTTP and gRPC integration suites re-validated.
|
||||
- Profiles covered: full existing HTTP/gRPC matrix under native mode.
|
||||
- Gaps introduced/remaining: gRPC control/data APIs and most HTTP data/static/auth paths remain delegated and require incremental native replacement.
|
||||
- Commit: `7e6e0261a`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Implemented native Rust HTTP `OPTIONS` handling for admin/public listeners in `VOLUME_SERVER_RUST_MODE=native`.
|
||||
- APIs covered: admin/public `OPTIONS` method allow-list and CORS origin handling now served from Rust-native path; full HTTP and gRPC packages re-validated.
|
||||
- Profiles covered: P2 split-port CORS/OPTIONS profiles plus full existing matrix in native mode.
|
||||
- Gaps introduced/remaining: static/UI/auth and data-path HTTP handlers, plus gRPC control/data/stream handlers, remain delegated and pending native replacement.
|
||||
- Commit: `fbff2cb39`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Implemented broader native Rust HTTP surface in `native` mode:
|
||||
- `/ui/index.html` auth-gating parity (JWT + access.ui handling)
|
||||
- static asset paths (`/favicon.ico`, `/seaweedfsstatic/*`)
|
||||
- public non-read method no-op parity (`200`) and admin unsupported-method rejection parity (`400`)
|
||||
- APIs covered: UI/static/admin/public control-surface behavior now served by Rust-native path; full `/test/volume_server/http` and `/test/volume_server/grpc` packages re-validated.
|
||||
- Profiles covered: P1/P2/P3 variants touched by UI/static/CORS/method behavior, plus full matrix in native mode.
|
||||
- Gaps introduced/remaining: HTTP data-path handlers and all gRPC handlers remain delegated and still require native replacement.
|
||||
- Commit: `23e4497b2`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Improved native Rust control-path parity for health/status and request-target parsing.
|
||||
- APIs covered: native `/healthz` now mirrors backend service health transitions (`200`/`503`) and native route matching now handles absolute-form HTTP request targets before path dispatch.
|
||||
- Profiles covered: full existing HTTP/gRPC integration matrix in native mode (`VOLUME_SERVER_IMPL=rust`, `VOLUME_SERVER_RUST_MODE=native`).
|
||||
- Gaps introduced/remaining: core HTTP data handlers and all gRPC RPC handlers are still delegated and remain the primary native implementation gap.
|
||||
- Commit: `d6ff6ed6d`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added Rust-native malformed fid-route validation ahead of delegated data handlers.
|
||||
- APIs covered: GET/HEAD/POST/PUT fid-shaped paths now return native `400` for invalid vid/fid tokens (including invalid-read and invalid-write path variants) while valid routes continue through delegated storage handlers.
|
||||
- Profiles covered: full existing HTTP/gRPC integration matrix in native mode, plus targeted invalid-path parity runs.
|
||||
- Gaps introduced/remaining: data-path success branches and all gRPC handler bodies remain delegated; native replacement work continues on those core execution paths.
|
||||
- Commit: `1ce0174b2`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added Rust-native write error prevalidation for delegated fid routes.
|
||||
- APIs covered: native path now rejects malformed multipart boundary requests, `Content-MD5` writes, and over-limit `Content-Length` writes (using `-fileSizeLimitMB`) with `400` parity behavior before delegation.
|
||||
- Profiles covered: P1 default and P1 with explicit file-size limit profile, plus full existing HTTP/gRPC native-mode matrix revalidation.
|
||||
- Gaps introduced/remaining: positive write/read/delete data handlers and all gRPC handler bodies are still delegated; native implementation continues incrementally.
|
||||
- Commit: `94cefd6f4`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Broadened Rust-native malformed-fid handling to slash-form read/write URL shapes with reserved-route exclusions.
|
||||
- APIs covered: `/{vid}/{fid}` and `/{vid}/{fid}/{filename}` malformed cases now hit native `400` validation path, while `/status`, `/healthz`, `/ui/index.html`, `/stats/*`, and static asset routes remain excluded from fid parsing.
|
||||
- Profiles covered: full existing HTTP/gRPC native-mode matrix, plus targeted admin/read-path malformed-slash validation run.
|
||||
- Gaps introduced/remaining: successful data-path read/write/delete handlers and all gRPC method bodies remain delegated and are still the primary native implementation gap.
|
||||
- Commit: `e66983e9b`
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
.PHONY: test-volume-server test-volume-server-short test-volume-server-rust-smoke
|
||||
.PHONY: test-volume-server test-volume-server-short
|
||||
|
||||
test-volume-server:
|
||||
go test ./test/volume_server/... -v
|
||||
|
||||
test-volume-server-short:
|
||||
go test ./test/volume_server/... -short -v
|
||||
|
||||
test-volume-server-rust-smoke:
|
||||
VOLUME_SERVER_IMPL=rust go test ./test/volume_server/http ./test/volume_server/grpc -run 'TestAdminStatusAndHealthz|TestUploadReadRangeHeadDeleteRoundTrip|TestStateAndStatusRPCs|TestVolumeSyncStatusAndReadVolumeFileStatus' -v
|
||||
|
||||
@@ -15,10 +15,6 @@ If a `weed` binary is not found, the harness will build one automatically.
|
||||
## Optional environment variables
|
||||
|
||||
- `WEED_BINARY`: explicit path to the `weed` executable (disables auto-build).
|
||||
- `VOLUME_SERVER_IMPL`: select volume-server implementation (`go` default, `rust` for Rust launcher mode).
|
||||
- `VOLUME_SERVER_BINARY`: explicit path to the volume-server executable (overrides `VOLUME_SERVER_IMPL`).
|
||||
- `VOLUME_SERVER_RUST_MODE`: Rust launcher mode (`native` default, `exec` for direct Go delegation, `proxy` for Rust front proxy + Go backend).
|
||||
- `VOLUME_SERVER_RUST_REBUILD=1`: force rebuild of Rust volume-server binary in Rust mode.
|
||||
- `VOLUME_SERVER_IT_KEEP_LOGS=1`: keep temporary test directories and process logs.
|
||||
|
||||
## Current scope (Phase 0)
|
||||
@@ -28,4 +24,4 @@ If a `weed` binary is not found, the harness will build one automatically.
|
||||
- Initial HTTP admin endpoint checks
|
||||
- Initial gRPC state/status checks
|
||||
|
||||
More API coverage is tracked in `test/volume_server/DEV_PLAN.md`.
|
||||
More API coverage is tracked in `/Users/chris/dev/seaweedfs2/test/volume_server/DEV_PLAN.md`.
|
||||
|
||||
@@ -32,12 +32,11 @@ type Cluster struct {
|
||||
testingTB testing.TB
|
||||
profile matrix.Profile
|
||||
|
||||
weedBinary string
|
||||
volumeBinary string
|
||||
baseDir string
|
||||
configDir string
|
||||
logsDir string
|
||||
keepLogs bool
|
||||
weedBinary string
|
||||
baseDir string
|
||||
configDir string
|
||||
logsDir string
|
||||
keepLogs bool
|
||||
|
||||
masterPort int
|
||||
masterGrpcPort int
|
||||
@@ -55,9 +54,9 @@ type Cluster struct {
|
||||
func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster {
|
||||
t.Helper()
|
||||
|
||||
weedBinary, volumeBinary, err := FindOrBuildServerBinaries()
|
||||
weedBinary, err := FindOrBuildWeedBinary()
|
||||
if err != nil {
|
||||
t.Fatalf("resolve server binaries: %v", err)
|
||||
t.Fatalf("resolve weed binary: %v", err)
|
||||
}
|
||||
|
||||
baseDir, keepLogs, err := newWorkDir()
|
||||
@@ -93,7 +92,6 @@ func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster {
|
||||
testingTB: t,
|
||||
profile: profile,
|
||||
weedBinary: weedBinary,
|
||||
volumeBinary: volumeBinary,
|
||||
baseDir: baseDir,
|
||||
configDir: configDir,
|
||||
logsDir: logsDir,
|
||||
@@ -208,11 +206,8 @@ func (c *Cluster) startVolume(dataDir string) error {
|
||||
if c.profile.InflightDownloadTimeout > 0 {
|
||||
args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
|
||||
}
|
||||
if c.profile.FileSizeLimitMB > 0 {
|
||||
args = append(args, "-fileSizeLimitMB="+strconv.Itoa(c.profile.FileSizeLimitMB))
|
||||
}
|
||||
|
||||
c.volumeCmd = exec.Command(c.volumeBinary, args...)
|
||||
c.volumeCmd = exec.Command(c.weedBinary, args...)
|
||||
c.volumeCmd.Dir = c.baseDir
|
||||
c.volumeCmd.Stdout = logFile
|
||||
c.volumeCmd.Stderr = logFile
|
||||
@@ -269,46 +264,18 @@ func stopProcess(cmd *exec.Cmd) {
|
||||
}
|
||||
|
||||
func allocatePorts(count int) ([]int, error) {
|
||||
const minPort = 10000
|
||||
const maxPort = 55535
|
||||
const host = "127.0.0.1"
|
||||
rangeSize := maxPort - minPort + 1
|
||||
|
||||
listeners := make([]net.Listener, 0, count)
|
||||
ports := make([]int, 0, count)
|
||||
seen := make(map[int]struct{}, count)
|
||||
|
||||
closeAll := func() {
|
||||
for _, ll := range listeners {
|
||||
_ = ll.Close()
|
||||
}
|
||||
}
|
||||
|
||||
startOffset := int(time.Now().UnixNano() % int64(rangeSize))
|
||||
for i := 0; i < count; i++ {
|
||||
found := false
|
||||
for offset := 0; offset < rangeSize; offset++ {
|
||||
port := minPort + (startOffset+offset)%rangeSize
|
||||
if _, exists := seen[port]; exists {
|
||||
continue
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
for _, ll := range listeners {
|
||||
_ = ll.Close()
|
||||
}
|
||||
|
||||
l, err := net.Listen("tcp", net.JoinHostPort(host, strconv.Itoa(port)))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
seen[port] = struct{}{}
|
||||
listeners = append(listeners, l)
|
||||
ports = append(ports, port)
|
||||
found = true
|
||||
startOffset = (startOffset + offset + 1) % rangeSize
|
||||
break
|
||||
}
|
||||
if !found {
|
||||
closeAll()
|
||||
return nil, fmt.Errorf("unable to allocate %d ports within range [%d,%d]", count, minPort, maxPort)
|
||||
return nil, err
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
ports = append(ports, l.Addr().(*net.TCPAddr).Port)
|
||||
}
|
||||
for _, l := range listeners {
|
||||
_ = l.Close()
|
||||
@@ -359,13 +326,6 @@ func writeSecurityConfig(configDir string, profile matrix.Profile) error {
|
||||
b.WriteString("\"\n")
|
||||
b.WriteString("expires_after_seconds = 60\n")
|
||||
}
|
||||
if profile.AccessUI {
|
||||
if b.Len() > 0 {
|
||||
b.WriteString("\n")
|
||||
}
|
||||
b.WriteString("[access]\n")
|
||||
b.WriteString("ui = true\n")
|
||||
}
|
||||
if b.Len() == 0 {
|
||||
b.WriteString("# optional security config generated for integration tests\n")
|
||||
}
|
||||
@@ -381,12 +341,17 @@ func FindOrBuildWeedBinary() (string, error) {
|
||||
return "", fmt.Errorf("WEED_BINARY is set but not executable: %s", fromEnv)
|
||||
}
|
||||
|
||||
repoRoot, err := detectRepoRoot()
|
||||
if err != nil {
|
||||
return "", err
|
||||
repoRoot := ""
|
||||
if _, file, _, ok := runtime.Caller(0); ok {
|
||||
repoRoot = filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", ".."))
|
||||
candidate := filepath.Join(repoRoot, "weed", "weed")
|
||||
if isExecutableFile(candidate) {
|
||||
return candidate, nil
|
||||
}
|
||||
}
|
||||
if candidate := filepath.Join(repoRoot, "weed", "weed"); isExecutableFile(candidate) {
|
||||
return candidate, nil
|
||||
|
||||
if repoRoot == "" {
|
||||
return "", errors.New("unable to detect repository root")
|
||||
}
|
||||
|
||||
binDir := filepath.Join(os.TempDir(), "seaweedfs_volume_server_it_bin")
|
||||
@@ -412,82 +377,6 @@ func FindOrBuildWeedBinary() (string, error) {
|
||||
return binPath, nil
|
||||
}
|
||||
|
||||
// FindOrBuildServerBinaries returns master and volume executables.
|
||||
// Master always runs from the Go weed binary; volume can be switched via env.
|
||||
func FindOrBuildServerBinaries() (masterBinary string, volumeBinary string, err error) {
|
||||
masterBinary, err = FindOrBuildWeedBinary()
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
volumeBinary, err = FindOrBuildVolumeServerBinary(masterBinary)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return masterBinary, volumeBinary, nil
|
||||
}
|
||||
|
||||
// FindOrBuildVolumeServerBinary resolves the executable used for volume-server processes.
|
||||
//
|
||||
// Behavior:
|
||||
// - `VOLUME_SERVER_BINARY=/path/to/bin`: use explicit executable path.
|
||||
// - `VOLUME_SERVER_IMPL=rust`: build/use Rust volume server launcher.
|
||||
// - default: use the same Go `weed` binary.
|
||||
func FindOrBuildVolumeServerBinary(defaultBinary string) (string, error) {
|
||||
if fromEnv := os.Getenv("VOLUME_SERVER_BINARY"); fromEnv != "" {
|
||||
if isExecutableFile(fromEnv) {
|
||||
return fromEnv, nil
|
||||
}
|
||||
return "", fmt.Errorf("VOLUME_SERVER_BINARY is set but not executable: %s", fromEnv)
|
||||
}
|
||||
|
||||
impl := strings.ToLower(strings.TrimSpace(os.Getenv("VOLUME_SERVER_IMPL")))
|
||||
if impl == "" || impl == "go" {
|
||||
return defaultBinary, nil
|
||||
}
|
||||
if impl != "rust" {
|
||||
return "", fmt.Errorf("unsupported VOLUME_SERVER_IMPL %q (supported: go, rust)", impl)
|
||||
}
|
||||
|
||||
repoRoot, err := detectRepoRoot()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return FindOrBuildRustVolumeServerBinary(repoRoot)
|
||||
}
|
||||
|
||||
// FindOrBuildRustVolumeServerBinary builds the Rust volume server launcher when needed.
|
||||
func FindOrBuildRustVolumeServerBinary(repoRoot string) (string, error) {
|
||||
manifestPath := filepath.Join(repoRoot, "rust", "volume_server", "Cargo.toml")
|
||||
if _, err := os.Stat(manifestPath); err != nil {
|
||||
return "", fmt.Errorf("rust volume server manifest not found at %s: %w", manifestPath, err)
|
||||
}
|
||||
|
||||
targetDir := filepath.Join(os.TempDir(), "seaweedfs_volume_server_it_rust_target")
|
||||
binPath := filepath.Join(targetDir, "release", "weed-volume-rs")
|
||||
if isExecutableFile(binPath) && os.Getenv("VOLUME_SERVER_RUST_REBUILD") != "1" {
|
||||
return binPath, nil
|
||||
}
|
||||
|
||||
cmd := exec.Command("cargo", "build", "--release", "--manifest-path", manifestPath, "--target-dir", targetDir)
|
||||
var out bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
cmd.Stderr = &out
|
||||
if err := cmd.Run(); err != nil {
|
||||
return "", fmt.Errorf("build rust volume server binary: %w\n%s", err, out.String())
|
||||
}
|
||||
if !isExecutableFile(binPath) {
|
||||
return "", fmt.Errorf("built rust volume server binary is not executable: %s", binPath)
|
||||
}
|
||||
return binPath, nil
|
||||
}
|
||||
|
||||
func detectRepoRoot() (string, error) {
|
||||
if _, file, _, ok := runtime.Caller(0); ok {
|
||||
return filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", "..")), nil
|
||||
}
|
||||
return "", errors.New("unable to detect repository root")
|
||||
}
|
||||
|
||||
func isExecutableFile(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil || info.IsDir() {
|
||||
|
||||
@@ -17,12 +17,11 @@ type DualVolumeCluster struct {
|
||||
testingTB testing.TB
|
||||
profile matrix.Profile
|
||||
|
||||
weedBinary string
|
||||
volumeBinary string
|
||||
baseDir string
|
||||
configDir string
|
||||
logsDir string
|
||||
keepLogs bool
|
||||
weedBinary string
|
||||
baseDir string
|
||||
configDir string
|
||||
logsDir string
|
||||
keepLogs bool
|
||||
|
||||
masterPort int
|
||||
masterGrpcPort int
|
||||
@@ -44,9 +43,9 @@ type DualVolumeCluster struct {
|
||||
func StartDualVolumeCluster(t testing.TB, profile matrix.Profile) *DualVolumeCluster {
|
||||
t.Helper()
|
||||
|
||||
weedBinary, volumeBinary, err := FindOrBuildServerBinaries()
|
||||
weedBinary, err := FindOrBuildWeedBinary()
|
||||
if err != nil {
|
||||
t.Fatalf("resolve server binaries: %v", err)
|
||||
t.Fatalf("resolve weed binary: %v", err)
|
||||
}
|
||||
|
||||
baseDir, keepLogs, err := newWorkDir()
|
||||
@@ -83,7 +82,6 @@ func StartDualVolumeCluster(t testing.TB, profile matrix.Profile) *DualVolumeClu
|
||||
testingTB: t,
|
||||
profile: profile,
|
||||
weedBinary: weedBinary,
|
||||
volumeBinary: volumeBinary,
|
||||
baseDir: baseDir,
|
||||
configDir: configDir,
|
||||
logsDir: logsDir,
|
||||
@@ -229,7 +227,7 @@ func (c *DualVolumeCluster) startVolume(index int, dataDir string) error {
|
||||
args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
|
||||
}
|
||||
|
||||
cmd := exec.Command(c.volumeBinary, args...)
|
||||
cmd := exec.Command(c.weedBinary, args...)
|
||||
cmd.Dir = c.baseDir
|
||||
cmd.Stdout = logFile
|
||||
cmd.Stderr = logFile
|
||||
|
||||
@@ -14,26 +14,18 @@ import (
|
||||
|
||||
func AllocateVolume(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection string) {
|
||||
t.Helper()
|
||||
AllocateVolumeWithReplication(t, client, volumeID, collection, "000")
|
||||
}
|
||||
|
||||
func AllocateVolumeWithReplication(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection, replication string) {
|
||||
t.Helper()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if replication == "" {
|
||||
replication = "000"
|
||||
}
|
||||
|
||||
_, err := client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: collection,
|
||||
Replication: replication,
|
||||
Replication: "000",
|
||||
Version: uint32(needle.GetCurrentVersion()),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("allocate volume %d (replication=%s): %v", volumeID, replication, err)
|
||||
t.Fatalf("allocate volume %d: %v", volumeID, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -388,17 +388,6 @@ func TestPingUnknownAndUnreachableTargetPaths(t *testing.T) {
|
||||
if !strings.Contains(err.Error(), "ping filer") {
|
||||
t.Fatalf("Ping filer unreachable error mismatch: %v", err)
|
||||
}
|
||||
|
||||
_, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
|
||||
TargetType: cluster.VolumeServerType,
|
||||
Target: "127.0.0.1:1.1",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("Ping volume server target should fail when target is unreachable")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "ping volumeServer") {
|
||||
t.Fatalf("Ping volume server unreachable error mismatch: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPingMasterTargetSuccess(t *testing.T) {
|
||||
|
||||
@@ -3,10 +3,7 @@ package volume_server_grpc_test
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -265,141 +262,3 @@ func TestBatchDeleteRejectsChunkManifestNeedles(t *testing.T) {
|
||||
t.Fatalf("chunk manifest should not be deleted by BatchDelete reject path, got %d", readResp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchDeleteEcNeedleSuccess(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
if stopProxy := maybeStartGrpcOffsetProxy(t, cluster.VolumeAdminAddress(), cluster.VolumeGRPCAddress()); stopProxy != nil {
|
||||
t.Cleanup(stopProxy)
|
||||
}
|
||||
|
||||
const volumeID = uint32(34)
|
||||
const needleID = uint64(930001)
|
||||
const cookie = uint32(0x6677BBCC)
|
||||
framework.AllocateVolume(t, client, volumeID, "")
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, needleID, cookie)
|
||||
uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), fid, []byte("batch-delete-ec-success"))
|
||||
_ = framework.ReadAllAndClose(t, uploadResp)
|
||||
if uploadResp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := client.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = client.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsMount failed: %v", err)
|
||||
}
|
||||
|
||||
deleteResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{fid}})
|
||||
if err != nil {
|
||||
t.Fatalf("BatchDelete EC needle should return response, got grpc error: %v", err)
|
||||
}
|
||||
if len(deleteResp.GetResults()) != 1 {
|
||||
t.Fatalf("BatchDelete EC needle expected one result, got %d", len(deleteResp.GetResults()))
|
||||
}
|
||||
if deleteResp.GetResults()[0].GetStatus() != http.StatusAccepted {
|
||||
t.Fatalf("BatchDelete EC needle expected status 202, got %d error=%q", deleteResp.GetResults()[0].GetStatus(), deleteResp.GetResults()[0].GetError())
|
||||
}
|
||||
if deleteResp.GetResults()[0].GetSize() == 0 {
|
||||
t.Fatalf("BatchDelete EC needle expected non-zero deleted size")
|
||||
}
|
||||
|
||||
deletedStream, err := client.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
|
||||
VolumeId: volumeID,
|
||||
ShardId: 0,
|
||||
FileKey: needleID,
|
||||
Offset: 0,
|
||||
Size: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardRead deleted-check start failed: %v", err)
|
||||
}
|
||||
deletedMsg, err := deletedStream.Recv()
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardRead deleted-check recv failed: %v", err)
|
||||
}
|
||||
if !deletedMsg.GetIsDeleted() {
|
||||
t.Fatalf("VolumeEcShardRead expected IsDeleted=true after EC batch delete")
|
||||
}
|
||||
}
|
||||
|
||||
func maybeStartGrpcOffsetProxy(t testing.TB, adminAddr, actualGrpcAddr string) func() {
|
||||
t.Helper()
|
||||
|
||||
host, portText, err := net.SplitHostPort(adminAddr)
|
||||
if err != nil {
|
||||
t.Fatalf("split admin address %q: %v", adminAddr, err)
|
||||
}
|
||||
port, err := strconv.Atoi(portText)
|
||||
if err != nil {
|
||||
t.Fatalf("parse admin port %q: %v", portText, err)
|
||||
}
|
||||
|
||||
expectedGrpcAddr := net.JoinHostPort(host, strconv.Itoa(port+10000))
|
||||
if expectedGrpcAddr == actualGrpcAddr {
|
||||
return nil
|
||||
}
|
||||
|
||||
listener, err := net.Listen("tcp", expectedGrpcAddr)
|
||||
if err != nil {
|
||||
t.Fatalf("listen grpc-offset proxy %s -> %s: %v", expectedGrpcAddr, actualGrpcAddr, err)
|
||||
}
|
||||
|
||||
stopped := make(chan struct{})
|
||||
go func() {
|
||||
for {
|
||||
conn, acceptErr := listener.Accept()
|
||||
if acceptErr != nil {
|
||||
select {
|
||||
case <-stopped:
|
||||
return
|
||||
default:
|
||||
continue
|
||||
}
|
||||
}
|
||||
go proxyBidirectional(conn, actualGrpcAddr)
|
||||
}
|
||||
}()
|
||||
|
||||
return func() {
|
||||
close(stopped)
|
||||
_ = listener.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func proxyBidirectional(inbound net.Conn, targetAddr string) {
|
||||
outbound, err := net.Dial("tcp", targetAddr)
|
||||
if err != nil {
|
||||
_ = inbound.Close()
|
||||
return
|
||||
}
|
||||
|
||||
go func() {
|
||||
_, _ = io.Copy(outbound, inbound)
|
||||
_ = outbound.Close()
|
||||
}()
|
||||
_, _ = io.Copy(inbound, outbound)
|
||||
_ = inbound.Close()
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@ package volume_server_grpc_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -145,66 +144,3 @@ func TestReadNeedleBlobAndMetaInvalidOffsets(t *testing.T) {
|
||||
t.Fatalf("ReadNeedleMeta should fail for invalid offset")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadNeedleMetaUnsupportedForEcMountedVolume(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(93)
|
||||
const needleID = uint64(880002)
|
||||
const cookie = uint32(0x11AA22BB)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, needleID, cookie)
|
||||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-meta-unsupported"))
|
||||
_ = framework.ReadAllAndClose(t, uploadResp)
|
||||
if uploadResp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsMount failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = grpcClient.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeUnmount failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{
|
||||
VolumeId: volumeID,
|
||||
NeedleId: needleID,
|
||||
Offset: 0,
|
||||
Size: 128,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("ReadNeedleMeta should fail when only EC shards are mounted")
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(err.Error()), "ec shards is not supported") {
|
||||
t.Fatalf("ReadNeedleMeta EC-only error mismatch: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package volume_server_grpc_test
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -231,71 +230,6 @@ func TestReadAllNeedlesExistingThenMissingVolumeAbortsStream(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadAllNeedlesStreamsAcrossMultipleVolumes(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeIDA = uint32(86)
|
||||
const volumeIDB = uint32(87)
|
||||
const needleIDA = uint64(445561)
|
||||
const needleIDB = uint64(445562)
|
||||
framework.AllocateVolume(t, grpcClient, volumeIDA, "")
|
||||
framework.AllocateVolume(t, grpcClient, volumeIDB, "")
|
||||
|
||||
client := framework.NewHTTPClient()
|
||||
fidA := framework.NewFileID(volumeIDA, needleIDA, 0xAA11CC22)
|
||||
fidB := framework.NewFileID(volumeIDB, needleIDB, 0xBB22DD33)
|
||||
payloadA := "read-all-multi-volume-a"
|
||||
payloadB := "read-all-multi-volume-b"
|
||||
|
||||
uploadA := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fidA, []byte(payloadA))
|
||||
_ = framework.ReadAllAndClose(t, uploadA)
|
||||
if uploadA.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload A expected 201, got %d", uploadA.StatusCode)
|
||||
}
|
||||
uploadB := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fidB, []byte(payloadB))
|
||||
_ = framework.ReadAllAndClose(t, uploadB)
|
||||
if uploadB.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload B expected 201, got %d", uploadB.StatusCode)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{
|
||||
VolumeIds: []uint32{volumeIDA, volumeIDB},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAllNeedles start failed: %v", err)
|
||||
}
|
||||
|
||||
seen := map[uint64]string{}
|
||||
for {
|
||||
msg, recvErr := stream.Recv()
|
||||
if recvErr == io.EOF {
|
||||
break
|
||||
}
|
||||
if recvErr != nil {
|
||||
t.Fatalf("ReadAllNeedles recv failed: %v", recvErr)
|
||||
}
|
||||
if msg.GetNeedleId() == needleIDA || msg.GetNeedleId() == needleIDB {
|
||||
seen[msg.GetNeedleId()] = string(msg.GetNeedleBlob())
|
||||
}
|
||||
}
|
||||
|
||||
if got := seen[needleIDA]; got != payloadA {
|
||||
t.Fatalf("ReadAllNeedles missing/mismatched payload for volume A needle: got %q want %q", got, payloadA)
|
||||
}
|
||||
if got := seen[needleIDB]; got != payloadB {
|
||||
t.Fatalf("ReadAllNeedles missing/mismatched payload for volume B needle: got %q want %q", got, payloadB)
|
||||
}
|
||||
}
|
||||
|
||||
func copyFileBytes(t testing.TB, grpcClient volume_server_pb.VolumeServerClient, req *volume_server_pb.CopyFileRequest) []byte {
|
||||
t.Helper()
|
||||
|
||||
|
||||
@@ -280,54 +280,6 @@ func TestQueryJsonSuccessAndCsvNoOutput(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryCsvInputWithCsvPayloadStillReturnsNoRows(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(67)
|
||||
const needleID = uint64(777004)
|
||||
const cookie = uint32(0xDCDCADAD)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
csvPayload := []byte("name,score\nalice,12\nbob,3\n")
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, needleID, cookie)
|
||||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, csvPayload)
|
||||
_ = framework.ReadAllAndClose(t, uploadResp)
|
||||
if uploadResp.StatusCode != 201 {
|
||||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
csvStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
|
||||
FromFileIds: []string{fid},
|
||||
Selections: []string{"score"},
|
||||
Filter: &volume_server_pb.QueryRequest_Filter{
|
||||
Field: "score",
|
||||
Operand: ">",
|
||||
Value: "10",
|
||||
},
|
||||
InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
|
||||
CsvInput: &volume_server_pb.QueryRequest_InputSerialization_CSVInput{},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Query csv start failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = csvStream.Recv()
|
||||
if err != io.EOF {
|
||||
t.Fatalf("Query csv with csv payload expected EOF with no rows, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryJsonNoMatchReturnsEmptyStripe(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
|
||||
@@ -12,8 +12,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
func TestVolumeTailSenderMissingVolume(t *testing.T) {
|
||||
@@ -93,36 +91,6 @@ func TestVolumeTailReceiverMissingVolume(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeTailReceiverSourceUnavailable(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(89)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := grpcClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
|
||||
VolumeId: volumeID,
|
||||
SourceVolumeServer: "127.0.0.1:19999.29999",
|
||||
SinceNs: 0,
|
||||
IdleTimeoutSeconds: 1,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("VolumeTailReceiver should fail when source volume server is unavailable")
|
||||
}
|
||||
lowered := strings.ToLower(err.Error())
|
||||
if !strings.Contains(lowered, "dial") && !strings.Contains(lowered, "unavailable") && !strings.Contains(lowered, "connection refused") {
|
||||
t.Fatalf("VolumeTailReceiver source-unavailable error mismatch: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeTailReceiverReplicatesSourceUpdates(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
@@ -236,46 +204,3 @@ func TestVolumeTailSenderLargeNeedleChunking(t *testing.T) {
|
||||
t.Fatalf("VolumeTailSender expected a final data chunk marked IsLastChunk=true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeTailSenderStreamCancellation(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(74)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{
|
||||
VolumeId: volumeID,
|
||||
SinceNs: 0,
|
||||
IdleTimeoutSeconds: 30,
|
||||
})
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatalf("VolumeTailSender start failed: %v", err)
|
||||
}
|
||||
|
||||
firstMsg, err := stream.Recv()
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatalf("VolumeTailSender first recv failed: %v", err)
|
||||
}
|
||||
if !firstMsg.GetIsLastChunk() {
|
||||
cancel()
|
||||
t.Fatalf("expected heartbeat first message before cancellation")
|
||||
}
|
||||
|
||||
cancel()
|
||||
_, err = stream.Recv()
|
||||
if err == nil {
|
||||
t.Fatalf("VolumeTailSender recv after cancellation expected error")
|
||||
}
|
||||
if status.Code(err) != codes.Canceled {
|
||||
t.Fatalf("VolumeTailSender cancellation expected grpc code Canceled, got %v (%v)", status.Code(err), err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -164,26 +164,6 @@ func TestUiIndexNotExposedWhenJwtSigningEnabled(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUiIndexExposedWhenJwtSigningEnabledAndAccessUITrue(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
profile := matrix.P3()
|
||||
profile.AccessUI = true
|
||||
cluster := framework.StartSingleVolumeCluster(t, profile)
|
||||
client := framework.NewHTTPClient()
|
||||
|
||||
resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html"))
|
||||
body := framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("expected /ui/index.html to be exposed when access.ui=true under JWT profile, got %d body=%s", resp.StatusCode, string(body))
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(string(body)), "volume") {
|
||||
t.Fatalf("ui page does not look like volume status page")
|
||||
}
|
||||
}
|
||||
|
||||
func mustNewRequest(t testing.TB, method, url string) *http.Request {
|
||||
t.Helper()
|
||||
req, err := http.NewRequest(method, url, nil)
|
||||
|
||||
@@ -251,50 +251,6 @@ func TestUnsupportedMethodConnectParity(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsupportedMethodMkcolParity(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(88)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
fid := framework.NewFileID(volumeID, 124004, 0x06060606)
|
||||
client := framework.NewHTTPClient()
|
||||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("mkcol-method-check"))
|
||||
_ = framework.ReadAllAndClose(t, uploadResp)
|
||||
if uploadResp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
|
||||
}
|
||||
|
||||
adminReq := mustNewRequest(t, "MKCOL", clusterHarness.VolumeAdminURL()+"/"+fid)
|
||||
adminResp := framework.DoRequest(t, client, adminReq)
|
||||
_ = framework.ReadAllAndClose(t, adminResp)
|
||||
if adminResp.StatusCode != http.StatusBadRequest {
|
||||
t.Fatalf("admin MKCOL expected 400, got %d", adminResp.StatusCode)
|
||||
}
|
||||
|
||||
publicReq := mustNewRequest(t, "MKCOL", clusterHarness.VolumePublicURL()+"/"+fid)
|
||||
publicResp := framework.DoRequest(t, client, publicReq)
|
||||
_ = framework.ReadAllAndClose(t, publicResp)
|
||||
if publicResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("public MKCOL expected passthrough 200, got %d", publicResp.StatusCode)
|
||||
}
|
||||
|
||||
verifyResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
|
||||
verifyBody := framework.ReadAllAndClose(t, verifyResp)
|
||||
if verifyResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("verify GET expected 200, got %d", verifyResp.StatusCode)
|
||||
}
|
||||
if string(verifyBody) != "mkcol-method-check" {
|
||||
t.Fatalf("MKCOL should not mutate data, got %q", string(verifyBody))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicPortHeadReadParity(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
|
||||
@@ -2,7 +2,6 @@ package volume_server_http_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
@@ -52,17 +51,4 @@ func TestReadDeletedQueryReturnsDeletedNeedleData(t *testing.T) {
|
||||
if string(readDeletedBody) != string(payload) {
|
||||
t.Fatalf("readDeleted body mismatch: got %q want %q", string(readDeletedBody), string(payload))
|
||||
}
|
||||
|
||||
headReadDeletedReq := mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+fid+"?readDeleted=true")
|
||||
headReadDeletedResp := framework.DoRequest(t, client, headReadDeletedReq)
|
||||
headReadDeletedBody := framework.ReadAllAndClose(t, headReadDeletedResp)
|
||||
if headReadDeletedResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("HEAD with readDeleted=true expected 200, got %d", headReadDeletedResp.StatusCode)
|
||||
}
|
||||
if len(headReadDeletedBody) != 0 {
|
||||
t.Fatalf("HEAD with readDeleted=true expected empty body, got %d bytes", len(headReadDeletedBody))
|
||||
}
|
||||
if got := headReadDeletedResp.Header.Get("Content-Length"); got != strconv.Itoa(len(payload)) {
|
||||
t.Fatalf("HEAD with readDeleted=true content-length mismatch: got %q want %d", got, len(payload))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -164,10 +164,6 @@ func TestConditionalHeaderPrecedenceAndInvalidIfModifiedSince(t *testing.T) {
|
||||
if lastModified == "" {
|
||||
t.Fatalf("baseline read expected Last-Modified header")
|
||||
}
|
||||
etag := baselineResp.Header.Get("ETag")
|
||||
if etag == "" {
|
||||
t.Fatalf("baseline read expected ETag header")
|
||||
}
|
||||
|
||||
precedenceReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
|
||||
precedenceReq.Header.Set("If-Modified-Since", lastModified)
|
||||
@@ -192,28 +188,4 @@ func TestConditionalHeaderPrecedenceAndInvalidIfModifiedSince(t *testing.T) {
|
||||
if string(invalidIMSBody) != string(payload) {
|
||||
t.Fatalf("invalid If-Modified-Since fallback body mismatch: got %q want %q", string(invalidIMSBody), string(payload))
|
||||
}
|
||||
|
||||
headIMSPrecedenceReq := mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+fid)
|
||||
headIMSPrecedenceReq.Header.Set("If-Modified-Since", lastModified)
|
||||
headIMSPrecedenceReq.Header.Set("If-None-Match", "\"definitely-different-etag\"")
|
||||
headIMSPrecedenceResp := framework.DoRequest(t, client, headIMSPrecedenceReq)
|
||||
headIMSPrecedenceBody := framework.ReadAllAndClose(t, headIMSPrecedenceResp)
|
||||
if headIMSPrecedenceResp.StatusCode != http.StatusNotModified {
|
||||
t.Fatalf("HEAD conditional precedence expected 304 when If-Modified-Since matches, got %d", headIMSPrecedenceResp.StatusCode)
|
||||
}
|
||||
if len(headIMSPrecedenceBody) != 0 {
|
||||
t.Fatalf("HEAD conditional precedence expected empty body, got %d bytes", len(headIMSPrecedenceBody))
|
||||
}
|
||||
|
||||
headETagPrecedenceReq := mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+fid)
|
||||
headETagPrecedenceReq.Header.Set("If-Modified-Since", "Thu, 01 Jan 1970 00:00:00 GMT")
|
||||
headETagPrecedenceReq.Header.Set("If-None-Match", etag)
|
||||
headETagPrecedenceResp := framework.DoRequest(t, client, headETagPrecedenceReq)
|
||||
headETagPrecedenceBody := framework.ReadAllAndClose(t, headETagPrecedenceResp)
|
||||
if headETagPrecedenceResp.StatusCode != http.StatusNotModified {
|
||||
t.Fatalf("HEAD conditional precedence expected 304 when If-None-Match matches, got %d", headETagPrecedenceResp.StatusCode)
|
||||
}
|
||||
if len(headETagPrecedenceBody) != 0 {
|
||||
t.Fatalf("HEAD conditional etag precedence expected empty body, got %d bytes", len(headETagPrecedenceBody))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package volume_server_http_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -73,64 +72,3 @@ func TestWriteMalformedMultipartAndMD5Mismatch(t *testing.T) {
|
||||
t.Fatalf("content-md5 mismatch response should mention Content-MD5, got %q", string(md5MismatchBody))
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteRejectsPayloadOverFileSizeLimit(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
profile := matrix.P1()
|
||||
profile.FileSizeLimitMB = 1
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, profile)
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(99)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
client := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, 772002, 0x2A3B4C5D)
|
||||
oversizedPayload := bytes.Repeat([]byte("z"), 1024*1024+1)
|
||||
|
||||
oversizedReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, oversizedPayload)
|
||||
oversizedResp := framework.DoRequest(t, client, oversizedReq)
|
||||
oversizedBody := framework.ReadAllAndClose(t, oversizedResp)
|
||||
if oversizedResp.StatusCode != http.StatusBadRequest {
|
||||
t.Fatalf("oversized write expected 400, got %d", oversizedResp.StatusCode)
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(string(oversizedBody)), "limited") {
|
||||
t.Fatalf("oversized write response should mention limit, got %q", string(oversizedBody))
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplicatedWriteFailsWhenReplicaRequirementsNotMet(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(109)
|
||||
framework.AllocateVolumeWithReplication(t, grpcClient, volumeID, "", "001")
|
||||
|
||||
client := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, 772003, 0x3A4B5C6D)
|
||||
payload := []byte("replicated-write-failure-path")
|
||||
|
||||
writeResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
|
||||
writeBody := framework.ReadAllAndClose(t, writeResp)
|
||||
if writeResp.StatusCode != http.StatusInternalServerError {
|
||||
t.Fatalf("replicated write with unmet replication requirements expected 500, got %d body=%s", writeResp.StatusCode, string(writeBody))
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(string(writeBody)), "replica") {
|
||||
t.Fatalf("replicated write failure response should mention replica write failure, got %q", string(writeBody))
|
||||
}
|
||||
|
||||
readResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
|
||||
_ = framework.ReadAllAndClose(t, readResp)
|
||||
if readResp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("local read after failed replicate write expected 404 (not committed locally), got %d", readResp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,14 +12,12 @@ type Profile struct {
|
||||
EnableJWT bool
|
||||
JWTSigningKey string
|
||||
JWTReadKey string
|
||||
AccessUI bool
|
||||
EnableMaintain bool
|
||||
|
||||
ConcurrentUploadLimitMB int
|
||||
ConcurrentDownloadLimitMB int
|
||||
InflightUploadTimeout time.Duration
|
||||
InflightDownloadTimeout time.Duration
|
||||
FileSizeLimitMB int
|
||||
|
||||
ReplicatedLayout bool
|
||||
HasErasureCoding bool
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
//go:build !windows && !openbsd && !netbsd && !plan9 && !solaris
|
||||
// +build !windows,!openbsd,!netbsd,!plan9,!solaris
|
||||
|
||||
package main
|
||||
|
||||
|
||||
@@ -0,0 +1,515 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
mrand "math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os/signal"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
type config struct {
|
||||
MasterAddresses []string
|
||||
FilerURL string
|
||||
PathPrefix string
|
||||
Collection string
|
||||
FileSizeBytes int64
|
||||
BatchSize int
|
||||
WriteInterval time.Duration
|
||||
CleanupInterval time.Duration
|
||||
EcMinAge time.Duration
|
||||
MaxCleanupPerCycle int
|
||||
RequestTimeout time.Duration
|
||||
MaxRuntime time.Duration
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
type runner struct {
|
||||
cfg config
|
||||
|
||||
httpClient *http.Client
|
||||
grpcDialOption grpc.DialOption
|
||||
|
||||
mu sync.Mutex
|
||||
sequence int64
|
||||
ecFirstSeenAt map[uint32]time.Time
|
||||
rng *mrand.Rand
|
||||
}
|
||||
|
||||
type ecVolumeInfo struct {
|
||||
Collection string
|
||||
NodeShards map[pb.ServerAddress][]uint32
|
||||
}
|
||||
|
||||
type ecCleanupCandidate struct {
|
||||
VolumeID uint32
|
||||
FirstSeenAt time.Time
|
||||
Info *ecVolumeInfo
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg, err := loadConfig()
|
||||
if err != nil {
|
||||
log.Fatalf("invalid flags: %v", err)
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
if cfg.MaxRuntime > 0 {
|
||||
runCtx, cancel := context.WithTimeout(ctx, cfg.MaxRuntime)
|
||||
defer cancel()
|
||||
ctx = runCtx
|
||||
}
|
||||
|
||||
r := &runner{
|
||||
cfg: cfg,
|
||||
httpClient: &http.Client{Timeout: cfg.RequestTimeout},
|
||||
grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
ecFirstSeenAt: make(map[uint32]time.Time),
|
||||
rng: mrand.New(mrand.NewSource(time.Now().UnixNano())),
|
||||
}
|
||||
|
||||
log.Printf(
|
||||
"starting EC stress runner: masters=%s filer=%s prefix=%s collection=%s file_size=%d batch=%d write_interval=%s cleanup_interval=%s ec_min_age=%s max_cleanup=%d dry_run=%v",
|
||||
strings.Join(cfg.MasterAddresses, ","),
|
||||
cfg.FilerURL,
|
||||
cfg.PathPrefix,
|
||||
cfg.Collection,
|
||||
cfg.FileSizeBytes,
|
||||
cfg.BatchSize,
|
||||
cfg.WriteInterval,
|
||||
cfg.CleanupInterval,
|
||||
cfg.EcMinAge,
|
||||
cfg.MaxCleanupPerCycle,
|
||||
cfg.DryRun,
|
||||
)
|
||||
|
||||
if err := r.run(ctx); err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
|
||||
log.Fatalf("runner stopped with error: %v", err)
|
||||
}
|
||||
|
||||
log.Printf("runner stopped")
|
||||
}
|
||||
|
||||
func loadConfig() (config, error) {
|
||||
var masters string
|
||||
cfg := config{}
|
||||
|
||||
flag.StringVar(&masters, "masters", "127.0.0.1:9333", "comma-separated master server addresses")
|
||||
flag.StringVar(&cfg.FilerURL, "filer", "http://127.0.0.1:8888", "filer base URL")
|
||||
flag.StringVar(&cfg.PathPrefix, "path_prefix", "/tmp/ec-stress", "filer path prefix for generated files")
|
||||
flag.StringVar(&cfg.Collection, "collection", "ec_stress", "target collection for stress data")
|
||||
|
||||
fileSizeMB := flag.Int("file_size_mb", 8, "size per generated file in MB")
|
||||
flag.IntVar(&cfg.BatchSize, "batch_size", 4, "files generated per write cycle")
|
||||
flag.DurationVar(&cfg.WriteInterval, "write_interval", 5*time.Second, "interval between write cycles")
|
||||
flag.DurationVar(&cfg.CleanupInterval, "cleanup_interval", 2*time.Minute, "interval between EC cleanup cycles")
|
||||
flag.DurationVar(&cfg.EcMinAge, "ec_min_age", 30*time.Minute, "minimum observed EC age before deletion")
|
||||
flag.IntVar(&cfg.MaxCleanupPerCycle, "max_cleanup_per_cycle", 4, "maximum EC volumes deleted per cleanup cycle")
|
||||
flag.DurationVar(&cfg.RequestTimeout, "request_timeout", 20*time.Second, "HTTP/gRPC request timeout")
|
||||
flag.DurationVar(&cfg.MaxRuntime, "max_runtime", 0, "maximum run duration; 0 means run until interrupted")
|
||||
flag.BoolVar(&cfg.DryRun, "dry_run", false, "log actions without deleting EC shards")
|
||||
flag.Parse()
|
||||
|
||||
cfg.MasterAddresses = splitNonEmpty(masters)
|
||||
cfg.FileSizeBytes = int64(*fileSizeMB) * 1024 * 1024
|
||||
|
||||
if len(cfg.MasterAddresses) == 0 {
|
||||
return cfg, fmt.Errorf("at least one master is required")
|
||||
}
|
||||
if cfg.FileSizeBytes <= 0 {
|
||||
return cfg, fmt.Errorf("file_size_mb must be positive")
|
||||
}
|
||||
if cfg.BatchSize <= 0 {
|
||||
return cfg, fmt.Errorf("batch_size must be positive")
|
||||
}
|
||||
if cfg.WriteInterval <= 0 {
|
||||
return cfg, fmt.Errorf("write_interval must be positive")
|
||||
}
|
||||
if cfg.CleanupInterval <= 0 {
|
||||
return cfg, fmt.Errorf("cleanup_interval must be positive")
|
||||
}
|
||||
if cfg.EcMinAge < 0 {
|
||||
return cfg, fmt.Errorf("ec_min_age must be zero or positive")
|
||||
}
|
||||
// Note: EcMinAge == 0 intentionally disables the age guard, making EC volumes eligible for cleanup immediately.
|
||||
if cfg.MaxCleanupPerCycle <= 0 {
|
||||
return cfg, fmt.Errorf("max_cleanup_per_cycle must be positive")
|
||||
}
|
||||
if cfg.RequestTimeout <= 0 {
|
||||
return cfg, fmt.Errorf("request_timeout must be positive")
|
||||
}
|
||||
|
||||
cfg.PathPrefix = ensureLeadingSlash(strings.TrimSpace(cfg.PathPrefix))
|
||||
cfg.Collection = strings.TrimSpace(cfg.Collection)
|
||||
|
||||
cfg.FilerURL = strings.TrimRight(strings.TrimSpace(cfg.FilerURL), "/")
|
||||
if cfg.FilerURL == "" {
|
||||
return cfg, fmt.Errorf("filer URL is required")
|
||||
}
|
||||
|
||||
if _, err := url.ParseRequestURI(cfg.FilerURL); err != nil {
|
||||
return cfg, fmt.Errorf("invalid filer URL %q: %w", cfg.FilerURL, err)
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func (r *runner) run(ctx context.Context) error {
|
||||
writeTicker := time.NewTicker(r.cfg.WriteInterval)
|
||||
defer writeTicker.Stop()
|
||||
|
||||
cleanupTicker := time.NewTicker(r.cfg.CleanupInterval)
|
||||
defer cleanupTicker.Stop()
|
||||
|
||||
r.runWriteCycle(ctx)
|
||||
r.runCleanupCycle(ctx)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-writeTicker.C:
|
||||
r.runWriteCycle(ctx)
|
||||
case <-cleanupTicker.C:
|
||||
r.runCleanupCycle(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *runner) runWriteCycle(ctx context.Context) {
|
||||
for i := 0; i < r.cfg.BatchSize; i++ {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if err := r.uploadOneFile(ctx); err != nil {
|
||||
log.Printf("upload failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *runner) uploadOneFile(ctx context.Context) error {
|
||||
sequence := r.nextSequence()
|
||||
filePath := path.Join(r.cfg.PathPrefix, fmt.Sprintf("ec-stress-%d-%d.bin", time.Now().UnixNano(), sequence))
|
||||
fileURL := r.cfg.FilerURL + filePath
|
||||
if r.cfg.Collection != "" {
|
||||
fileURL += "?collection=" + url.QueryEscape(r.cfg.Collection)
|
||||
}
|
||||
|
||||
uploadCtx, cancel := context.WithTimeout(ctx, r.cfg.RequestTimeout)
|
||||
defer cancel()
|
||||
|
||||
body := io.LimitReader(r.rng, r.cfg.FileSizeBytes)
|
||||
request, err := http.NewRequestWithContext(uploadCtx, http.MethodPut, fileURL, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request.ContentLength = r.cfg.FileSizeBytes
|
||||
request.Header.Set("Content-Type", "application/octet-stream")
|
||||
|
||||
response, err := r.httpClient.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
io.Copy(io.Discard, response.Body)
|
||||
if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {
|
||||
return fmt.Errorf("upload %s returned %s", filePath, response.Status)
|
||||
}
|
||||
|
||||
log.Printf("uploaded %s size=%d", filePath, r.cfg.FileSizeBytes)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *runner) runCleanupCycle(ctx context.Context) {
|
||||
volumeList, err := r.fetchVolumeList(ctx)
|
||||
if err != nil {
|
||||
log.Printf("cleanup skipped: fetch volume list failed: %v", err)
|
||||
return
|
||||
}
|
||||
if volumeList == nil || volumeList.TopologyInfo == nil {
|
||||
log.Printf("cleanup skipped: topology is empty")
|
||||
return
|
||||
}
|
||||
|
||||
ecVolumes := collectEcVolumes(volumeList.TopologyInfo, r.cfg.Collection)
|
||||
candidates := r.selectCleanupCandidates(ecVolumes)
|
||||
if len(candidates) == 0 {
|
||||
log.Printf("cleanup: no EC volume candidate aged >= %s in collection=%q", r.cfg.EcMinAge, r.cfg.Collection)
|
||||
return
|
||||
}
|
||||
|
||||
log.Printf("cleanup: deleting up to %d EC volumes (found=%d)", r.cfg.MaxCleanupPerCycle, len(candidates))
|
||||
deleted := 0
|
||||
for _, candidate := range candidates {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if r.cfg.DryRun {
|
||||
log.Printf(
|
||||
"cleanup dry-run: would delete EC volume=%d collection=%q seen_for=%s nodes=%d",
|
||||
candidate.VolumeID,
|
||||
candidate.Info.Collection,
|
||||
time.Since(candidate.FirstSeenAt).Round(time.Second),
|
||||
len(candidate.Info.NodeShards),
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
if err := r.deleteEcVolume(ctx, candidate.VolumeID, candidate.Info); err != nil {
|
||||
log.Printf("cleanup volume=%d failed: %v", candidate.VolumeID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
deleted++
|
||||
r.mu.Lock()
|
||||
delete(r.ecFirstSeenAt, candidate.VolumeID)
|
||||
r.mu.Unlock()
|
||||
log.Printf("cleanup volume=%d completed", candidate.VolumeID)
|
||||
}
|
||||
|
||||
log.Printf("cleanup finished: deleted=%d attempted=%d", deleted, len(candidates))
|
||||
}
|
||||
|
||||
func (r *runner) fetchVolumeList(ctx context.Context) (*master_pb.VolumeListResponse, error) {
|
||||
var lastErr error
|
||||
for _, master := range r.cfg.MasterAddresses {
|
||||
masterAddress := strings.TrimSpace(master)
|
||||
if masterAddress == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
var response *master_pb.VolumeListResponse
|
||||
err := pb.WithMasterClient(false, pb.ServerAddress(masterAddress), r.grpcDialOption, false, func(client master_pb.SeaweedClient) error {
|
||||
callCtx, cancel := context.WithTimeout(ctx, r.cfg.RequestTimeout)
|
||||
defer cancel()
|
||||
|
||||
resp, callErr := client.VolumeList(callCtx, &master_pb.VolumeListRequest{})
|
||||
if callErr != nil {
|
||||
return callErr
|
||||
}
|
||||
response = resp
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
return response, nil
|
||||
}
|
||||
|
||||
lastErr = err
|
||||
}
|
||||
|
||||
if lastErr == nil {
|
||||
lastErr = fmt.Errorf("no valid master address")
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
func collectEcVolumes(topology *master_pb.TopologyInfo, collection string) map[uint32]*ecVolumeInfo {
|
||||
normalizedCollection := strings.TrimSpace(collection)
|
||||
volumeShardSets := make(map[uint32]map[pb.ServerAddress]map[uint32]struct{})
|
||||
volumeCollection := make(map[uint32]string)
|
||||
|
||||
for _, dc := range topology.GetDataCenterInfos() {
|
||||
for _, rack := range dc.GetRackInfos() {
|
||||
for _, node := range rack.GetDataNodeInfos() {
|
||||
server := pb.NewServerAddressFromDataNode(node)
|
||||
for _, disk := range node.GetDiskInfos() {
|
||||
for _, shardInfo := range disk.GetEcShardInfos() {
|
||||
if shardInfo == nil || shardInfo.Id == 0 {
|
||||
continue
|
||||
}
|
||||
if normalizedCollection != "" && strings.TrimSpace(shardInfo.Collection) != normalizedCollection {
|
||||
continue
|
||||
}
|
||||
|
||||
shards := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(shardInfo).IdsUint32()
|
||||
if len(shards) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
perVolume := volumeShardSets[shardInfo.Id]
|
||||
if perVolume == nil {
|
||||
perVolume = make(map[pb.ServerAddress]map[uint32]struct{})
|
||||
volumeShardSets[shardInfo.Id] = perVolume
|
||||
}
|
||||
perNode := perVolume[server]
|
||||
if perNode == nil {
|
||||
perNode = make(map[uint32]struct{})
|
||||
perVolume[server] = perNode
|
||||
}
|
||||
for _, shardID := range shards {
|
||||
perNode[shardID] = struct{}{}
|
||||
}
|
||||
volumeCollection[shardInfo.Id] = shardInfo.Collection
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result := make(map[uint32]*ecVolumeInfo, len(volumeShardSets))
|
||||
for volumeID, perNode := range volumeShardSets {
|
||||
info := &ecVolumeInfo{
|
||||
Collection: volumeCollection[volumeID],
|
||||
NodeShards: make(map[pb.ServerAddress][]uint32, len(perNode)),
|
||||
}
|
||||
for server, shardSet := range perNode {
|
||||
shardIDs := make([]uint32, 0, len(shardSet))
|
||||
for shardID := range shardSet {
|
||||
shardIDs = append(shardIDs, shardID)
|
||||
}
|
||||
sort.Slice(shardIDs, func(i, j int) bool { return shardIDs[i] < shardIDs[j] })
|
||||
info.NodeShards[server] = shardIDs
|
||||
}
|
||||
result[volumeID] = info
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (r *runner) selectCleanupCandidates(ecVolumes map[uint32]*ecVolumeInfo) []ecCleanupCandidate {
|
||||
now := time.Now()
|
||||
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
for volumeID := range ecVolumes {
|
||||
if _, exists := r.ecFirstSeenAt[volumeID]; !exists {
|
||||
r.ecFirstSeenAt[volumeID] = now
|
||||
}
|
||||
}
|
||||
for volumeID := range r.ecFirstSeenAt {
|
||||
if _, exists := ecVolumes[volumeID]; !exists {
|
||||
delete(r.ecFirstSeenAt, volumeID)
|
||||
}
|
||||
}
|
||||
|
||||
candidates := make([]ecCleanupCandidate, 0, len(ecVolumes))
|
||||
for volumeID, info := range ecVolumes {
|
||||
firstSeenAt := r.ecFirstSeenAt[volumeID]
|
||||
if r.cfg.EcMinAge > 0 && now.Sub(firstSeenAt) < r.cfg.EcMinAge {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, ecCleanupCandidate{
|
||||
VolumeID: volumeID,
|
||||
FirstSeenAt: firstSeenAt,
|
||||
Info: info,
|
||||
})
|
||||
}
|
||||
|
||||
sort.Slice(candidates, func(i, j int) bool {
|
||||
if candidates[i].FirstSeenAt.Equal(candidates[j].FirstSeenAt) {
|
||||
return candidates[i].VolumeID < candidates[j].VolumeID
|
||||
}
|
||||
return candidates[i].FirstSeenAt.Before(candidates[j].FirstSeenAt)
|
||||
})
|
||||
|
||||
if len(candidates) > r.cfg.MaxCleanupPerCycle {
|
||||
candidates = candidates[:r.cfg.MaxCleanupPerCycle]
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
func (r *runner) deleteEcVolume(ctx context.Context, volumeID uint32, info *ecVolumeInfo) error {
|
||||
if info == nil {
|
||||
return fmt.Errorf("ec volume %d has no topology info", volumeID)
|
||||
}
|
||||
|
||||
failureCount := 0
|
||||
for server, shardIDs := range info.NodeShards {
|
||||
err := pb.WithVolumeServerClient(false, server, r.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
|
||||
unmountCtx, unmountCancel := context.WithTimeout(ctx, r.cfg.RequestTimeout)
|
||||
defer unmountCancel()
|
||||
if _, err := client.VolumeEcShardsUnmount(unmountCtx, &volume_server_pb.VolumeEcShardsUnmountRequest{
|
||||
VolumeId: volumeID,
|
||||
ShardIds: shardIDs,
|
||||
}); err != nil {
|
||||
log.Printf("volume %d ec shards unmount on %s failed: %v", volumeID, server, err)
|
||||
}
|
||||
|
||||
if len(shardIDs) > 0 {
|
||||
deleteCtx, deleteCancel := context.WithTimeout(ctx, r.cfg.RequestTimeout)
|
||||
defer deleteCancel()
|
||||
if _, err := client.VolumeEcShardsDelete(deleteCtx, &volume_server_pb.VolumeEcShardsDeleteRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: r.cfg.Collection,
|
||||
ShardIds: shardIDs,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
finalDeleteCtx, finalDeleteCancel := context.WithTimeout(ctx, r.cfg.RequestTimeout)
|
||||
defer finalDeleteCancel()
|
||||
if _, err := client.VolumeDelete(finalDeleteCtx, &volume_server_pb.VolumeDeleteRequest{
|
||||
VolumeId: volumeID,
|
||||
}); err != nil {
|
||||
log.Printf("volume %d delete on %s failed: %v", volumeID, server, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
failureCount++
|
||||
log.Printf("cleanup volume=%d server=%s shards=%v failed: %v", volumeID, server, shardIDs, err)
|
||||
}
|
||||
}
|
||||
|
||||
if failureCount == len(info.NodeShards) && failureCount > 0 {
|
||||
return fmt.Errorf("all shard deletions failed for volume %d", volumeID)
|
||||
}
|
||||
if failureCount > 0 {
|
||||
return fmt.Errorf("partial shard deletion failure for volume %d", volumeID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *runner) nextSequence() int64 {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.sequence++
|
||||
return r.sequence
|
||||
}
|
||||
|
||||
func splitNonEmpty(value string) []string {
|
||||
parts := strings.Split(value, ",")
|
||||
result := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
trimmed := strings.TrimSpace(part)
|
||||
if trimmed != "" {
|
||||
result = append(result, trimmed)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func ensureLeadingSlash(value string) string {
|
||||
if value == "" {
|
||||
return "/"
|
||||
}
|
||||
if strings.HasPrefix(value, "/") {
|
||||
return value
|
||||
}
|
||||
return "/" + value
|
||||
}
|
||||
@@ -9,6 +9,14 @@ all: install
|
||||
install:
|
||||
go install -ldflags="-s -w"
|
||||
|
||||
.PHONY: weed-db weed-sql
|
||||
|
||||
weed-db:
|
||||
go build -ldflags="-s -w" -o weed-db ./cmd/weed-db
|
||||
|
||||
weed-sql:
|
||||
go build -ldflags="-s -w" -o weed-sql ./cmd/weed-sql
|
||||
|
||||
build_docker:
|
||||
CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w"
|
||||
|
||||
|
||||
+1
-1
@@ -23,7 +23,7 @@ install-templ:
|
||||
.PHONY: generate
|
||||
generate: install-templ
|
||||
@echo "Generating templ files..."
|
||||
@cd $(BUILD_DIR) && templ generate
|
||||
@templ generate ./view
|
||||
@echo "Generated: $(TEMPL_GO_FILES)"
|
||||
|
||||
# Clean generated files
|
||||
|
||||
@@ -6,10 +6,10 @@ import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/iam"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
)
|
||||
|
||||
@@ -184,28 +184,28 @@ func (s *AdminServer) GetAdminData(username string) (AdminData, error) {
|
||||
}
|
||||
|
||||
// ShowAdmin displays the main admin page (now uses GetAdminData)
|
||||
func (s *AdminServer) ShowAdmin(c *gin.Context) {
|
||||
username := c.GetString("username")
|
||||
func (s *AdminServer) ShowAdmin(w http.ResponseWriter, r *http.Request) {
|
||||
username := UsernameFromContext(r.Context())
|
||||
|
||||
adminData, err := s.GetAdminData(username)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get admin data: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get admin data: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Return JSON for API calls
|
||||
c.JSON(http.StatusOK, adminData)
|
||||
writeJSON(w, http.StatusOK, adminData)
|
||||
}
|
||||
|
||||
// ShowOverview displays cluster overview
|
||||
func (s *AdminServer) ShowOverview(c *gin.Context) {
|
||||
func (s *AdminServer) ShowOverview(w http.ResponseWriter, r *http.Request) {
|
||||
topology, err := s.GetClusterTopology()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, topology)
|
||||
writeJSON(w, http.StatusOK, topology)
|
||||
}
|
||||
|
||||
// getMasterNodesStatus checks status of all master nodes
|
||||
@@ -233,7 +233,7 @@ func (s *AdminServer) getMasterNodesStatus() []MasterNode {
|
||||
currentMaster := s.masterClient.GetMaster(context.Background())
|
||||
if currentMaster != "" {
|
||||
masterNodes = append(masterNodes, MasterNode{
|
||||
Address: string(currentMaster),
|
||||
Address: pb.ServerAddress(currentMaster).ToHttpAddress(),
|
||||
IsLeader: isLeader,
|
||||
})
|
||||
}
|
||||
|
||||
+203
-815
File diff suppressed because it is too large
Load Diff
@@ -4,81 +4,91 @@ import (
|
||||
"crypto/subtle"
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-contrib/sessions"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/gorilla/sessions"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
)
|
||||
|
||||
// ShowLogin displays the login page
|
||||
func (s *AdminServer) ShowLogin(c *gin.Context) {
|
||||
// If authentication is not required, redirect to admin
|
||||
session := sessions.Default(c)
|
||||
if session.Get("authenticated") == true {
|
||||
c.Redirect(http.StatusSeeOther, "/admin")
|
||||
return
|
||||
}
|
||||
|
||||
// For now, return a simple login form as JSON
|
||||
c.HTML(http.StatusOK, "login.html", gin.H{
|
||||
"title": "SeaweedFS Admin Login",
|
||||
"error": c.Query("error"),
|
||||
})
|
||||
// ShowLogin displays the login page.
|
||||
func (s *AdminServer) ShowLogin(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, "/login", http.StatusSeeOther)
|
||||
}
|
||||
|
||||
// HandleLogin handles login form submission
|
||||
func (s *AdminServer) HandleLogin(adminUser, adminPassword, readOnlyUser, readOnlyPassword string) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
loginUsername := c.PostForm("username")
|
||||
loginPassword := c.PostForm("password")
|
||||
// HandleLogin handles login form submission.
|
||||
func (s *AdminServer) HandleLogin(store sessions.Store, adminUser, adminPassword, readOnlyUser, readOnlyPassword string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Redirect(w, r, "/login?error=Invalid form submission", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
session, err := store.Get(r, sessionName)
|
||||
if err != nil {
|
||||
http.Redirect(w, r, "/login?error=Unable to create session. Please try again or contact administrator.", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
|
||||
if err := ValidateSessionCSRFToken(session, r); err != nil {
|
||||
http.Redirect(w, r, "/login?error=Invalid CSRF token", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
|
||||
loginUsername := r.FormValue("username")
|
||||
loginPassword := r.FormValue("password")
|
||||
|
||||
var role string
|
||||
var authenticated bool
|
||||
|
||||
// Check admin credentials
|
||||
// Check admin credentials.
|
||||
if adminPassword != "" && loginUsername == adminUser && subtle.ConstantTimeCompare([]byte(loginPassword), []byte(adminPassword)) == 1 {
|
||||
role = "admin"
|
||||
authenticated = true
|
||||
} else if readOnlyPassword != "" && loginUsername == readOnlyUser && subtle.ConstantTimeCompare([]byte(loginPassword), []byte(readOnlyPassword)) == 1 {
|
||||
// Check read-only credentials
|
||||
// Check read-only credentials.
|
||||
role = "readonly"
|
||||
authenticated = true
|
||||
}
|
||||
|
||||
if authenticated {
|
||||
session := sessions.Default(c)
|
||||
// Clear any existing invalid session data before setting new values
|
||||
session.Clear()
|
||||
session.Set("authenticated", true)
|
||||
session.Set("username", loginUsername)
|
||||
session.Set("role", role)
|
||||
for key := range session.Values {
|
||||
delete(session.Values, key)
|
||||
}
|
||||
session.Values["authenticated"] = true
|
||||
session.Values["username"] = loginUsername
|
||||
session.Values["role"] = role
|
||||
csrfToken, err := generateCSRFToken()
|
||||
if err != nil {
|
||||
c.Redirect(http.StatusSeeOther, "/login?error=Unable to create session. Please try again or contact administrator.")
|
||||
http.Redirect(w, r, "/login?error=Unable to create session. Please try again or contact administrator.", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
session.Set(sessionCSRFTokenKey, csrfToken)
|
||||
if err := session.Save(); err != nil {
|
||||
// Log the detailed error server-side for diagnostics
|
||||
session.Values[sessionCSRFTokenKey] = csrfToken
|
||||
if err := session.Save(r, w); err != nil {
|
||||
// Log the detailed error server-side for diagnostics.
|
||||
glog.Errorf("Failed to save session for user %s: %v", loginUsername, err)
|
||||
c.Redirect(http.StatusSeeOther, "/login?error=Unable to create session. Please try again or contact administrator.")
|
||||
http.Redirect(w, r, "/login?error=Unable to create session. Please try again or contact administrator.", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
|
||||
c.Redirect(http.StatusSeeOther, "/admin")
|
||||
http.Redirect(w, r, "/admin", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
|
||||
// Authentication failed
|
||||
c.Redirect(http.StatusSeeOther, "/login?error=Invalid credentials")
|
||||
// Authentication failed.
|
||||
http.Redirect(w, r, "/login?error=Invalid credentials", http.StatusSeeOther)
|
||||
}
|
||||
}
|
||||
|
||||
// HandleLogout handles user logout
|
||||
func (s *AdminServer) HandleLogout(c *gin.Context) {
|
||||
session := sessions.Default(c)
|
||||
session.Clear()
|
||||
if err := session.Save(); err != nil {
|
||||
// HandleLogout handles user logout.
|
||||
func (s *AdminServer) HandleLogout(store sessions.Store, w http.ResponseWriter, r *http.Request) {
|
||||
session, err := store.Get(r, sessionName)
|
||||
if err != nil {
|
||||
http.Redirect(w, r, "/login", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
for key := range session.Values {
|
||||
delete(session.Values, key)
|
||||
}
|
||||
session.Options.MaxAge = -1
|
||||
if err := session.Save(r, w); err != nil {
|
||||
glog.Warningf("Failed to save session during logout: %v", err)
|
||||
}
|
||||
c.Redirect(http.StatusSeeOther, "/login")
|
||||
http.Redirect(w, r, "/login", http.StatusSeeOther)
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
@@ -29,7 +29,7 @@ type S3BucketsData struct {
|
||||
}
|
||||
|
||||
type CreateBucketRequest struct {
|
||||
Name string `json:"name" binding:"required"`
|
||||
Name string `json:"name"` // validated manually in CreateBucket
|
||||
Region string `json:"region"`
|
||||
QuotaSize int64 `json:"quota_size"` // Quota size in bytes
|
||||
QuotaUnit string `json:"quota_unit"` // Unit: MB, GB, TB
|
||||
@@ -45,47 +45,51 @@ type CreateBucketRequest struct {
|
||||
// S3 Bucket Management Handlers
|
||||
|
||||
// ShowS3Buckets displays the Object Store buckets management page
|
||||
func (s *AdminServer) ShowS3Buckets(c *gin.Context) {
|
||||
username := c.GetString("username")
|
||||
func (s *AdminServer) ShowS3Buckets(w http.ResponseWriter, r *http.Request) {
|
||||
username := UsernameFromContext(r.Context())
|
||||
|
||||
data, err := s.GetS3BucketsData()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get Object Store buckets: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get Object Store buckets: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
data.Username = username
|
||||
c.JSON(http.StatusOK, data)
|
||||
writeJSON(w, http.StatusOK, data)
|
||||
}
|
||||
|
||||
// ShowBucketDetails displays detailed information about a specific bucket
|
||||
func (s *AdminServer) ShowBucketDetails(c *gin.Context) {
|
||||
bucketName := c.Param("bucket")
|
||||
func (s *AdminServer) ShowBucketDetails(w http.ResponseWriter, r *http.Request) {
|
||||
bucketName := mux.Vars(r)["bucket"]
|
||||
if bucketName == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Bucket name is required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Bucket name is required")
|
||||
return
|
||||
}
|
||||
|
||||
details, err := s.GetBucketDetails(bucketName)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get bucket details: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get bucket details: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, details)
|
||||
writeJSON(w, http.StatusOK, details)
|
||||
}
|
||||
|
||||
// CreateBucket creates a new S3 bucket
|
||||
func (s *AdminServer) CreateBucket(c *gin.Context) {
|
||||
func (s *AdminServer) CreateBucket(w http.ResponseWriter, r *http.Request) {
|
||||
var req CreateBucketRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(req.Name) == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "Bucket name is required")
|
||||
return
|
||||
}
|
||||
|
||||
// Validate bucket name (basic validation)
|
||||
if len(req.Name) < 3 || len(req.Name) > 63 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Bucket name must be between 3 and 63 characters"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Bucket name must be between 3 and 63 characters")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -96,42 +100,47 @@ func (s *AdminServer) CreateBucket(c *gin.Context) {
|
||||
|
||||
// Validate object lock mode
|
||||
if req.ObjectLockMode != "GOVERNANCE" && req.ObjectLockMode != "COMPLIANCE" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Object lock mode must be either GOVERNANCE or COMPLIANCE"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Object lock mode must be either GOVERNANCE or COMPLIANCE")
|
||||
return
|
||||
}
|
||||
|
||||
// Validate retention duration if default retention is enabled
|
||||
if req.SetDefaultRetention {
|
||||
if req.ObjectLockDuration <= 0 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Object lock duration must be greater than 0 days when default retention is enabled"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Object lock duration must be greater than 0 days when default retention is enabled")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Convert quota to bytes
|
||||
quotaBytes := convertQuotaToBytes(req.QuotaSize, req.QuotaUnit)
|
||||
normalizedUnit, err := normalizeQuotaUnit(req.QuotaUnit)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
req.QuotaUnit = normalizedUnit
|
||||
quotaBytes := convertQuotaToBytes(req.QuotaSize, normalizedUnit)
|
||||
|
||||
// Validate quota: if enabled, size must be greater than 0
|
||||
if req.QuotaEnabled && quotaBytes <= 0 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Quota size must be greater than 0 when quota is enabled"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Quota size must be greater than 0 when quota is enabled")
|
||||
return
|
||||
}
|
||||
|
||||
// Sanitize owner: trim whitespace and enforce max length
|
||||
owner := strings.TrimSpace(req.Owner)
|
||||
if len(owner) > MaxOwnerNameLength {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Owner name must be %d characters or less", MaxOwnerNameLength)})
|
||||
writeJSONError(w, http.StatusBadRequest, fmt.Sprintf("Owner name must be %d characters or less", MaxOwnerNameLength))
|
||||
return
|
||||
}
|
||||
|
||||
err := s.CreateS3BucketWithObjectLock(req.Name, quotaBytes, req.QuotaEnabled, req.VersioningEnabled, req.ObjectLockEnabled, req.ObjectLockMode, req.SetDefaultRetention, req.ObjectLockDuration, owner)
|
||||
err = s.CreateS3BucketWithObjectLock(req.Name, quotaBytes, req.QuotaEnabled, req.VersioningEnabled, req.ObjectLockEnabled, req.ObjectLockMode, req.SetDefaultRetention, req.ObjectLockDuration, owner)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create bucket: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to create bucket: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusCreated, gin.H{
|
||||
writeJSON(w, http.StatusCreated, map[string]interface{}{
|
||||
"message": "Bucket created successfully",
|
||||
"bucket": req.Name,
|
||||
"quota_size": req.QuotaSize,
|
||||
@@ -146,10 +155,10 @@ func (s *AdminServer) CreateBucket(c *gin.Context) {
|
||||
}
|
||||
|
||||
// UpdateBucketQuota updates the quota settings for a bucket
|
||||
func (s *AdminServer) UpdateBucketQuota(c *gin.Context) {
|
||||
bucketName := c.Param("bucket")
|
||||
func (s *AdminServer) UpdateBucketQuota(w http.ResponseWriter, r *http.Request) {
|
||||
bucketName := mux.Vars(r)["bucket"]
|
||||
if bucketName == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Bucket name is required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Bucket name is required")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -158,21 +167,32 @@ func (s *AdminServer) UpdateBucketQuota(c *gin.Context) {
|
||||
QuotaUnit string `json:"quota_unit"`
|
||||
QuotaEnabled bool `json:"quota_enabled"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Convert quota to bytes
|
||||
quotaBytes := convertQuotaToBytes(req.QuotaSize, req.QuotaUnit)
|
||||
if req.QuotaEnabled && req.QuotaSize <= 0 {
|
||||
writeJSONError(w, http.StatusBadRequest, "quota_size must be > 0 when quota_enabled is true")
|
||||
return
|
||||
}
|
||||
|
||||
err := s.SetBucketQuota(bucketName, quotaBytes, req.QuotaEnabled)
|
||||
normalizedUnit, err := normalizeQuotaUnit(req.QuotaUnit)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update bucket quota: " + err.Error()})
|
||||
writeJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
req.QuotaUnit = normalizedUnit
|
||||
// Convert quota to bytes
|
||||
quotaBytes := convertQuotaToBytes(req.QuotaSize, normalizedUnit)
|
||||
|
||||
err = s.SetBucketQuota(bucketName, quotaBytes, req.QuotaEnabled)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to update bucket quota: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"message": "Bucket quota updated successfully",
|
||||
"bucket": bucketName,
|
||||
"quota_size": req.QuotaSize,
|
||||
@@ -182,30 +202,30 @@ func (s *AdminServer) UpdateBucketQuota(c *gin.Context) {
|
||||
}
|
||||
|
||||
// DeleteBucket deletes an S3 bucket
|
||||
func (s *AdminServer) DeleteBucket(c *gin.Context) {
|
||||
bucketName := c.Param("bucket")
|
||||
func (s *AdminServer) DeleteBucket(w http.ResponseWriter, r *http.Request) {
|
||||
bucketName := mux.Vars(r)["bucket"]
|
||||
if bucketName == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Bucket name is required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Bucket name is required")
|
||||
return
|
||||
}
|
||||
|
||||
err := s.DeleteS3Bucket(bucketName)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to delete bucket: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to delete bucket: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"message": "Bucket deleted successfully",
|
||||
"bucket": bucketName,
|
||||
})
|
||||
}
|
||||
|
||||
// UpdateBucketOwner updates the owner of an S3 bucket
|
||||
func (s *AdminServer) UpdateBucketOwner(c *gin.Context) {
|
||||
bucketName := c.Param("bucket")
|
||||
func (s *AdminServer) UpdateBucketOwner(w http.ResponseWriter, r *http.Request) {
|
||||
bucketName := mux.Vars(r)["bucket"]
|
||||
if bucketName == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Bucket name is required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Bucket name is required")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -213,31 +233,31 @@ func (s *AdminServer) UpdateBucketOwner(c *gin.Context) {
|
||||
var req struct {
|
||||
Owner *string `json:"owner"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Require owner field to be explicitly provided
|
||||
if req.Owner == nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Owner field is required (use empty string to clear owner)"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Owner field is required (use empty string to clear owner)")
|
||||
return
|
||||
}
|
||||
|
||||
// Trim and validate owner
|
||||
owner := strings.TrimSpace(*req.Owner)
|
||||
if len(owner) > MaxOwnerNameLength {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Owner name must be %d characters or less", MaxOwnerNameLength)})
|
||||
writeJSONError(w, http.StatusBadRequest, fmt.Sprintf("Owner name must be %d characters or less", MaxOwnerNameLength))
|
||||
return
|
||||
}
|
||||
|
||||
err := s.SetBucketOwner(bucketName, owner)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update bucket owner: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to update bucket owner: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"message": "Bucket owner updated successfully",
|
||||
"bucket": bucketName,
|
||||
"owner": owner,
|
||||
@@ -284,14 +304,14 @@ func (s *AdminServer) SetBucketOwner(bucketName string, owner string) error {
|
||||
}
|
||||
|
||||
// ListBucketsAPI returns the list of buckets as JSON
|
||||
func (s *AdminServer) ListBucketsAPI(c *gin.Context) {
|
||||
func (s *AdminServer) ListBucketsAPI(w http.ResponseWriter, r *http.Request) {
|
||||
buckets, err := s.GetS3Buckets()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get buckets: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get buckets: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"buckets": buckets,
|
||||
"total": len(buckets),
|
||||
})
|
||||
@@ -303,16 +323,32 @@ func convertQuotaToBytes(size int64, unit string) int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
switch strings.ToUpper(unit) {
|
||||
switch unit {
|
||||
case "TB":
|
||||
return size * 1024 * 1024 * 1024 * 1024
|
||||
case "GB":
|
||||
return size * 1024 * 1024 * 1024
|
||||
case "MB":
|
||||
return size * 1024 * 1024
|
||||
case "KB":
|
||||
return size * 1024
|
||||
case "B":
|
||||
return size
|
||||
default:
|
||||
// Default to MB if unit is not recognized
|
||||
return size * 1024 * 1024
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeQuotaUnit(unit string) (string, error) {
|
||||
normalized := strings.ToUpper(strings.TrimSpace(unit))
|
||||
if normalized == "" {
|
||||
return "MB", nil
|
||||
}
|
||||
switch normalized {
|
||||
case "B", "KB", "MB", "GB", "TB":
|
||||
return normalized, nil
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported quota unit: %s", unit)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
|
||||
@@ -87,6 +88,11 @@ func isValidTaskID(taskID string) bool {
|
||||
// ConfigPersistence handles saving and loading configuration files
|
||||
type ConfigPersistence struct {
|
||||
dataDir string
|
||||
// tasksMu serializes all filesystem operations on the tasks/ directory.
|
||||
// SaveTaskState, LoadTaskState, LoadAllTaskStates, DeleteTaskState, and
|
||||
// CleanupCompletedTasks are called from multiple goroutines concurrently
|
||||
// after saveTaskState was moved outside mq.mutex in the maintenance queue.
|
||||
tasksMu sync.Mutex
|
||||
}
|
||||
|
||||
// NewConfigPersistence creates a new configuration persistence manager
|
||||
@@ -937,6 +943,8 @@ func (cp *ConfigPersistence) ListTaskDetails() ([]string, error) {
|
||||
|
||||
// CleanupCompletedTasks removes old completed tasks beyond the retention limit
|
||||
func (cp *ConfigPersistence) CleanupCompletedTasks() error {
|
||||
cp.tasksMu.Lock()
|
||||
defer cp.tasksMu.Unlock()
|
||||
if cp.dataDir == "" {
|
||||
return fmt.Errorf("no data directory specified, cannot cleanup completed tasks")
|
||||
}
|
||||
@@ -946,8 +954,8 @@ func (cp *ConfigPersistence) CleanupCompletedTasks() error {
|
||||
return nil // No tasks directory, nothing to cleanup
|
||||
}
|
||||
|
||||
// Load all tasks and find completed/failed ones
|
||||
allTasks, err := cp.LoadAllTaskStates()
|
||||
// Use unlocked helpers to avoid deadlock (tasksMu is already held)
|
||||
allTasks, err := cp.loadAllTaskStatesLocked()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load tasks for cleanup: %w", err)
|
||||
}
|
||||
@@ -998,7 +1006,7 @@ func (cp *ConfigPersistence) CleanupCompletedTasks() error {
|
||||
if len(completedTasks) > MaxCompletedTasks {
|
||||
tasksToDelete := completedTasks[MaxCompletedTasks:]
|
||||
for _, task := range tasksToDelete {
|
||||
if err := cp.DeleteTaskState(task.ID); err != nil {
|
||||
if err := cp.deleteTaskStateLocked(task.ID); err != nil {
|
||||
glog.Warningf("Failed to delete old completed task %s: %v", task.ID, err)
|
||||
} else {
|
||||
glog.V(2).Infof("Cleaned up old completed task %s (completed: %v)", task.ID, task.CompletedAt)
|
||||
@@ -1012,6 +1020,8 @@ func (cp *ConfigPersistence) CleanupCompletedTasks() error {
|
||||
|
||||
// SaveTaskState saves a task state to protobuf file
|
||||
func (cp *ConfigPersistence) SaveTaskState(task *maintenance.MaintenanceTask) error {
|
||||
cp.tasksMu.Lock()
|
||||
defer cp.tasksMu.Unlock()
|
||||
if cp.dataDir == "" {
|
||||
return fmt.Errorf("no data directory specified, cannot save task state")
|
||||
}
|
||||
@@ -1051,6 +1061,13 @@ func (cp *ConfigPersistence) SaveTaskState(task *maintenance.MaintenanceTask) er
|
||||
|
||||
// LoadTaskState loads a task state from protobuf file
|
||||
func (cp *ConfigPersistence) LoadTaskState(taskID string) (*maintenance.MaintenanceTask, error) {
|
||||
cp.tasksMu.Lock()
|
||||
defer cp.tasksMu.Unlock()
|
||||
return cp.loadTaskStateLocked(taskID)
|
||||
}
|
||||
|
||||
// loadTaskStateLocked loads a single task state. Must be called with tasksMu held.
|
||||
func (cp *ConfigPersistence) loadTaskStateLocked(taskID string) (*maintenance.MaintenanceTask, error) {
|
||||
if cp.dataDir == "" {
|
||||
return nil, fmt.Errorf("no data directory specified, cannot load task state")
|
||||
}
|
||||
@@ -1084,6 +1101,13 @@ func (cp *ConfigPersistence) LoadTaskState(taskID string) (*maintenance.Maintena
|
||||
|
||||
// LoadAllTaskStates loads all task states from disk
|
||||
func (cp *ConfigPersistence) LoadAllTaskStates() ([]*maintenance.MaintenanceTask, error) {
|
||||
cp.tasksMu.Lock()
|
||||
defer cp.tasksMu.Unlock()
|
||||
return cp.loadAllTaskStatesLocked()
|
||||
}
|
||||
|
||||
// loadAllTaskStatesLocked loads all task states from disk. Must be called with tasksMu held.
|
||||
func (cp *ConfigPersistence) loadAllTaskStatesLocked() ([]*maintenance.MaintenanceTask, error) {
|
||||
if cp.dataDir == "" {
|
||||
return []*maintenance.MaintenanceTask{}, nil
|
||||
}
|
||||
@@ -1102,7 +1126,7 @@ func (cp *ConfigPersistence) LoadAllTaskStates() ([]*maintenance.MaintenanceTask
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() && filepath.Ext(entry.Name()) == ".pb" {
|
||||
taskID := entry.Name()[:len(entry.Name())-3] // Remove .pb extension
|
||||
task, err := cp.LoadTaskState(taskID)
|
||||
task, err := cp.loadTaskStateLocked(taskID)
|
||||
if err != nil {
|
||||
glog.Warningf("Failed to load task state for %s: %v", taskID, err)
|
||||
continue
|
||||
@@ -1117,6 +1141,13 @@ func (cp *ConfigPersistence) LoadAllTaskStates() ([]*maintenance.MaintenanceTask
|
||||
|
||||
// DeleteTaskState removes a task state file from disk
|
||||
func (cp *ConfigPersistence) DeleteTaskState(taskID string) error {
|
||||
cp.tasksMu.Lock()
|
||||
defer cp.tasksMu.Unlock()
|
||||
return cp.deleteTaskStateLocked(taskID)
|
||||
}
|
||||
|
||||
// deleteTaskStateLocked removes a task state file. Must be called with tasksMu held.
|
||||
func (cp *ConfigPersistence) deleteTaskStateLocked(taskID string) error {
|
||||
if cp.dataDir == "" {
|
||||
return fmt.Errorf("no data directory specified, cannot delete task state")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package dash
|
||||
|
||||
import "context"
|
||||
|
||||
type contextKey string
|
||||
|
||||
const (
|
||||
contextUsernameKey contextKey = "admin.username"
|
||||
contextRoleKey contextKey = "admin.role"
|
||||
contextCSRFKey contextKey = "admin.csrf"
|
||||
)
|
||||
|
||||
// WithAuthContext stores auth metadata on the request context.
|
||||
func WithAuthContext(ctx context.Context, username, role, csrfToken string) context.Context {
|
||||
if username != "" {
|
||||
ctx = context.WithValue(ctx, contextUsernameKey, username)
|
||||
}
|
||||
if role != "" {
|
||||
ctx = context.WithValue(ctx, contextRoleKey, role)
|
||||
}
|
||||
if csrfToken != "" {
|
||||
ctx = context.WithValue(ctx, contextCSRFKey, csrfToken)
|
||||
}
|
||||
return ctx
|
||||
}
|
||||
|
||||
// UsernameFromContext retrieves the username from context.
|
||||
func UsernameFromContext(ctx context.Context) string {
|
||||
if ctx == nil {
|
||||
return ""
|
||||
}
|
||||
if value, ok := ctx.Value(contextUsernameKey).(string); ok {
|
||||
return value
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// RoleFromContext retrieves the role from context.
|
||||
func RoleFromContext(ctx context.Context) string {
|
||||
if ctx == nil {
|
||||
return ""
|
||||
}
|
||||
if value, ok := ctx.Value(contextRoleKey).(string); ok {
|
||||
return value
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// CSRFTokenFromContext retrieves the CSRF token from context.
|
||||
func CSRFTokenFromContext(ctx context.Context) string {
|
||||
if ctx == nil {
|
||||
return ""
|
||||
}
|
||||
if value, ok := ctx.Value(contextCSRFKey).(string); ok {
|
||||
return value
|
||||
}
|
||||
return ""
|
||||
}
|
||||
+55
-19
@@ -4,10 +4,10 @@ import (
|
||||
"crypto/rand"
|
||||
"crypto/subtle"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-contrib/sessions"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/gorilla/sessions"
|
||||
)
|
||||
|
||||
const sessionCSRFTokenKey = "csrf_token"
|
||||
@@ -20,41 +20,77 @@ func generateCSRFToken() (string, error) {
|
||||
return hex.EncodeToString(tokenBytes), nil
|
||||
}
|
||||
|
||||
func getOrCreateSessionCSRFToken(session sessions.Session) (string, error) {
|
||||
if existing, ok := session.Get(sessionCSRFTokenKey).(string); ok && existing != "" {
|
||||
func getOrCreateSessionCSRFToken(session *sessions.Session, r *http.Request, w http.ResponseWriter) (string, error) {
|
||||
if existing, ok := session.Values[sessionCSRFTokenKey].(string); ok && existing != "" {
|
||||
return existing, nil
|
||||
}
|
||||
token, err := generateCSRFToken()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
session.Set(sessionCSRFTokenKey, token)
|
||||
if err := session.Save(); err != nil {
|
||||
session.Values[sessionCSRFTokenKey] = token
|
||||
if err := session.Save(r, w); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func requireSessionCSRFToken(c *gin.Context) bool {
|
||||
session := sessions.Default(c)
|
||||
if session.Get("authenticated") != true {
|
||||
func requireSessionCSRFToken(w http.ResponseWriter, r *http.Request) bool {
|
||||
expectedToken := CSRFTokenFromContext(r.Context())
|
||||
username := UsernameFromContext(r.Context())
|
||||
if expectedToken == "" {
|
||||
// Admin UI can run without auth; in that mode CSRF token checks are not applicable.
|
||||
return true
|
||||
}
|
||||
|
||||
expectedToken, ok := session.Get(sessionCSRFTokenKey).(string)
|
||||
if !ok || expectedToken == "" {
|
||||
c.JSON(http.StatusForbidden, gin.H{"error": "missing CSRF session token"})
|
||||
if username == "" {
|
||||
return true
|
||||
}
|
||||
writeJSONError(w, http.StatusForbidden, "missing CSRF session token")
|
||||
return false
|
||||
}
|
||||
|
||||
providedToken := c.GetHeader("X-CSRF-Token")
|
||||
if providedToken == "" {
|
||||
providedToken = c.PostForm("csrf_token")
|
||||
providedToken, err := getProvidedCSRFToken(r)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Failed to parse form: "+err.Error())
|
||||
return false
|
||||
}
|
||||
if providedToken == "" || subtle.ConstantTimeCompare([]byte(expectedToken), []byte(providedToken)) != 1 {
|
||||
c.JSON(http.StatusForbidden, gin.H{"error": "invalid CSRF token"})
|
||||
writeJSONError(w, http.StatusForbidden, "invalid CSRF token")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func getProvidedCSRFToken(r *http.Request) (string, error) {
|
||||
providedToken := r.Header.Get("X-CSRF-Token")
|
||||
if providedToken != "" {
|
||||
return providedToken, nil
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return r.FormValue("csrf_token"), nil
|
||||
}
|
||||
|
||||
func EnsureSessionCSRFToken(session *sessions.Session, r *http.Request, w http.ResponseWriter) (string, error) {
|
||||
if session == nil {
|
||||
return "", fmt.Errorf("session is nil")
|
||||
}
|
||||
return getOrCreateSessionCSRFToken(session, r, w)
|
||||
}
|
||||
|
||||
func ValidateSessionCSRFToken(session *sessions.Session, r *http.Request) error {
|
||||
if session == nil {
|
||||
return fmt.Errorf("session is nil")
|
||||
}
|
||||
expectedToken, _ := session.Values[sessionCSRFTokenKey].(string)
|
||||
providedToken, err := getProvidedCSRFToken(r)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read CSRF token: %w", err)
|
||||
}
|
||||
if expectedToken == "" {
|
||||
return fmt.Errorf("missing session CSRF token")
|
||||
}
|
||||
if providedToken == "" || subtle.ConstantTimeCompare([]byte(expectedToken), []byte(providedToken)) != 1 {
|
||||
return fmt.Errorf("invalid CSRF token")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -6,10 +6,7 @@ import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
)
|
||||
|
||||
@@ -333,10 +330,19 @@ func (s *AdminServer) GetClusterEcVolumes(page int, pageSize int, sortBy string,
|
||||
|
||||
// Process each shard this server has for this volume
|
||||
shardBits := ecShardInfo.EcIndexBits
|
||||
shardSizes := ecShardInfo.ShardSizes
|
||||
sizeIndex := 0
|
||||
for shardId := 0; shardId < erasure_coding.MaxShardCount; shardId++ {
|
||||
if (shardBits & (1 << uint(shardId))) != 0 {
|
||||
// Record shard location
|
||||
volume.ShardLocations[shardId] = node.Id
|
||||
if sizeIndex < len(shardSizes) {
|
||||
size := shardSizes[sizeIndex]
|
||||
if size >= 0 {
|
||||
volume.ShardSizes[shardId] = size
|
||||
}
|
||||
}
|
||||
sizeIndex++
|
||||
totalShards++
|
||||
}
|
||||
}
|
||||
@@ -354,38 +360,6 @@ func (s *AdminServer) GetClusterEcVolumes(page int, pageSize int, sortBy string,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Collect shard size information from volume servers
|
||||
for volumeId, volume := range volumeData {
|
||||
// Group servers by volume to minimize gRPC calls
|
||||
serverHasVolume := make(map[string]bool)
|
||||
for _, server := range volume.Servers {
|
||||
serverHasVolume[server] = true
|
||||
}
|
||||
|
||||
// Query each server for shard sizes
|
||||
for server := range serverHasVolume {
|
||||
err := s.WithVolumeServerClient(pb.ServerAddress(server), func(client volume_server_pb.VolumeServerClient) error {
|
||||
resp, err := client.VolumeEcShardsInfo(context.Background(), &volume_server_pb.VolumeEcShardsInfoRequest{
|
||||
VolumeId: volumeId,
|
||||
})
|
||||
if err != nil {
|
||||
glog.V(1).Infof("Failed to get EC shard info from %s for volume %d: %v", server, volumeId, err)
|
||||
return nil // Continue with other servers, don't fail the entire request
|
||||
}
|
||||
|
||||
// Update shard sizes
|
||||
for _, shardInfo := range resp.EcShardInfos {
|
||||
volume.ShardSizes[int(shardInfo.ShardId)] = shardInfo.Size
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
glog.V(1).Infof("Failed to connect to volume server %s: %v", server, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate completeness for each volume
|
||||
completeVolumes := 0
|
||||
incompleteVolumes := 0
|
||||
@@ -616,13 +590,23 @@ func (s *AdminServer) GetEcVolumeDetails(volumeID uint32, sortBy string, sortOrd
|
||||
|
||||
// Create individual shard entries for each shard this server has
|
||||
shardBits := ecShardInfo.EcIndexBits
|
||||
shardSizes := ecShardInfo.ShardSizes
|
||||
sizeIndex := 0
|
||||
for shardId := 0; shardId < erasure_coding.MaxShardCount; shardId++ {
|
||||
if (shardBits & (1 << uint(shardId))) != 0 {
|
||||
var shardSize uint64
|
||||
if sizeIndex < len(shardSizes) {
|
||||
size := shardSizes[sizeIndex]
|
||||
if size >= 0 {
|
||||
shardSize = uint64(size)
|
||||
}
|
||||
}
|
||||
sizeIndex++
|
||||
ecShard := EcShardWithInfo{
|
||||
VolumeID: ecShardInfo.Id,
|
||||
ShardID: uint32(shardId),
|
||||
Collection: ecShardInfo.Collection,
|
||||
Size: 0, // EC shards don't have individual size in the API response
|
||||
Size: shardSize,
|
||||
Server: node.Id,
|
||||
DataCenter: dc.Id,
|
||||
Rack: rack.Id,
|
||||
@@ -653,46 +637,6 @@ func (s *AdminServer) GetEcVolumeDetails(volumeID uint32, sortBy string, sortOrd
|
||||
return nil, fmt.Errorf("EC volume %d not found", volumeID)
|
||||
}
|
||||
|
||||
// Collect shard size information from volume servers
|
||||
shardSizeMap := make(map[string]map[uint32]uint64) // server -> shardId -> size
|
||||
for _, shard := range shards {
|
||||
server := shard.Server
|
||||
if _, exists := shardSizeMap[server]; !exists {
|
||||
// Query this server for shard sizes
|
||||
err := s.WithVolumeServerClient(pb.ServerAddress(server), func(client volume_server_pb.VolumeServerClient) error {
|
||||
resp, err := client.VolumeEcShardsInfo(context.Background(), &volume_server_pb.VolumeEcShardsInfoRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
glog.V(1).Infof("Failed to get EC shard info from %s for volume %d: %v", server, volumeID, err)
|
||||
return nil // Continue with other servers, don't fail the entire request
|
||||
}
|
||||
|
||||
// Store shard sizes for this server
|
||||
shardSizeMap[server] = make(map[uint32]uint64)
|
||||
for _, shardInfo := range resp.EcShardInfos {
|
||||
shardSizeMap[server][shardInfo.ShardId] = uint64(shardInfo.Size)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
glog.V(1).Infof("Failed to connect to volume server %s: %v", server, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update shard sizes in the shards array
|
||||
for i := range shards {
|
||||
server := shards[i].Server
|
||||
shardId := shards[i].ShardID
|
||||
if serverSizes, exists := shardSizeMap[server]; exists {
|
||||
if size, exists := serverSizes[shardId]; exists {
|
||||
shards[i].Size = size
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate completeness based on unique shard IDs
|
||||
foundShards := make(map[int]bool)
|
||||
for _, shard := range shards {
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
package dash
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/internal/httputil"
|
||||
)
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, payload interface{}) {
|
||||
httputil.WriteJSON(w, status, payload)
|
||||
}
|
||||
|
||||
func writeJSONError(w http.ResponseWriter, status int, message string) {
|
||||
httputil.WriteJSONError(w, status, message)
|
||||
}
|
||||
|
||||
func decodeJSONBody(r io.Reader, v interface{}) error {
|
||||
return httputil.DecodeJSONBody(r, v)
|
||||
}
|
||||
|
||||
func newJSONMaxReader(w http.ResponseWriter, r *http.Request) io.Reader {
|
||||
return httputil.NewJSONMaxReader(w, r)
|
||||
}
|
||||
+110
-90
@@ -4,109 +4,129 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/gin-contrib/sessions"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/gorilla/sessions"
|
||||
)
|
||||
|
||||
// setAuthContext sets username and role in context for use in handlers
|
||||
func setAuthContext(c *gin.Context, username, role interface{}) {
|
||||
c.Set("username", username)
|
||||
if role != nil {
|
||||
c.Set("role", role)
|
||||
} else {
|
||||
// Default to admin for backward compatibility
|
||||
c.Set("role", "admin")
|
||||
}
|
||||
const sessionName = "admin-session"
|
||||
|
||||
// SessionName returns the cookie session name used by the admin UI.
|
||||
func SessionName() string {
|
||||
return sessionName
|
||||
}
|
||||
|
||||
// RequireAuth checks if user is authenticated
|
||||
func RequireAuth() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
session := sessions.Default(c)
|
||||
authenticated := session.Get("authenticated")
|
||||
username := session.Get("username")
|
||||
role := session.Get("role")
|
||||
type sessionValidationErrorKind int
|
||||
|
||||
if authenticated != true || username == nil {
|
||||
c.Redirect(http.StatusTemporaryRedirect, "/login")
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
const (
|
||||
sessionValidationErrorKindUnauthenticated sessionValidationErrorKind = iota
|
||||
sessionValidationErrorKindSessionInit
|
||||
)
|
||||
|
||||
csrfToken, err := getOrCreateSessionCSRFToken(session)
|
||||
if err != nil {
|
||||
c.Redirect(http.StatusTemporaryRedirect, "/login?error=Unable to initialize session")
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
|
||||
// Set username and role in context for use in handlers
|
||||
setAuthContext(c, username, role)
|
||||
c.Set("csrf_token", csrfToken)
|
||||
c.Next()
|
||||
}
|
||||
type sessionValidationError struct {
|
||||
kind sessionValidationErrorKind
|
||||
err error
|
||||
}
|
||||
|
||||
// RequireAuthAPI checks if user is authenticated for API endpoints
|
||||
// Returns JSON error instead of redirecting to login page
|
||||
func RequireAuthAPI() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
session := sessions.Default(c)
|
||||
authenticated := session.Get("authenticated")
|
||||
username := session.Get("username")
|
||||
role := session.Get("role")
|
||||
|
||||
if authenticated != true || username == nil {
|
||||
c.JSON(http.StatusUnauthorized, gin.H{
|
||||
"error": "Authentication required",
|
||||
"message": "Please log in to access this endpoint",
|
||||
})
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
|
||||
csrfToken, err := getOrCreateSessionCSRFToken(session)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{
|
||||
"error": "Failed to initialize session",
|
||||
"message": "Unable to initialize CSRF token",
|
||||
})
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
|
||||
// Set username and role in context for use in handlers
|
||||
setAuthContext(c, username, role)
|
||||
c.Set("csrf_token", csrfToken)
|
||||
c.Next()
|
||||
func (e *sessionValidationError) Error() string {
|
||||
if e.err != nil {
|
||||
return e.err.Error()
|
||||
}
|
||||
return "session validation failed"
|
||||
}
|
||||
|
||||
// RequireWriteAccess checks if user has admin role (write access)
|
||||
// Returns JSON error for API endpoints, redirects for HTML endpoints
|
||||
func RequireWriteAccess() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
role, exists := c.Get("role")
|
||||
if !exists {
|
||||
role = "admin" // Default for backward compatibility
|
||||
}
|
||||
func (e *sessionValidationError) Unwrap() error {
|
||||
return e.err
|
||||
}
|
||||
|
||||
roleStr, ok := role.(string)
|
||||
if !ok || roleStr != "admin" {
|
||||
// Check if this is an API request (path starts with /api) or HTML request
|
||||
path := c.Request.URL.Path
|
||||
if strings.HasPrefix(path, "/api") {
|
||||
c.JSON(http.StatusForbidden, gin.H{
|
||||
"error": "Insufficient permissions",
|
||||
"message": "This operation requires admin access. Read-only users can only view data.",
|
||||
})
|
||||
} else {
|
||||
c.Redirect(http.StatusSeeOther, "/admin?error=Insufficient permissions")
|
||||
func validateSession(store sessions.Store, w http.ResponseWriter, r *http.Request) (string, string, string, error) {
|
||||
session, err := store.Get(r, sessionName)
|
||||
if err != nil {
|
||||
return "", "", "", &sessionValidationError{kind: sessionValidationErrorKindSessionInit, err: err}
|
||||
}
|
||||
|
||||
authenticated, _ := session.Values["authenticated"].(bool)
|
||||
username, _ := session.Values["username"].(string)
|
||||
role, _ := session.Values["role"].(string)
|
||||
if !authenticated || username == "" {
|
||||
return "", "", "", &sessionValidationError{kind: sessionValidationErrorKindUnauthenticated}
|
||||
}
|
||||
|
||||
csrfToken, err := getOrCreateSessionCSRFToken(session, r, w)
|
||||
if err != nil {
|
||||
return "", "", "", &sessionValidationError{kind: sessionValidationErrorKindSessionInit, err: err}
|
||||
}
|
||||
|
||||
return username, role, csrfToken, nil
|
||||
}
|
||||
|
||||
// RequireAuth checks if user is authenticated.
|
||||
func RequireAuth(store sessions.Store) mux.MiddlewareFunc {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
username, role, csrfToken, err := validateSession(store, w, r)
|
||||
if err != nil {
|
||||
if verr, ok := err.(*sessionValidationError); ok && verr.kind == sessionValidationErrorKindUnauthenticated {
|
||||
http.Redirect(w, r, "/login", http.StatusTemporaryRedirect)
|
||||
} else {
|
||||
http.Redirect(w, r, "/login?error=Unable to initialize session", http.StatusTemporaryRedirect)
|
||||
}
|
||||
return
|
||||
}
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
|
||||
c.Next()
|
||||
ctx := WithAuthContext(r.Context(), username, role, csrfToken)
|
||||
next.ServeHTTP(w, r.WithContext(ctx))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// RequireAuthAPI checks if user is authenticated for API endpoints.
|
||||
// Returns JSON error instead of redirecting to login page.
|
||||
func RequireAuthAPI(store sessions.Store) mux.MiddlewareFunc {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
username, role, csrfToken, err := validateSession(store, w, r)
|
||||
if err != nil {
|
||||
if verr, ok := err.(*sessionValidationError); ok && verr.kind == sessionValidationErrorKindUnauthenticated {
|
||||
writeJSON(w, http.StatusUnauthorized, map[string]string{
|
||||
"error": "Authentication required",
|
||||
"message": "Please log in to access this endpoint",
|
||||
})
|
||||
} else {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{
|
||||
"error": "Failed to initialize session",
|
||||
"message": "Unable to initialize session",
|
||||
})
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
ctx := WithAuthContext(r.Context(), username, role, csrfToken)
|
||||
next.ServeHTTP(w, r.WithContext(ctx))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// RequireWriteAccess checks if user has admin role (write access).
|
||||
// Returns JSON error for API endpoints, redirects for HTML endpoints.
|
||||
func RequireWriteAccess() mux.MiddlewareFunc {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
role := RoleFromContext(r.Context())
|
||||
|
||||
if role != "admin" {
|
||||
// Check if this is an API request (path starts with /api) or HTML request.
|
||||
if strings.HasPrefix(r.URL.Path, "/api") {
|
||||
writeJSON(w, http.StatusForbidden, map[string]string{
|
||||
"error": "Insufficient permissions",
|
||||
"message": "This operation requires admin access. Read-only users can only view data.",
|
||||
})
|
||||
} else {
|
||||
http.Redirect(w, r, "/admin?error=Insufficient permissions", http.StatusSeeOther)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,740 @@
|
||||
package dash
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/plugin"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
"google.golang.org/protobuf/encoding/protojson"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultPluginDetectionTimeout = 45 * time.Second
|
||||
defaultPluginExecutionTimeout = 90 * time.Second
|
||||
maxPluginDetectionTimeout = 5 * time.Minute
|
||||
maxPluginExecutionTimeout = 10 * time.Minute
|
||||
defaultPluginRunTimeout = 5 * time.Minute
|
||||
maxPluginRunTimeout = 30 * time.Minute
|
||||
)
|
||||
|
||||
// GetPluginStatusAPI returns plugin status.
|
||||
func (s *AdminServer) GetPluginStatusAPI(w http.ResponseWriter, r *http.Request) {
|
||||
plugin := s.GetPlugin()
|
||||
if plugin == nil {
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"enabled": false,
|
||||
"worker_grpc_port": s.GetWorkerGrpcPort(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"enabled": true,
|
||||
"configured": plugin.IsConfigured(),
|
||||
"base_dir": plugin.BaseDir(),
|
||||
"worker_count": len(plugin.ListWorkers()),
|
||||
"worker_grpc_port": s.GetWorkerGrpcPort(),
|
||||
})
|
||||
}
|
||||
|
||||
// GetPluginWorkersAPI returns currently connected plugin workers.
|
||||
func (s *AdminServer) GetPluginWorkersAPI(w http.ResponseWriter, r *http.Request) {
|
||||
workers := s.GetPluginWorkers()
|
||||
if workers == nil {
|
||||
writeJSON(w, http.StatusOK, []interface{}{})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, workers)
|
||||
}
|
||||
|
||||
// GetPluginJobTypesAPI returns known plugin job types from workers and persisted data.
|
||||
func (s *AdminServer) GetPluginJobTypesAPI(w http.ResponseWriter, r *http.Request) {
|
||||
jobTypes, err := s.ListPluginJobTypes()
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if jobTypes == nil {
|
||||
writeJSON(w, http.StatusOK, []interface{}{})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, jobTypes)
|
||||
}
|
||||
|
||||
// GetPluginJobsAPI returns tracked jobs for monitoring.
|
||||
func (s *AdminServer) GetPluginJobsAPI(w http.ResponseWriter, r *http.Request) {
|
||||
query := r.URL.Query()
|
||||
jobType := strings.TrimSpace(query.Get("job_type"))
|
||||
state := strings.TrimSpace(query.Get("state"))
|
||||
limit := parsePositiveInt(query.Get("limit"), 200)
|
||||
jobs := s.ListPluginJobs(jobType, state, limit)
|
||||
if jobs == nil {
|
||||
writeJSON(w, http.StatusOK, []interface{}{})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, jobs)
|
||||
}
|
||||
|
||||
// GetPluginJobAPI returns one tracked job.
|
||||
func (s *AdminServer) GetPluginJobAPI(w http.ResponseWriter, r *http.Request) {
|
||||
jobID := strings.TrimSpace(mux.Vars(r)["jobId"])
|
||||
if jobID == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "jobId is required")
|
||||
return
|
||||
}
|
||||
|
||||
job, found := s.GetPluginJob(jobID)
|
||||
if !found {
|
||||
writeJSONError(w, http.StatusNotFound, "job not found")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, job)
|
||||
}
|
||||
|
||||
// GetPluginJobDetailAPI returns detailed information for one tracked plugin job.
|
||||
func (s *AdminServer) GetPluginJobDetailAPI(w http.ResponseWriter, r *http.Request) {
|
||||
jobID := strings.TrimSpace(mux.Vars(r)["jobId"])
|
||||
if jobID == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "jobId is required")
|
||||
return
|
||||
}
|
||||
|
||||
query := r.URL.Query()
|
||||
activityLimit := parsePositiveInt(query.Get("activity_limit"), 500)
|
||||
relatedLimit := parsePositiveInt(query.Get("related_limit"), 20)
|
||||
|
||||
detail, found, err := s.GetPluginJobDetail(jobID, activityLimit, relatedLimit)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if !found || detail == nil {
|
||||
writeJSONError(w, http.StatusNotFound, "job detail not found")
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, detail)
|
||||
}
|
||||
|
||||
// GetPluginActivitiesAPI returns recent plugin activities.
|
||||
func (s *AdminServer) GetPluginActivitiesAPI(w http.ResponseWriter, r *http.Request) {
|
||||
query := r.URL.Query()
|
||||
jobType := strings.TrimSpace(query.Get("job_type"))
|
||||
limit := parsePositiveInt(query.Get("limit"), 500)
|
||||
activities := s.ListPluginActivities(jobType, limit)
|
||||
if activities == nil {
|
||||
writeJSON(w, http.StatusOK, []interface{}{})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, activities)
|
||||
}
|
||||
|
||||
// GetPluginSchedulerStatesAPI returns per-job-type scheduler status for monitoring.
|
||||
func (s *AdminServer) GetPluginSchedulerStatesAPI(w http.ResponseWriter, r *http.Request) {
|
||||
jobTypeFilter := strings.TrimSpace(r.URL.Query().Get("job_type"))
|
||||
|
||||
states, err := s.ListPluginSchedulerStates()
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if jobTypeFilter != "" {
|
||||
filtered := make([]interface{}, 0, len(states))
|
||||
for _, state := range states {
|
||||
if state.JobType == jobTypeFilter {
|
||||
filtered = append(filtered, state)
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, filtered)
|
||||
return
|
||||
}
|
||||
|
||||
if states == nil {
|
||||
writeJSON(w, http.StatusOK, []interface{}{})
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, states)
|
||||
}
|
||||
|
||||
// RequestPluginJobTypeSchemaAPI asks a worker for one job type schema.
|
||||
func (s *AdminServer) RequestPluginJobTypeSchemaAPI(w http.ResponseWriter, r *http.Request) {
|
||||
jobType := strings.TrimSpace(mux.Vars(r)["jobType"])
|
||||
if jobType == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "jobType is required")
|
||||
return
|
||||
}
|
||||
|
||||
forceRefresh := strings.EqualFold(r.URL.Query().Get("force_refresh"), "true")
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), defaultPluginDetectionTimeout)
|
||||
defer cancel()
|
||||
descriptor, err := s.RequestPluginJobTypeDescriptor(ctx, jobType, forceRefresh)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
renderProtoJSON(w, http.StatusOK, descriptor)
|
||||
}
|
||||
|
||||
// GetPluginJobTypeDescriptorAPI returns persisted descriptor for a job type.
|
||||
func (s *AdminServer) GetPluginJobTypeDescriptorAPI(w http.ResponseWriter, r *http.Request) {
|
||||
jobType := strings.TrimSpace(mux.Vars(r)["jobType"])
|
||||
if jobType == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "jobType is required")
|
||||
return
|
||||
}
|
||||
|
||||
descriptor, err := s.LoadPluginJobTypeDescriptor(jobType)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if descriptor == nil {
|
||||
writeJSONError(w, http.StatusNotFound, "descriptor not found")
|
||||
return
|
||||
}
|
||||
|
||||
renderProtoJSON(w, http.StatusOK, descriptor)
|
||||
}
|
||||
|
||||
// GetPluginJobTypeConfigAPI loads persisted config for a job type.
|
||||
func (s *AdminServer) GetPluginJobTypeConfigAPI(w http.ResponseWriter, r *http.Request) {
|
||||
jobType := strings.TrimSpace(mux.Vars(r)["jobType"])
|
||||
if jobType == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "jobType is required")
|
||||
return
|
||||
}
|
||||
|
||||
config, err := s.LoadPluginJobTypeConfig(jobType)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if config == nil {
|
||||
config = &plugin_pb.PersistedJobTypeConfig{
|
||||
JobType: jobType,
|
||||
AdminConfigValues: map[string]*plugin_pb.ConfigValue{},
|
||||
WorkerConfigValues: map[string]*plugin_pb.ConfigValue{},
|
||||
AdminRuntime: &plugin_pb.AdminRuntimeConfig{},
|
||||
}
|
||||
}
|
||||
|
||||
renderProtoJSON(w, http.StatusOK, config)
|
||||
}
|
||||
|
||||
// UpdatePluginJobTypeConfigAPI stores persisted config for a job type.
|
||||
func (s *AdminServer) UpdatePluginJobTypeConfigAPI(w http.ResponseWriter, r *http.Request) {
|
||||
jobType := strings.TrimSpace(mux.Vars(r)["jobType"])
|
||||
if jobType == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "jobType is required")
|
||||
return
|
||||
}
|
||||
|
||||
config := &plugin_pb.PersistedJobTypeConfig{}
|
||||
if err := parseProtoJSONBody(w, r, config); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
config.JobType = jobType
|
||||
if config.UpdatedAt == nil {
|
||||
config.UpdatedAt = timestamppb.Now()
|
||||
}
|
||||
if config.AdminRuntime == nil {
|
||||
config.AdminRuntime = &plugin_pb.AdminRuntimeConfig{}
|
||||
}
|
||||
if config.AdminConfigValues == nil {
|
||||
config.AdminConfigValues = map[string]*plugin_pb.ConfigValue{}
|
||||
}
|
||||
if config.WorkerConfigValues == nil {
|
||||
config.WorkerConfigValues = map[string]*plugin_pb.ConfigValue{}
|
||||
}
|
||||
|
||||
username := UsernameFromContext(r.Context())
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
config.UpdatedBy = username
|
||||
|
||||
if err := s.SavePluginJobTypeConfig(config); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
renderProtoJSON(w, http.StatusOK, config)
|
||||
}
|
||||
|
||||
// GetPluginRunHistoryAPI returns bounded run history for a job type.
|
||||
func (s *AdminServer) GetPluginRunHistoryAPI(w http.ResponseWriter, r *http.Request) {
|
||||
jobType := strings.TrimSpace(mux.Vars(r)["jobType"])
|
||||
if jobType == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "jobType is required")
|
||||
return
|
||||
}
|
||||
|
||||
history, err := s.GetPluginRunHistory(jobType)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if history == nil {
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"job_type": jobType,
|
||||
"successful_runs": []interface{}{},
|
||||
"error_runs": []interface{}{},
|
||||
"last_updated_time": nil,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, history)
|
||||
}
|
||||
|
||||
// TriggerPluginDetectionAPI runs one detector for this job type and returns proposals.
|
||||
func (s *AdminServer) TriggerPluginDetectionAPI(w http.ResponseWriter, r *http.Request) {
|
||||
jobType := strings.TrimSpace(mux.Vars(r)["jobType"])
|
||||
if jobType == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "jobType is required")
|
||||
return
|
||||
}
|
||||
|
||||
var req struct {
|
||||
ClusterContext json.RawMessage `json:"cluster_context"`
|
||||
MaxResults int32 `json:"max_results"`
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
}
|
||||
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil && err != io.EOF {
|
||||
writeJSONError(w, http.StatusBadRequest, "invalid request body: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
clusterContext, err := s.parseOrBuildClusterContext(req.ClusterContext)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
timeout := normalizeTimeout(req.TimeoutSeconds, defaultPluginDetectionTimeout, maxPluginDetectionTimeout)
|
||||
ctx, cancel := context.WithTimeout(r.Context(), timeout)
|
||||
defer cancel()
|
||||
|
||||
report, err := s.RunPluginDetectionWithReport(ctx, jobType, clusterContext, req.MaxResults)
|
||||
proposals := make([]*plugin_pb.JobProposal, 0)
|
||||
requestID := ""
|
||||
detectorWorkerID := ""
|
||||
totalProposals := int32(0)
|
||||
if report != nil {
|
||||
proposals = report.Proposals
|
||||
requestID = report.RequestID
|
||||
detectorWorkerID = report.WorkerID
|
||||
if report.Complete != nil {
|
||||
totalProposals = report.Complete.TotalProposals
|
||||
}
|
||||
}
|
||||
|
||||
proposalPayloads := make([]map[string]interface{}, 0, len(proposals))
|
||||
for _, proposal := range proposals {
|
||||
payload, marshalErr := protoMessageToMap(proposal)
|
||||
if marshalErr != nil {
|
||||
glog.Warningf("failed to marshal proposal for jobType=%s: %v", jobType, marshalErr)
|
||||
continue
|
||||
}
|
||||
proposalPayloads = append(proposalPayloads, payload)
|
||||
}
|
||||
|
||||
sort.Slice(proposalPayloads, func(i, j int) bool {
|
||||
iPriorityStr, _ := proposalPayloads[i]["priority"].(string)
|
||||
jPriorityStr, _ := proposalPayloads[j]["priority"].(string)
|
||||
|
||||
iPriority := plugin_pb.JobPriority_value[iPriorityStr]
|
||||
jPriority := plugin_pb.JobPriority_value[jPriorityStr]
|
||||
|
||||
if iPriority != jPriority {
|
||||
return iPriority > jPriority
|
||||
}
|
||||
iID, _ := proposalPayloads[i]["proposal_id"].(string)
|
||||
jID, _ := proposalPayloads[j]["proposal_id"].(string)
|
||||
return iID < jID
|
||||
})
|
||||
|
||||
activities := s.ListPluginActivities(jobType, 500)
|
||||
filteredActivities := make([]interface{}, 0, len(activities))
|
||||
if requestID != "" {
|
||||
for i := len(activities) - 1; i >= 0; i-- {
|
||||
activity := activities[i]
|
||||
if activity.RequestID != requestID {
|
||||
continue
|
||||
}
|
||||
filteredActivities = append(filteredActivities, activity)
|
||||
}
|
||||
}
|
||||
|
||||
response := map[string]interface{}{
|
||||
"job_type": jobType,
|
||||
"request_id": requestID,
|
||||
"detector_worker_id": detectorWorkerID,
|
||||
"total_proposals": totalProposals,
|
||||
"count": len(proposalPayloads),
|
||||
"proposals": proposalPayloads,
|
||||
"activities": filteredActivities,
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
response["error"] = err.Error()
|
||||
writeJSON(w, http.StatusInternalServerError, response)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, response)
|
||||
}
|
||||
|
||||
// RunPluginJobTypeAPI runs full workflow for one job type: detect then dispatch detected jobs.
|
||||
func (s *AdminServer) RunPluginJobTypeAPI(w http.ResponseWriter, r *http.Request) {
|
||||
jobType := strings.TrimSpace(mux.Vars(r)["jobType"])
|
||||
if jobType == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "jobType is required")
|
||||
return
|
||||
}
|
||||
|
||||
var req struct {
|
||||
ClusterContext json.RawMessage `json:"cluster_context"`
|
||||
MaxResults int32 `json:"max_results"`
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
Attempt int32 `json:"attempt"`
|
||||
}
|
||||
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil && err != io.EOF {
|
||||
writeJSONError(w, http.StatusBadRequest, "invalid request body: "+err.Error())
|
||||
return
|
||||
}
|
||||
if req.Attempt < 1 {
|
||||
req.Attempt = 1
|
||||
}
|
||||
|
||||
clusterContext, err := s.parseOrBuildClusterContext(req.ClusterContext)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
timeout := normalizeTimeout(req.TimeoutSeconds, defaultPluginRunTimeout, maxPluginRunTimeout)
|
||||
ctx, cancel := context.WithTimeout(r.Context(), timeout)
|
||||
defer cancel()
|
||||
|
||||
proposals, err := s.RunPluginDetection(ctx, jobType, clusterContext, req.MaxResults)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
detectedCount := len(proposals)
|
||||
|
||||
filteredProposals, skippedActiveCount, err := s.FilterPluginProposalsWithActiveJobs(jobType, proposals)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
type executionResult struct {
|
||||
JobID string `json:"job_id"`
|
||||
Success bool `json:"success"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Completion map[string]interface{} `json:"completion,omitempty"`
|
||||
}
|
||||
|
||||
results := make([]executionResult, 0, len(filteredProposals))
|
||||
successCount := 0
|
||||
errorCount := 0
|
||||
|
||||
for index, proposal := range filteredProposals {
|
||||
job := buildJobSpecFromProposal(jobType, proposal, index)
|
||||
completed, execErr := s.ExecutePluginJob(ctx, job, clusterContext, req.Attempt)
|
||||
|
||||
result := executionResult{
|
||||
JobID: job.JobId,
|
||||
Success: execErr == nil,
|
||||
}
|
||||
|
||||
if completed != nil {
|
||||
if payload, marshalErr := protoMessageToMap(completed); marshalErr == nil {
|
||||
result.Completion = payload
|
||||
}
|
||||
}
|
||||
|
||||
if execErr != nil {
|
||||
result.Error = execErr.Error()
|
||||
errorCount++
|
||||
} else {
|
||||
successCount++
|
||||
}
|
||||
|
||||
results = append(results, result)
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"job_type": jobType,
|
||||
"detected_count": detectedCount,
|
||||
"ready_to_execute_count": len(filteredProposals),
|
||||
"skipped_active_count": skippedActiveCount,
|
||||
"executed_count": len(results),
|
||||
"success_count": successCount,
|
||||
"error_count": errorCount,
|
||||
"execution_results": results,
|
||||
})
|
||||
}
|
||||
|
||||
// ExecutePluginJobAPI executes one job on a capable worker and waits for completion.
|
||||
func (s *AdminServer) ExecutePluginJobAPI(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
Job json.RawMessage `json:"job"`
|
||||
ClusterContext json.RawMessage `json:"cluster_context"`
|
||||
Attempt int32 `json:"attempt"`
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
}
|
||||
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "invalid request body: "+err.Error())
|
||||
return
|
||||
}
|
||||
if len(req.Job) == 0 {
|
||||
writeJSONError(w, http.StatusBadRequest, "job is required")
|
||||
return
|
||||
}
|
||||
|
||||
job := &plugin_pb.JobSpec{}
|
||||
if err := (protojson.UnmarshalOptions{DiscardUnknown: true}).Unmarshal(req.Job, job); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "invalid job payload: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
clusterContext, err := s.parseOrBuildClusterContext(req.ClusterContext)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if req.Attempt < 1 {
|
||||
req.Attempt = 1
|
||||
}
|
||||
|
||||
timeout := normalizeTimeout(req.TimeoutSeconds, defaultPluginExecutionTimeout, maxPluginExecutionTimeout)
|
||||
ctx, cancel := context.WithTimeout(r.Context(), timeout)
|
||||
defer cancel()
|
||||
|
||||
completed, err := s.ExecutePluginJob(ctx, job, clusterContext, req.Attempt)
|
||||
if err != nil {
|
||||
if completed != nil {
|
||||
payload, marshalErr := protoMessageToMap(completed)
|
||||
if marshalErr == nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]interface{}{"error": err.Error(), "completion": payload})
|
||||
return
|
||||
}
|
||||
}
|
||||
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
renderProtoJSON(w, http.StatusOK, completed)
|
||||
}
|
||||
|
||||
func (s *AdminServer) parseOrBuildClusterContext(raw json.RawMessage) (*plugin_pb.ClusterContext, error) {
|
||||
if len(raw) == 0 {
|
||||
return s.buildDefaultPluginClusterContext(), nil
|
||||
}
|
||||
|
||||
contextMessage := &plugin_pb.ClusterContext{}
|
||||
if err := (protojson.UnmarshalOptions{DiscardUnknown: true}).Unmarshal(raw, contextMessage); err != nil {
|
||||
return nil, fmt.Errorf("invalid cluster_context payload: %w", err)
|
||||
}
|
||||
|
||||
fallback := s.buildDefaultPluginClusterContext()
|
||||
if len(contextMessage.MasterGrpcAddresses) == 0 {
|
||||
contextMessage.MasterGrpcAddresses = append(contextMessage.MasterGrpcAddresses, fallback.MasterGrpcAddresses...)
|
||||
}
|
||||
if len(contextMessage.FilerGrpcAddresses) == 0 {
|
||||
contextMessage.FilerGrpcAddresses = append(contextMessage.FilerGrpcAddresses, fallback.FilerGrpcAddresses...)
|
||||
}
|
||||
if len(contextMessage.VolumeGrpcAddresses) == 0 {
|
||||
contextMessage.VolumeGrpcAddresses = append(contextMessage.VolumeGrpcAddresses, fallback.VolumeGrpcAddresses...)
|
||||
}
|
||||
if contextMessage.Metadata == nil {
|
||||
contextMessage.Metadata = map[string]string{}
|
||||
}
|
||||
contextMessage.Metadata["source"] = "admin"
|
||||
|
||||
return contextMessage, nil
|
||||
}
|
||||
|
||||
func (s *AdminServer) buildDefaultPluginClusterContext() *plugin_pb.ClusterContext {
|
||||
clusterContext := &plugin_pb.ClusterContext{
|
||||
MasterGrpcAddresses: make([]string, 0),
|
||||
FilerGrpcAddresses: make([]string, 0),
|
||||
VolumeGrpcAddresses: make([]string, 0),
|
||||
Metadata: map[string]string{
|
||||
"source": "admin",
|
||||
},
|
||||
}
|
||||
|
||||
masterAddress := string(s.masterClient.GetMaster(context.Background()))
|
||||
if masterAddress != "" {
|
||||
clusterContext.MasterGrpcAddresses = append(clusterContext.MasterGrpcAddresses, masterAddress)
|
||||
}
|
||||
|
||||
filerSeen := map[string]struct{}{}
|
||||
for _, filer := range s.GetAllFilers() {
|
||||
filer = strings.TrimSpace(filer)
|
||||
if filer == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := filerSeen[filer]; exists {
|
||||
continue
|
||||
}
|
||||
filerSeen[filer] = struct{}{}
|
||||
clusterContext.FilerGrpcAddresses = append(clusterContext.FilerGrpcAddresses, filer)
|
||||
}
|
||||
|
||||
volumeSeen := map[string]struct{}{}
|
||||
if volumeServers, err := s.GetClusterVolumeServers(); err == nil {
|
||||
for _, server := range volumeServers.VolumeServers {
|
||||
address := strings.TrimSpace(server.GetDisplayAddress())
|
||||
if address == "" {
|
||||
address = strings.TrimSpace(server.Address)
|
||||
}
|
||||
if address == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := volumeSeen[address]; exists {
|
||||
continue
|
||||
}
|
||||
volumeSeen[address] = struct{}{}
|
||||
clusterContext.VolumeGrpcAddresses = append(clusterContext.VolumeGrpcAddresses, address)
|
||||
}
|
||||
} else {
|
||||
glog.V(1).Infof("failed to build default plugin volume context: %v", err)
|
||||
}
|
||||
|
||||
sort.Strings(clusterContext.MasterGrpcAddresses)
|
||||
sort.Strings(clusterContext.FilerGrpcAddresses)
|
||||
sort.Strings(clusterContext.VolumeGrpcAddresses)
|
||||
|
||||
return clusterContext
|
||||
}
|
||||
|
||||
const parseProtoJSONBodyMaxBytes = 1 << 20 // 1 MB
|
||||
|
||||
func parseProtoJSONBody(w http.ResponseWriter, r *http.Request, message proto.Message) error {
|
||||
limitedBody := http.MaxBytesReader(w, r.Body, parseProtoJSONBodyMaxBytes)
|
||||
data, err := io.ReadAll(limitedBody)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read request body: %w", err)
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return fmt.Errorf("request body is empty")
|
||||
}
|
||||
if err := (protojson.UnmarshalOptions{DiscardUnknown: true}).Unmarshal(data, message); err != nil {
|
||||
return fmt.Errorf("invalid protobuf json: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func renderProtoJSON(w http.ResponseWriter, statusCode int, message proto.Message) {
|
||||
payload, err := protojson.MarshalOptions{
|
||||
UseProtoNames: true,
|
||||
EmitUnpopulated: true,
|
||||
}.Marshal(message)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "failed to encode response: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(statusCode)
|
||||
_, _ = w.Write(payload)
|
||||
}
|
||||
|
||||
func protoMessageToMap(message proto.Message) (map[string]interface{}, error) {
|
||||
payload, err := protojson.MarshalOptions{UseProtoNames: true}.Marshal(message)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]interface{}{}
|
||||
if err := json.Unmarshal(payload, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func normalizeTimeout(timeoutSeconds int, defaultTimeout, maxTimeout time.Duration) time.Duration {
|
||||
if timeoutSeconds <= 0 {
|
||||
return defaultTimeout
|
||||
}
|
||||
timeout := time.Duration(timeoutSeconds) * time.Second
|
||||
if timeout > maxTimeout {
|
||||
return maxTimeout
|
||||
}
|
||||
return timeout
|
||||
}
|
||||
|
||||
func buildJobSpecFromProposal(jobType string, proposal *plugin_pb.JobProposal, index int) *plugin_pb.JobSpec {
|
||||
now := timestamppb.Now()
|
||||
suffix := make([]byte, 4)
|
||||
if _, err := rand.Read(suffix); err != nil {
|
||||
// Fallback to simpler ID if rand fails
|
||||
suffix = []byte(fmt.Sprintf("%d", index))
|
||||
}
|
||||
jobID := fmt.Sprintf("%s-%d-%s", jobType, now.AsTime().UnixNano(), hex.EncodeToString(suffix))
|
||||
|
||||
jobSpec := &plugin_pb.JobSpec{
|
||||
JobId: jobID,
|
||||
JobType: jobType,
|
||||
Priority: plugin_pb.JobPriority_JOB_PRIORITY_NORMAL,
|
||||
CreatedAt: now,
|
||||
Labels: make(map[string]string),
|
||||
Parameters: make(map[string]*plugin_pb.ConfigValue),
|
||||
DedupeKey: "",
|
||||
}
|
||||
|
||||
if proposal != nil {
|
||||
jobSpec.Summary = proposal.Summary
|
||||
jobSpec.Detail = proposal.Detail
|
||||
if proposal.Priority != plugin_pb.JobPriority_JOB_PRIORITY_UNSPECIFIED {
|
||||
jobSpec.Priority = proposal.Priority
|
||||
}
|
||||
jobSpec.DedupeKey = proposal.DedupeKey
|
||||
jobSpec.Parameters = plugin.CloneConfigValueMap(proposal.Parameters)
|
||||
if proposal.Labels != nil {
|
||||
for k, v := range proposal.Labels {
|
||||
jobSpec.Labels[k] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return jobSpec
|
||||
}
|
||||
|
||||
func parsePositiveInt(raw string, defaultValue int) int {
|
||||
value, err := strconv.Atoi(strings.TrimSpace(raw))
|
||||
if err != nil || value <= 0 {
|
||||
return defaultValue
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// cloneConfigValueMap is now exported by the plugin package as CloneConfigValueMap
|
||||
@@ -0,0 +1,33 @@
|
||||
package dash
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
)
|
||||
|
||||
func TestBuildJobSpecFromProposalDoesNotReuseProposalID(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
proposal := &plugin_pb.JobProposal{
|
||||
ProposalId: "vacuum-2",
|
||||
DedupeKey: "vacuum:2",
|
||||
JobType: "vacuum",
|
||||
}
|
||||
|
||||
jobA := buildJobSpecFromProposal("vacuum", proposal, 0)
|
||||
jobB := buildJobSpecFromProposal("vacuum", proposal, 1)
|
||||
|
||||
if jobA.JobId == proposal.ProposalId {
|
||||
t.Fatalf("job id must not reuse proposal id: %s", jobA.JobId)
|
||||
}
|
||||
if jobB.JobId == proposal.ProposalId {
|
||||
t.Fatalf("job id must not reuse proposal id: %s", jobB.JobId)
|
||||
}
|
||||
if jobA.JobId == jobB.JobId {
|
||||
t.Fatalf("job ids must be unique across jobs: %s", jobA.JobId)
|
||||
}
|
||||
if jobA.DedupeKey != proposal.DedupeKey {
|
||||
t.Fatalf("dedupe key must be preserved: got=%s want=%s", jobA.DedupeKey, proposal.DedupeKey)
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
@@ -67,10 +66,10 @@ func parseNamespaceInput(namespace string) ([]string, error) {
|
||||
return s3tables.ParseNamespace(namespace)
|
||||
}
|
||||
|
||||
func (s *AdminServer) parseNamespaceFromGin(c *gin.Context, namespace string) ([]string, bool) {
|
||||
func (s *AdminServer) parseNamespaceFromRequest(w http.ResponseWriter, namespace string) ([]string, bool) {
|
||||
parts, err := parseNamespaceInput(namespace)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid namespace: " + err.Error()})
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid namespace: "+err.Error())
|
||||
return nil, false
|
||||
}
|
||||
return parts, true
|
||||
@@ -569,58 +568,61 @@ func parseSummaryInt(summary map[string]string, keys ...string) (int64, bool) {
|
||||
|
||||
// API handlers
|
||||
|
||||
func (s *AdminServer) ListS3TablesBucketsAPI(c *gin.Context) {
|
||||
data, err := s.GetS3TablesBucketsData(c.Request.Context())
|
||||
func (s *AdminServer) ListS3TablesBucketsAPI(w http.ResponseWriter, r *http.Request) {
|
||||
data, err := s.GetS3TablesBucketsData(r.Context())
|
||||
if err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
writeS3TablesError(w, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, data)
|
||||
writeJSON(w, http.StatusOK, data)
|
||||
}
|
||||
|
||||
func (s *AdminServer) CreateS3TablesBucket(c *gin.Context) {
|
||||
func (s *AdminServer) CreateS3TablesBucket(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Name string `json:"name"`
|
||||
Tags map[string]string `json:"tags"`
|
||||
Owner string `json:"owner"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
if req.Name == "" {
|
||||
c.JSON(400, gin.H{"error": "Bucket name is required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Bucket name is required")
|
||||
return
|
||||
}
|
||||
owner := strings.TrimSpace(req.Owner)
|
||||
if len(owner) > MaxOwnerNameLength {
|
||||
c.JSON(400, gin.H{"error": fmt.Sprintf("Owner name must be %d characters or less", MaxOwnerNameLength)})
|
||||
writeJSONError(w, http.StatusBadRequest, fmt.Sprintf("Owner name must be %d characters or less", MaxOwnerNameLength))
|
||||
return
|
||||
}
|
||||
if len(req.Tags) > 0 {
|
||||
if err := s3tables.ValidateTags(req.Tags); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid tags: " + err.Error()})
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid tags: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
createReq := &s3tables.CreateTableBucketRequest{Name: req.Name, Tags: req.Tags}
|
||||
var resp s3tables.CreateTableBucketResponse
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "CreateTableBucket", createReq, &resp); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
if err := s.executeS3TablesOperation(r.Context(), "CreateTableBucket", createReq, &resp); err != nil {
|
||||
writeS3TablesError(w, err)
|
||||
return
|
||||
}
|
||||
if owner != "" {
|
||||
if err := s.SetTableBucketOwner(c.Request.Context(), req.Name, owner); err != nil {
|
||||
if err := s.SetTableBucketOwner(r.Context(), req.Name, owner); err != nil {
|
||||
deleteReq := &s3tables.DeleteTableBucketRequest{TableBucketARN: resp.ARN}
|
||||
if deleteErr := s.executeS3TablesOperation(c.Request.Context(), "DeleteTableBucket", deleteReq, nil); deleteErr != nil {
|
||||
c.JSON(500, gin.H{"error": fmt.Sprintf("Failed to set table bucket owner: %v; rollback delete failed: %v", err, deleteErr)})
|
||||
if deleteErr := s.executeS3TablesOperation(r.Context(), "DeleteTableBucket", deleteReq, nil); deleteErr != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to set table bucket owner: %v; rollback delete failed: %v", err, deleteErr))
|
||||
return
|
||||
}
|
||||
writeS3TablesError(c, err)
|
||||
writeS3TablesError(w, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
c.JSON(201, gin.H{"arn": resp.ARN})
|
||||
writeJSON(w, http.StatusCreated, map[string]interface{}{"arn": resp.ARN})
|
||||
}
|
||||
|
||||
func (s *AdminServer) SetTableBucketOwner(ctx context.Context, bucketName, owner string) error {
|
||||
@@ -663,101 +665,107 @@ func (s *AdminServer) SetTableBucketOwner(ctx context.Context, bucketName, owner
|
||||
})
|
||||
}
|
||||
|
||||
func (s *AdminServer) DeleteS3TablesBucket(c *gin.Context) {
|
||||
bucketArn := c.Query("bucket")
|
||||
func (s *AdminServer) DeleteS3TablesBucket(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
bucketArn := r.URL.Query().Get("bucket")
|
||||
if bucketArn == "" {
|
||||
c.JSON(400, gin.H{"error": "Bucket ARN is required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Bucket ARN is required")
|
||||
return
|
||||
}
|
||||
req := &s3tables.DeleteTableBucketRequest{TableBucketARN: bucketArn}
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "DeleteTableBucket", req, nil); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
if err := s.executeS3TablesOperation(r.Context(), "DeleteTableBucket", req, nil); err != nil {
|
||||
writeS3TablesError(w, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"message": "Bucket deleted"})
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"message": "Bucket deleted"})
|
||||
}
|
||||
|
||||
func (s *AdminServer) ListS3TablesNamespacesAPI(c *gin.Context) {
|
||||
bucketArn := c.Query("bucket")
|
||||
func (s *AdminServer) ListS3TablesNamespacesAPI(w http.ResponseWriter, r *http.Request) {
|
||||
bucketArn := r.URL.Query().Get("bucket")
|
||||
if bucketArn == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket query parameter is required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "bucket query parameter is required")
|
||||
return
|
||||
}
|
||||
data, err := s.GetS3TablesNamespacesData(c.Request.Context(), bucketArn)
|
||||
data, err := s.GetS3TablesNamespacesData(r.Context(), bucketArn)
|
||||
if err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
writeS3TablesError(w, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, data)
|
||||
writeJSON(w, http.StatusOK, data)
|
||||
}
|
||||
|
||||
func (s *AdminServer) CreateS3TablesNamespace(c *gin.Context) {
|
||||
if !requireSessionCSRFToken(c) {
|
||||
func (s *AdminServer) CreateS3TablesNamespace(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
BucketARN string `json:"bucket_arn"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
if req.BucketARN == "" || req.Name == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket_arn and name are required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "bucket_arn and name are required")
|
||||
return
|
||||
}
|
||||
namespaceParts, ok := s.parseNamespaceFromGin(c, req.Name)
|
||||
namespaceParts, ok := s.parseNamespaceFromRequest(w, req.Name)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
createReq := &s3tables.CreateNamespaceRequest{TableBucketARN: req.BucketARN, Namespace: namespaceParts}
|
||||
var resp s3tables.CreateNamespaceResponse
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "CreateNamespace", createReq, &resp); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
if err := s.executeS3TablesOperation(r.Context(), "CreateNamespace", createReq, &resp); err != nil {
|
||||
writeS3TablesError(w, err)
|
||||
return
|
||||
}
|
||||
c.JSON(201, gin.H{"namespace": resp.Namespace})
|
||||
writeJSON(w, http.StatusCreated, map[string]interface{}{"namespace": resp.Namespace})
|
||||
}
|
||||
|
||||
func (s *AdminServer) DeleteS3TablesNamespace(c *gin.Context) {
|
||||
if !requireSessionCSRFToken(c) {
|
||||
func (s *AdminServer) DeleteS3TablesNamespace(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
bucketArn := c.Query("bucket")
|
||||
namespace := c.Query("name")
|
||||
bucketArn := r.URL.Query().Get("bucket")
|
||||
namespace := r.URL.Query().Get("name")
|
||||
if bucketArn == "" || namespace == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket and name query parameters are required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "bucket and name query parameters are required")
|
||||
return
|
||||
}
|
||||
namespaceParts, ok := s.parseNamespaceFromGin(c, namespace)
|
||||
namespaceParts, ok := s.parseNamespaceFromRequest(w, namespace)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
req := &s3tables.DeleteNamespaceRequest{TableBucketARN: bucketArn, Namespace: namespaceParts}
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "DeleteNamespace", req, nil); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
if err := s.executeS3TablesOperation(r.Context(), "DeleteNamespace", req, nil); err != nil {
|
||||
writeS3TablesError(w, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"message": "Namespace deleted"})
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"message": "Namespace deleted"})
|
||||
}
|
||||
|
||||
func (s *AdminServer) ListS3TablesTablesAPI(c *gin.Context) {
|
||||
bucketArn := c.Query("bucket")
|
||||
func (s *AdminServer) ListS3TablesTablesAPI(w http.ResponseWriter, r *http.Request) {
|
||||
bucketArn := r.URL.Query().Get("bucket")
|
||||
if bucketArn == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket query parameter is required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "bucket query parameter is required")
|
||||
return
|
||||
}
|
||||
namespace := c.Query("namespace")
|
||||
data, err := s.GetS3TablesTablesData(c.Request.Context(), bucketArn, namespace)
|
||||
namespace := r.URL.Query().Get("namespace")
|
||||
data, err := s.GetS3TablesTablesData(r.Context(), bucketArn, namespace)
|
||||
if err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
writeS3TablesError(w, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, data)
|
||||
writeJSON(w, http.StatusOK, data)
|
||||
}
|
||||
|
||||
func (s *AdminServer) CreateS3TablesTable(c *gin.Context) {
|
||||
func (s *AdminServer) CreateS3TablesTable(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
BucketARN string `json:"bucket_arn"`
|
||||
Namespace string `json:"namespace"`
|
||||
@@ -766,15 +774,15 @@ func (s *AdminServer) CreateS3TablesTable(c *gin.Context) {
|
||||
Tags map[string]string `json:"tags"`
|
||||
Metadata *s3tables.TableMetadata `json:"metadata"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
if req.BucketARN == "" || req.Namespace == "" || req.Name == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket_arn, namespace, and name are required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "bucket_arn, namespace, and name are required")
|
||||
return
|
||||
}
|
||||
namespaceParts, ok := s.parseNamespaceFromGin(c, req.Namespace)
|
||||
namespaceParts, ok := s.parseNamespaceFromRequest(w, req.Namespace)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
@@ -784,7 +792,7 @@ func (s *AdminServer) CreateS3TablesTable(c *gin.Context) {
|
||||
}
|
||||
if len(req.Tags) > 0 {
|
||||
if err := s3tables.ValidateTags(req.Tags); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid tags: " + err.Error()})
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid tags: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -797,211 +805,232 @@ func (s *AdminServer) CreateS3TablesTable(c *gin.Context) {
|
||||
Metadata: req.Metadata,
|
||||
}
|
||||
var resp s3tables.CreateTableResponse
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "CreateTable", createReq, &resp); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
if err := s.executeS3TablesOperation(r.Context(), "CreateTable", createReq, &resp); err != nil {
|
||||
writeS3TablesError(w, err)
|
||||
return
|
||||
}
|
||||
c.JSON(201, gin.H{"table_arn": resp.TableARN, "version_token": resp.VersionToken})
|
||||
writeJSON(w, http.StatusCreated, map[string]interface{}{"table_arn": resp.TableARN, "version_token": resp.VersionToken})
|
||||
}
|
||||
|
||||
func (s *AdminServer) DeleteS3TablesTable(c *gin.Context) {
|
||||
bucketArn := c.Query("bucket")
|
||||
namespace := c.Query("namespace")
|
||||
name := c.Query("name")
|
||||
version := c.Query("version")
|
||||
if bucketArn == "" || namespace == "" || name == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket, namespace, and name query parameters are required"})
|
||||
func (s *AdminServer) DeleteS3TablesTable(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
namespaceParts, ok := s.parseNamespaceFromGin(c, namespace)
|
||||
bucketArn := r.URL.Query().Get("bucket")
|
||||
namespace := r.URL.Query().Get("namespace")
|
||||
name := r.URL.Query().Get("name")
|
||||
version := r.URL.Query().Get("version")
|
||||
if bucketArn == "" || namespace == "" || name == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "bucket, namespace, and name query parameters are required")
|
||||
return
|
||||
}
|
||||
namespaceParts, ok := s.parseNamespaceFromRequest(w, namespace)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
req := &s3tables.DeleteTableRequest{TableBucketARN: bucketArn, Namespace: namespaceParts, Name: name, VersionToken: version}
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "DeleteTable", req, nil); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
if err := s.executeS3TablesOperation(r.Context(), "DeleteTable", req, nil); err != nil {
|
||||
writeS3TablesError(w, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"message": "Table deleted"})
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"message": "Table deleted"})
|
||||
}
|
||||
|
||||
func (s *AdminServer) PutS3TablesBucketPolicy(c *gin.Context) {
|
||||
func (s *AdminServer) PutS3TablesBucketPolicy(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
BucketARN string `json:"bucket_arn"`
|
||||
Policy string `json:"policy"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
if req.BucketARN == "" || req.Policy == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket_arn and policy are required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "bucket_arn and policy are required")
|
||||
return
|
||||
}
|
||||
putReq := &s3tables.PutTableBucketPolicyRequest{TableBucketARN: req.BucketARN, ResourcePolicy: req.Policy}
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "PutTableBucketPolicy", putReq, nil); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
if err := s.executeS3TablesOperation(r.Context(), "PutTableBucketPolicy", putReq, nil); err != nil {
|
||||
writeS3TablesError(w, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"message": "Policy updated"})
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"message": "Policy updated"})
|
||||
}
|
||||
|
||||
func (s *AdminServer) GetS3TablesBucketPolicy(c *gin.Context) {
|
||||
bucketArn := c.Query("bucket")
|
||||
func (s *AdminServer) GetS3TablesBucketPolicy(w http.ResponseWriter, r *http.Request) {
|
||||
bucketArn := r.URL.Query().Get("bucket")
|
||||
if bucketArn == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket query parameter is required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "bucket query parameter is required")
|
||||
return
|
||||
}
|
||||
getReq := &s3tables.GetTableBucketPolicyRequest{TableBucketARN: bucketArn}
|
||||
var resp s3tables.GetTableBucketPolicyResponse
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "GetTableBucketPolicy", getReq, &resp); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
if err := s.executeS3TablesOperation(r.Context(), "GetTableBucketPolicy", getReq, &resp); err != nil {
|
||||
writeS3TablesError(w, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"policy": resp.ResourcePolicy})
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"policy": resp.ResourcePolicy})
|
||||
}
|
||||
|
||||
func (s *AdminServer) DeleteS3TablesBucketPolicy(c *gin.Context) {
|
||||
bucketArn := c.Query("bucket")
|
||||
func (s *AdminServer) DeleteS3TablesBucketPolicy(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
bucketArn := r.URL.Query().Get("bucket")
|
||||
if bucketArn == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket query parameter is required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "bucket query parameter is required")
|
||||
return
|
||||
}
|
||||
deleteReq := &s3tables.DeleteTableBucketPolicyRequest{TableBucketARN: bucketArn}
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "DeleteTableBucketPolicy", deleteReq, nil); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
if err := s.executeS3TablesOperation(r.Context(), "DeleteTableBucketPolicy", deleteReq, nil); err != nil {
|
||||
writeS3TablesError(w, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"message": "Policy deleted"})
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"message": "Policy deleted"})
|
||||
}
|
||||
|
||||
func (s *AdminServer) PutS3TablesTablePolicy(c *gin.Context) {
|
||||
func (s *AdminServer) PutS3TablesTablePolicy(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
BucketARN string `json:"bucket_arn"`
|
||||
Namespace string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
Policy string `json:"policy"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
if req.BucketARN == "" || req.Namespace == "" || req.Name == "" || req.Policy == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket_arn, namespace, name, and policy are required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "bucket_arn, namespace, name, and policy are required")
|
||||
return
|
||||
}
|
||||
namespaceParts, ok := s.parseNamespaceFromGin(c, req.Namespace)
|
||||
namespaceParts, ok := s.parseNamespaceFromRequest(w, req.Namespace)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
putReq := &s3tables.PutTablePolicyRequest{TableBucketARN: req.BucketARN, Namespace: namespaceParts, Name: req.Name, ResourcePolicy: req.Policy}
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "PutTablePolicy", putReq, nil); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
if err := s.executeS3TablesOperation(r.Context(), "PutTablePolicy", putReq, nil); err != nil {
|
||||
writeS3TablesError(w, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"message": "Policy updated"})
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"message": "Policy updated"})
|
||||
}
|
||||
|
||||
func (s *AdminServer) GetS3TablesTablePolicy(c *gin.Context) {
|
||||
bucketArn := c.Query("bucket")
|
||||
namespace := c.Query("namespace")
|
||||
name := c.Query("name")
|
||||
func (s *AdminServer) GetS3TablesTablePolicy(w http.ResponseWriter, r *http.Request) {
|
||||
bucketArn := r.URL.Query().Get("bucket")
|
||||
namespace := r.URL.Query().Get("namespace")
|
||||
name := r.URL.Query().Get("name")
|
||||
if bucketArn == "" || namespace == "" || name == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket, namespace, and name query parameters are required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "bucket, namespace, and name query parameters are required")
|
||||
return
|
||||
}
|
||||
namespaceParts, ok := s.parseNamespaceFromGin(c, namespace)
|
||||
namespaceParts, ok := s.parseNamespaceFromRequest(w, namespace)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
getReq := &s3tables.GetTablePolicyRequest{TableBucketARN: bucketArn, Namespace: namespaceParts, Name: name}
|
||||
var resp s3tables.GetTablePolicyResponse
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "GetTablePolicy", getReq, &resp); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
if err := s.executeS3TablesOperation(r.Context(), "GetTablePolicy", getReq, &resp); err != nil {
|
||||
writeS3TablesError(w, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"policy": resp.ResourcePolicy})
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"policy": resp.ResourcePolicy})
|
||||
}
|
||||
|
||||
func (s *AdminServer) DeleteS3TablesTablePolicy(c *gin.Context) {
|
||||
bucketArn := c.Query("bucket")
|
||||
namespace := c.Query("namespace")
|
||||
name := c.Query("name")
|
||||
if bucketArn == "" || namespace == "" || name == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket, namespace, and name query parameters are required"})
|
||||
func (s *AdminServer) DeleteS3TablesTablePolicy(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
namespaceParts, ok := s.parseNamespaceFromGin(c, namespace)
|
||||
bucketArn := r.URL.Query().Get("bucket")
|
||||
namespace := r.URL.Query().Get("namespace")
|
||||
name := r.URL.Query().Get("name")
|
||||
if bucketArn == "" || namespace == "" || name == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "bucket, namespace, and name query parameters are required")
|
||||
return
|
||||
}
|
||||
namespaceParts, ok := s.parseNamespaceFromRequest(w, namespace)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
deleteReq := &s3tables.DeleteTablePolicyRequest{TableBucketARN: bucketArn, Namespace: namespaceParts, Name: name}
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "DeleteTablePolicy", deleteReq, nil); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
if err := s.executeS3TablesOperation(r.Context(), "DeleteTablePolicy", deleteReq, nil); err != nil {
|
||||
writeS3TablesError(w, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"message": "Policy deleted"})
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"message": "Policy deleted"})
|
||||
}
|
||||
|
||||
func (s *AdminServer) TagS3TablesResource(c *gin.Context) {
|
||||
func (s *AdminServer) TagS3TablesResource(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
ResourceARN string `json:"resource_arn"`
|
||||
Tags map[string]string `json:"tags"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
if req.ResourceARN == "" || len(req.Tags) == 0 {
|
||||
c.JSON(400, gin.H{"error": "resource_arn and tags are required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "resource_arn and tags are required")
|
||||
return
|
||||
}
|
||||
if err := s3tables.ValidateTags(req.Tags); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid tags: " + err.Error()})
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid tags: "+err.Error())
|
||||
return
|
||||
}
|
||||
tagReq := &s3tables.TagResourceRequest{ResourceARN: req.ResourceARN, Tags: req.Tags}
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "TagResource", tagReq, nil); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
if err := s.executeS3TablesOperation(r.Context(), "TagResource", tagReq, nil); err != nil {
|
||||
writeS3TablesError(w, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"message": "Tags updated"})
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"message": "Tags updated"})
|
||||
}
|
||||
|
||||
func (s *AdminServer) ListS3TablesTags(c *gin.Context) {
|
||||
resourceArn := c.Query("arn")
|
||||
func (s *AdminServer) ListS3TablesTags(w http.ResponseWriter, r *http.Request) {
|
||||
resourceArn := r.URL.Query().Get("arn")
|
||||
if resourceArn == "" {
|
||||
c.JSON(400, gin.H{"error": "arn query parameter is required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "arn query parameter is required")
|
||||
return
|
||||
}
|
||||
listReq := &s3tables.ListTagsForResourceRequest{ResourceARN: resourceArn}
|
||||
var resp s3tables.ListTagsForResourceResponse
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "ListTagsForResource", listReq, &resp); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
if err := s.executeS3TablesOperation(r.Context(), "ListTagsForResource", listReq, &resp); err != nil {
|
||||
writeS3TablesError(w, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, resp)
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
func (s *AdminServer) UntagS3TablesResource(c *gin.Context) {
|
||||
func (s *AdminServer) UntagS3TablesResource(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
ResourceARN string `json:"resource_arn"`
|
||||
TagKeys []string `json:"tag_keys"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
if req.ResourceARN == "" || len(req.TagKeys) == 0 {
|
||||
c.JSON(400, gin.H{"error": "resource_arn and tag_keys are required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "resource_arn and tag_keys are required")
|
||||
return
|
||||
}
|
||||
untagReq := &s3tables.UntagResourceRequest{ResourceARN: req.ResourceARN, TagKeys: req.TagKeys}
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "UntagResource", untagReq, nil); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
if err := s.executeS3TablesOperation(r.Context(), "UntagResource", untagReq, nil); err != nil {
|
||||
writeS3TablesError(w, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"message": "Tags removed"})
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"message": "Tags removed"})
|
||||
}
|
||||
|
||||
func parseS3TablesErrorMessage(err error) string {
|
||||
@@ -1018,8 +1047,8 @@ func parseS3TablesErrorMessage(err error) string {
|
||||
return err.Error()
|
||||
}
|
||||
|
||||
func writeS3TablesError(c *gin.Context, err error) {
|
||||
c.JSON(s3TablesErrorStatus(err), gin.H{"error": parseS3TablesErrorMessage(err)})
|
||||
func writeS3TablesError(w http.ResponseWriter, err error) {
|
||||
writeJSONError(w, s3TablesErrorStatus(err), parseS3TablesErrorMessage(err))
|
||||
}
|
||||
|
||||
func s3TablesErrorStatus(err error) int {
|
||||
|
||||
@@ -5,12 +5,14 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
@@ -93,6 +95,10 @@ func (s *WorkerGrpcServer) StartWithTLS(port int) error {
|
||||
grpcServer := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.admin"))
|
||||
|
||||
worker_pb.RegisterWorkerServiceServer(grpcServer, s)
|
||||
if plugin := s.adminServer.GetPlugin(); plugin != nil {
|
||||
plugin_pb.RegisterPluginControlServiceServer(grpcServer, plugin)
|
||||
glog.V(0).Infof("Plugin gRPC service registered on worker gRPC server")
|
||||
}
|
||||
|
||||
s.grpcServer = grpcServer
|
||||
s.listener = listener
|
||||
@@ -114,6 +120,25 @@ func (s *WorkerGrpcServer) StartWithTLS(port int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListenPort returns the currently bound worker gRPC listen port.
|
||||
func (s *WorkerGrpcServer) ListenPort() int {
|
||||
if s == nil || s.listener == nil {
|
||||
return 0
|
||||
}
|
||||
if tcpAddr, ok := s.listener.Addr().(*net.TCPAddr); ok {
|
||||
return tcpAddr.Port
|
||||
}
|
||||
_, portStr, err := net.SplitHostPort(s.listener.Addr().String())
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
port, err := strconv.Atoi(portStr)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return port
|
||||
}
|
||||
|
||||
// Stop stops the gRPC server
|
||||
func (s *WorkerGrpcServer) Stop() error {
|
||||
if !s.running {
|
||||
|
||||
@@ -5,7 +5,8 @@ import (
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/gorilla/sessions"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/view/app"
|
||||
@@ -18,51 +19,53 @@ import (
|
||||
// AdminHandlers contains all the HTTP handlers for the admin interface
|
||||
type AdminHandlers struct {
|
||||
adminServer *dash.AdminServer
|
||||
sessionStore sessions.Store
|
||||
authHandlers *AuthHandlers
|
||||
clusterHandlers *ClusterHandlers
|
||||
fileBrowserHandlers *FileBrowserHandlers
|
||||
userHandlers *UserHandlers
|
||||
policyHandlers *PolicyHandlers
|
||||
maintenanceHandlers *MaintenanceHandlers
|
||||
pluginHandlers *PluginHandlers
|
||||
mqHandlers *MessageQueueHandlers
|
||||
serviceAccountHandlers *ServiceAccountHandlers
|
||||
}
|
||||
|
||||
// NewAdminHandlers creates a new instance of AdminHandlers
|
||||
func NewAdminHandlers(adminServer *dash.AdminServer) *AdminHandlers {
|
||||
authHandlers := NewAuthHandlers(adminServer)
|
||||
func NewAdminHandlers(adminServer *dash.AdminServer, store sessions.Store) *AdminHandlers {
|
||||
authHandlers := NewAuthHandlers(adminServer, store)
|
||||
clusterHandlers := NewClusterHandlers(adminServer)
|
||||
fileBrowserHandlers := NewFileBrowserHandlers(adminServer)
|
||||
userHandlers := NewUserHandlers(adminServer)
|
||||
policyHandlers := NewPolicyHandlers(adminServer)
|
||||
maintenanceHandlers := NewMaintenanceHandlers(adminServer)
|
||||
pluginHandlers := NewPluginHandlers(adminServer)
|
||||
mqHandlers := NewMessageQueueHandlers(adminServer)
|
||||
serviceAccountHandlers := NewServiceAccountHandlers(adminServer)
|
||||
return &AdminHandlers{
|
||||
adminServer: adminServer,
|
||||
sessionStore: store,
|
||||
authHandlers: authHandlers,
|
||||
clusterHandlers: clusterHandlers,
|
||||
fileBrowserHandlers: fileBrowserHandlers,
|
||||
userHandlers: userHandlers,
|
||||
policyHandlers: policyHandlers,
|
||||
maintenanceHandlers: maintenanceHandlers,
|
||||
pluginHandlers: pluginHandlers,
|
||||
mqHandlers: mqHandlers,
|
||||
serviceAccountHandlers: serviceAccountHandlers,
|
||||
}
|
||||
}
|
||||
|
||||
// SetupRoutes configures all the routes for the admin interface
|
||||
func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser, adminPassword, readOnlyUser, readOnlyPassword string, enableUI bool) {
|
||||
func (h *AdminHandlers) SetupRoutes(r *mux.Router, authRequired bool, adminUser, adminPassword, readOnlyUser, readOnlyPassword string, enableUI bool) {
|
||||
// Health check (no auth required)
|
||||
r.GET("/health", h.HealthCheck)
|
||||
r.HandleFunc("/health", h.HealthCheck).Methods(http.MethodGet)
|
||||
|
||||
// Prometheus metrics endpoint (no auth required)
|
||||
r.GET("/metrics", gin.WrapH(promhttp.HandlerFor(stats.Gather, promhttp.HandlerOpts{})))
|
||||
r.Handle("/metrics", promhttp.HandlerFor(stats.Gather, promhttp.HandlerOpts{})).Methods(http.MethodGet)
|
||||
|
||||
// Favicon route (no auth required) - redirect to static version
|
||||
r.GET("/favicon.ico", func(c *gin.Context) {
|
||||
c.Redirect(http.StatusMovedPermanently, "/static/favicon.ico")
|
||||
})
|
||||
r.HandleFunc("/favicon.ico", func(w http.ResponseWriter, req *http.Request) {
|
||||
http.Redirect(w, req, "/static/favicon.ico", http.StatusMovedPermanently)
|
||||
}).Methods(http.MethodGet)
|
||||
|
||||
// Skip UI routes if UI is not enabled
|
||||
if !enableUI {
|
||||
@@ -71,493 +74,321 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser,
|
||||
|
||||
if authRequired {
|
||||
// Authentication routes (no auth required)
|
||||
r.GET("/login", h.authHandlers.ShowLogin)
|
||||
r.POST("/login", h.authHandlers.HandleLogin(adminUser, adminPassword, readOnlyUser, readOnlyPassword))
|
||||
r.GET("/logout", h.authHandlers.HandleLogout)
|
||||
r.HandleFunc("/login", h.authHandlers.ShowLogin).Methods(http.MethodGet)
|
||||
r.Handle("/login", h.authHandlers.HandleLogin(adminUser, adminPassword, readOnlyUser, readOnlyPassword)).Methods(http.MethodPost)
|
||||
r.HandleFunc("/logout", h.authHandlers.HandleLogout).Methods(http.MethodGet)
|
||||
|
||||
// Protected routes group
|
||||
protected := r.Group("/")
|
||||
protected.Use(dash.RequireAuth())
|
||||
protected := r.NewRoute().Subrouter()
|
||||
protected.Use(dash.RequireAuth(h.sessionStore))
|
||||
h.registerUIRoutes(protected)
|
||||
|
||||
// Main admin interface routes
|
||||
protected.GET("/", h.ShowDashboard)
|
||||
protected.GET("/admin", h.ShowDashboard)
|
||||
|
||||
// Object Store management routes
|
||||
protected.GET("/object-store/buckets", h.ShowS3Buckets)
|
||||
protected.GET("/object-store/buckets/:bucket", h.ShowBucketDetails)
|
||||
protected.GET("/object-store/users", h.userHandlers.ShowObjectStoreUsers)
|
||||
protected.GET("/object-store/policies", h.policyHandlers.ShowPolicies)
|
||||
protected.GET("/object-store/service-accounts", h.serviceAccountHandlers.ShowServiceAccounts)
|
||||
protected.GET("/object-store/s3tables/buckets", h.ShowS3TablesBuckets)
|
||||
protected.GET("/object-store/s3tables/buckets/:bucket/namespaces", h.ShowS3TablesNamespaces)
|
||||
protected.GET("/object-store/s3tables/buckets/:bucket/namespaces/:namespace/tables", h.ShowS3TablesTables)
|
||||
protected.GET("/object-store/s3tables/buckets/:bucket/namespaces/:namespace/tables/:table", h.ShowS3TablesTableDetails)
|
||||
protected.GET("/object-store/iceberg", h.ShowIcebergCatalog)
|
||||
protected.GET("/object-store/iceberg/:catalog/namespaces", h.ShowIcebergNamespaces)
|
||||
protected.GET("/object-store/iceberg/:catalog/namespaces/:namespace/tables", h.ShowIcebergTables)
|
||||
protected.GET("/object-store/iceberg/:catalog/namespaces/:namespace/tables/:table", h.ShowIcebergTableDetails)
|
||||
|
||||
// File browser routes
|
||||
protected.GET("/files", h.fileBrowserHandlers.ShowFileBrowser)
|
||||
|
||||
// Cluster management routes
|
||||
protected.GET("/cluster/masters", h.clusterHandlers.ShowClusterMasters)
|
||||
protected.GET("/cluster/filers", h.clusterHandlers.ShowClusterFilers)
|
||||
protected.GET("/cluster/volume-servers", h.clusterHandlers.ShowClusterVolumeServers)
|
||||
|
||||
// Storage management routes
|
||||
protected.GET("/storage/volumes", h.clusterHandlers.ShowClusterVolumes)
|
||||
protected.GET("/storage/volumes/:id/:server", h.clusterHandlers.ShowVolumeDetails)
|
||||
protected.GET("/storage/collections", h.clusterHandlers.ShowClusterCollections)
|
||||
protected.GET("/storage/collections/:name", h.clusterHandlers.ShowCollectionDetails)
|
||||
protected.GET("/storage/ec-shards", h.clusterHandlers.ShowClusterEcShards)
|
||||
protected.GET("/storage/ec-volumes/:id", h.clusterHandlers.ShowEcVolumeDetails)
|
||||
|
||||
// Message Queue management routes
|
||||
protected.GET("/mq/brokers", h.mqHandlers.ShowBrokers)
|
||||
protected.GET("/mq/topics", h.mqHandlers.ShowTopics)
|
||||
protected.GET("/mq/topics/:namespace/:topic", h.mqHandlers.ShowTopicDetails)
|
||||
|
||||
// Maintenance system routes
|
||||
protected.GET("/maintenance", h.maintenanceHandlers.ShowMaintenanceQueue)
|
||||
protected.GET("/maintenance/workers", h.maintenanceHandlers.ShowMaintenanceWorkers)
|
||||
protected.GET("/maintenance/config", h.maintenanceHandlers.ShowMaintenanceConfig)
|
||||
protected.POST("/maintenance/config", dash.RequireWriteAccess(), h.maintenanceHandlers.UpdateMaintenanceConfig)
|
||||
protected.GET("/maintenance/config/:taskType", h.maintenanceHandlers.ShowTaskConfig)
|
||||
protected.POST("/maintenance/config/:taskType", dash.RequireWriteAccess(), h.maintenanceHandlers.UpdateTaskConfig)
|
||||
protected.GET("/maintenance/tasks/:id", h.maintenanceHandlers.ShowTaskDetail)
|
||||
|
||||
// API routes for AJAX calls
|
||||
api := r.Group("/api")
|
||||
api.Use(dash.RequireAuthAPI()) // Use API-specific auth middleware
|
||||
{
|
||||
api.GET("/cluster/topology", h.clusterHandlers.GetClusterTopology)
|
||||
api.GET("/cluster/masters", h.clusterHandlers.GetMasters)
|
||||
api.GET("/cluster/volumes", h.clusterHandlers.GetVolumeServers)
|
||||
api.GET("/admin", h.adminServer.ShowAdmin) // JSON API for admin data
|
||||
api.GET("/config", h.adminServer.GetConfigInfo) // Configuration information
|
||||
|
||||
// S3 API routes
|
||||
s3Api := api.Group("/s3")
|
||||
{
|
||||
s3Api.GET("/buckets", h.adminServer.ListBucketsAPI)
|
||||
s3Api.POST("/buckets", dash.RequireWriteAccess(), h.adminServer.CreateBucket)
|
||||
s3Api.DELETE("/buckets/:bucket", dash.RequireWriteAccess(), h.adminServer.DeleteBucket)
|
||||
s3Api.GET("/buckets/:bucket", h.adminServer.ShowBucketDetails)
|
||||
s3Api.PUT("/buckets/:bucket/quota", dash.RequireWriteAccess(), h.adminServer.UpdateBucketQuota)
|
||||
s3Api.PUT("/buckets/:bucket/owner", dash.RequireWriteAccess(), h.adminServer.UpdateBucketOwner)
|
||||
}
|
||||
|
||||
// User management API routes
|
||||
usersApi := api.Group("/users")
|
||||
{
|
||||
usersApi.GET("", h.userHandlers.GetUsers)
|
||||
usersApi.POST("", dash.RequireWriteAccess(), h.userHandlers.CreateUser)
|
||||
usersApi.GET("/:username", h.userHandlers.GetUserDetails)
|
||||
usersApi.PUT("/:username", dash.RequireWriteAccess(), h.userHandlers.UpdateUser)
|
||||
usersApi.DELETE("/:username", dash.RequireWriteAccess(), h.userHandlers.DeleteUser)
|
||||
usersApi.POST("/:username/access-keys", dash.RequireWriteAccess(), h.userHandlers.CreateAccessKey)
|
||||
usersApi.DELETE("/:username/access-keys/:accessKeyId", dash.RequireWriteAccess(), h.userHandlers.DeleteAccessKey)
|
||||
usersApi.PUT("/:username/access-keys/:accessKeyId/status", dash.RequireWriteAccess(), h.userHandlers.UpdateAccessKeyStatus)
|
||||
usersApi.GET("/:username/policies", h.userHandlers.GetUserPolicies)
|
||||
usersApi.PUT("/:username/policies", dash.RequireWriteAccess(), h.userHandlers.UpdateUserPolicies)
|
||||
}
|
||||
|
||||
// Service Account management API routes
|
||||
saApi := api.Group("/service-accounts")
|
||||
{
|
||||
saApi.GET("", h.serviceAccountHandlers.GetServiceAccounts)
|
||||
saApi.POST("", dash.RequireWriteAccess(), h.serviceAccountHandlers.CreateServiceAccount)
|
||||
saApi.GET("/:id", h.serviceAccountHandlers.GetServiceAccountDetails)
|
||||
saApi.PUT("/:id", dash.RequireWriteAccess(), h.serviceAccountHandlers.UpdateServiceAccount)
|
||||
saApi.DELETE("/:id", dash.RequireWriteAccess(), h.serviceAccountHandlers.DeleteServiceAccount)
|
||||
}
|
||||
|
||||
// Object Store Policy management API routes
|
||||
objectStorePoliciesApi := api.Group("/object-store/policies")
|
||||
{
|
||||
objectStorePoliciesApi.GET("", h.policyHandlers.GetPolicies)
|
||||
objectStorePoliciesApi.POST("", dash.RequireWriteAccess(), h.policyHandlers.CreatePolicy)
|
||||
objectStorePoliciesApi.GET("/:name", h.policyHandlers.GetPolicy)
|
||||
objectStorePoliciesApi.PUT("/:name", dash.RequireWriteAccess(), h.policyHandlers.UpdatePolicy)
|
||||
objectStorePoliciesApi.DELETE("/:name", dash.RequireWriteAccess(), h.policyHandlers.DeletePolicy)
|
||||
objectStorePoliciesApi.POST("/validate", h.policyHandlers.ValidatePolicy)
|
||||
}
|
||||
|
||||
// S3 Tables API routes
|
||||
s3TablesApi := api.Group("/s3tables")
|
||||
{
|
||||
s3TablesApi.GET("/buckets", h.adminServer.ListS3TablesBucketsAPI)
|
||||
s3TablesApi.POST("/buckets", dash.RequireWriteAccess(), h.adminServer.CreateS3TablesBucket)
|
||||
s3TablesApi.DELETE("/buckets", dash.RequireWriteAccess(), h.adminServer.DeleteS3TablesBucket)
|
||||
s3TablesApi.GET("/namespaces", h.adminServer.ListS3TablesNamespacesAPI)
|
||||
s3TablesApi.POST("/namespaces", dash.RequireWriteAccess(), h.adminServer.CreateS3TablesNamespace)
|
||||
s3TablesApi.DELETE("/namespaces", dash.RequireWriteAccess(), h.adminServer.DeleteS3TablesNamespace)
|
||||
s3TablesApi.GET("/tables", h.adminServer.ListS3TablesTablesAPI)
|
||||
s3TablesApi.POST("/tables", dash.RequireWriteAccess(), h.adminServer.CreateS3TablesTable)
|
||||
s3TablesApi.DELETE("/tables", dash.RequireWriteAccess(), h.adminServer.DeleteS3TablesTable)
|
||||
s3TablesApi.PUT("/bucket-policy", dash.RequireWriteAccess(), h.adminServer.PutS3TablesBucketPolicy)
|
||||
s3TablesApi.GET("/bucket-policy", h.adminServer.GetS3TablesBucketPolicy)
|
||||
s3TablesApi.DELETE("/bucket-policy", dash.RequireWriteAccess(), h.adminServer.DeleteS3TablesBucketPolicy)
|
||||
s3TablesApi.PUT("/table-policy", dash.RequireWriteAccess(), h.adminServer.PutS3TablesTablePolicy)
|
||||
s3TablesApi.GET("/table-policy", h.adminServer.GetS3TablesTablePolicy)
|
||||
s3TablesApi.DELETE("/table-policy", dash.RequireWriteAccess(), h.adminServer.DeleteS3TablesTablePolicy)
|
||||
s3TablesApi.PUT("/tags", dash.RequireWriteAccess(), h.adminServer.TagS3TablesResource)
|
||||
s3TablesApi.GET("/tags", h.adminServer.ListS3TablesTags)
|
||||
s3TablesApi.DELETE("/tags", dash.RequireWriteAccess(), h.adminServer.UntagS3TablesResource)
|
||||
}
|
||||
|
||||
// File management API routes
|
||||
filesApi := api.Group("/files")
|
||||
{
|
||||
filesApi.DELETE("/delete", dash.RequireWriteAccess(), h.fileBrowserHandlers.DeleteFile)
|
||||
filesApi.DELETE("/delete-multiple", dash.RequireWriteAccess(), h.fileBrowserHandlers.DeleteMultipleFiles)
|
||||
filesApi.POST("/create-folder", dash.RequireWriteAccess(), h.fileBrowserHandlers.CreateFolder)
|
||||
filesApi.POST("/upload", dash.RequireWriteAccess(), h.fileBrowserHandlers.UploadFile)
|
||||
filesApi.GET("/download", h.fileBrowserHandlers.DownloadFile)
|
||||
filesApi.GET("/view", h.fileBrowserHandlers.ViewFile)
|
||||
filesApi.GET("/properties", h.fileBrowserHandlers.GetFileProperties)
|
||||
}
|
||||
|
||||
// Volume management API routes
|
||||
volumeApi := api.Group("/volumes")
|
||||
{
|
||||
volumeApi.POST("/:id/:server/vacuum", dash.RequireWriteAccess(), h.clusterHandlers.VacuumVolume)
|
||||
}
|
||||
|
||||
// Maintenance API routes
|
||||
maintenanceApi := api.Group("/maintenance")
|
||||
{
|
||||
maintenanceApi.POST("/scan", dash.RequireWriteAccess(), h.adminServer.TriggerMaintenanceScan)
|
||||
maintenanceApi.GET("/tasks", h.adminServer.GetMaintenanceTasks)
|
||||
maintenanceApi.GET("/tasks/:id", h.adminServer.GetMaintenanceTask)
|
||||
maintenanceApi.GET("/tasks/:id/detail", h.adminServer.GetMaintenanceTaskDetailAPI)
|
||||
maintenanceApi.POST("/tasks/:id/cancel", dash.RequireWriteAccess(), h.adminServer.CancelMaintenanceTask)
|
||||
maintenanceApi.GET("/workers", h.adminServer.GetMaintenanceWorkersAPI)
|
||||
maintenanceApi.GET("/workers/:id", h.adminServer.GetMaintenanceWorker)
|
||||
maintenanceApi.GET("/workers/:id/logs", h.adminServer.GetWorkerLogs)
|
||||
maintenanceApi.GET("/stats", h.adminServer.GetMaintenanceStats)
|
||||
maintenanceApi.GET("/config", h.adminServer.GetMaintenanceConfigAPI)
|
||||
maintenanceApi.PUT("/config", dash.RequireWriteAccess(), h.adminServer.UpdateMaintenanceConfigAPI)
|
||||
}
|
||||
|
||||
// Message Queue API routes
|
||||
mqApi := api.Group("/mq")
|
||||
{
|
||||
mqApi.GET("/topics/:namespace/:topic", h.mqHandlers.GetTopicDetailsAPI)
|
||||
mqApi.POST("/topics/create", dash.RequireWriteAccess(), h.mqHandlers.CreateTopicAPI)
|
||||
mqApi.POST("/topics/retention/update", dash.RequireWriteAccess(), h.mqHandlers.UpdateTopicRetentionAPI)
|
||||
mqApi.POST("/retention/purge", dash.RequireWriteAccess(), h.adminServer.TriggerTopicRetentionPurgeAPI)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// No authentication required - all routes are public
|
||||
r.GET("/", h.ShowDashboard)
|
||||
r.GET("/admin", h.ShowDashboard)
|
||||
|
||||
// Object Store management routes
|
||||
r.GET("/object-store/buckets", h.ShowS3Buckets)
|
||||
r.GET("/object-store/buckets/:bucket", h.ShowBucketDetails)
|
||||
r.GET("/object-store/users", h.userHandlers.ShowObjectStoreUsers)
|
||||
r.GET("/object-store/policies", h.policyHandlers.ShowPolicies)
|
||||
r.GET("/object-store/service-accounts", h.serviceAccountHandlers.ShowServiceAccounts)
|
||||
r.GET("/object-store/s3tables/buckets", h.ShowS3TablesBuckets)
|
||||
r.GET("/object-store/s3tables/buckets/:bucket/namespaces", h.ShowS3TablesNamespaces)
|
||||
r.GET("/object-store/s3tables/buckets/:bucket/namespaces/:namespace/tables", h.ShowS3TablesTables)
|
||||
r.GET("/object-store/s3tables/buckets/:bucket/namespaces/:namespace/tables/:table", h.ShowS3TablesTableDetails)
|
||||
r.GET("/object-store/iceberg", h.ShowIcebergCatalog)
|
||||
r.GET("/object-store/iceberg/:catalog/namespaces", h.ShowIcebergNamespaces)
|
||||
r.GET("/object-store/iceberg/:catalog/namespaces/:namespace/tables", h.ShowIcebergTables)
|
||||
r.GET("/object-store/iceberg/:catalog/namespaces/:namespace/tables/:table", h.ShowIcebergTableDetails)
|
||||
|
||||
// File browser routes
|
||||
r.GET("/files", h.fileBrowserHandlers.ShowFileBrowser)
|
||||
|
||||
// Cluster management routes
|
||||
r.GET("/cluster/masters", h.clusterHandlers.ShowClusterMasters)
|
||||
r.GET("/cluster/filers", h.clusterHandlers.ShowClusterFilers)
|
||||
r.GET("/cluster/volume-servers", h.clusterHandlers.ShowClusterVolumeServers)
|
||||
|
||||
// Storage management routes
|
||||
r.GET("/storage/volumes", h.clusterHandlers.ShowClusterVolumes)
|
||||
r.GET("/storage/volumes/:id/:server", h.clusterHandlers.ShowVolumeDetails)
|
||||
r.GET("/storage/collections", h.clusterHandlers.ShowClusterCollections)
|
||||
r.GET("/storage/collections/:name", h.clusterHandlers.ShowCollectionDetails)
|
||||
r.GET("/storage/ec-shards", h.clusterHandlers.ShowClusterEcShards)
|
||||
r.GET("/storage/ec-volumes/:id", h.clusterHandlers.ShowEcVolumeDetails)
|
||||
|
||||
// Message Queue management routes
|
||||
r.GET("/mq/brokers", h.mqHandlers.ShowBrokers)
|
||||
r.GET("/mq/topics", h.mqHandlers.ShowTopics)
|
||||
r.GET("/mq/topics/:namespace/:topic", h.mqHandlers.ShowTopicDetails)
|
||||
|
||||
// Maintenance system routes
|
||||
r.GET("/maintenance", h.maintenanceHandlers.ShowMaintenanceQueue)
|
||||
r.GET("/maintenance/workers", h.maintenanceHandlers.ShowMaintenanceWorkers)
|
||||
r.GET("/maintenance/config", h.maintenanceHandlers.ShowMaintenanceConfig)
|
||||
r.POST("/maintenance/config", h.maintenanceHandlers.UpdateMaintenanceConfig)
|
||||
r.GET("/maintenance/config/:taskType", h.maintenanceHandlers.ShowTaskConfig)
|
||||
r.POST("/maintenance/config/:taskType", h.maintenanceHandlers.UpdateTaskConfig)
|
||||
r.GET("/maintenance/tasks/:id", h.maintenanceHandlers.ShowTaskDetail)
|
||||
|
||||
// API routes for AJAX calls
|
||||
api := r.Group("/api")
|
||||
{
|
||||
api.GET("/cluster/topology", h.clusterHandlers.GetClusterTopology)
|
||||
api.GET("/cluster/masters", h.clusterHandlers.GetMasters)
|
||||
api.GET("/cluster/volumes", h.clusterHandlers.GetVolumeServers)
|
||||
api.GET("/admin", h.adminServer.ShowAdmin) // JSON API for admin data
|
||||
api.GET("/config", h.adminServer.GetConfigInfo) // Configuration information
|
||||
|
||||
// S3 API routes
|
||||
s3Api := api.Group("/s3")
|
||||
{
|
||||
s3Api.GET("/buckets", h.adminServer.ListBucketsAPI)
|
||||
s3Api.POST("/buckets", h.adminServer.CreateBucket)
|
||||
s3Api.DELETE("/buckets/:bucket", h.adminServer.DeleteBucket)
|
||||
s3Api.GET("/buckets/:bucket", h.adminServer.ShowBucketDetails)
|
||||
s3Api.PUT("/buckets/:bucket/quota", h.adminServer.UpdateBucketQuota)
|
||||
s3Api.PUT("/buckets/:bucket/owner", h.adminServer.UpdateBucketOwner)
|
||||
}
|
||||
|
||||
// User management API routes
|
||||
usersApi := api.Group("/users")
|
||||
{
|
||||
usersApi.GET("", h.userHandlers.GetUsers)
|
||||
usersApi.POST("", h.userHandlers.CreateUser)
|
||||
usersApi.GET("/:username", h.userHandlers.GetUserDetails)
|
||||
usersApi.PUT("/:username", h.userHandlers.UpdateUser)
|
||||
usersApi.DELETE("/:username", h.userHandlers.DeleteUser)
|
||||
usersApi.POST("/:username/access-keys", h.userHandlers.CreateAccessKey)
|
||||
usersApi.DELETE("/:username/access-keys/:accessKeyId", h.userHandlers.DeleteAccessKey)
|
||||
usersApi.PUT("/:username/access-keys/:accessKeyId/status", h.userHandlers.UpdateAccessKeyStatus)
|
||||
usersApi.GET("/:username/policies", h.userHandlers.GetUserPolicies)
|
||||
usersApi.PUT("/:username/policies", h.userHandlers.UpdateUserPolicies)
|
||||
}
|
||||
|
||||
// Service Account management API routes
|
||||
saApi := api.Group("/service-accounts")
|
||||
{
|
||||
saApi.GET("", h.serviceAccountHandlers.GetServiceAccounts)
|
||||
saApi.POST("", h.serviceAccountHandlers.CreateServiceAccount)
|
||||
saApi.GET("/:id", h.serviceAccountHandlers.GetServiceAccountDetails)
|
||||
saApi.PUT("/:id", h.serviceAccountHandlers.UpdateServiceAccount)
|
||||
saApi.DELETE("/:id", h.serviceAccountHandlers.DeleteServiceAccount)
|
||||
}
|
||||
|
||||
// Object Store Policy management API routes
|
||||
objectStorePoliciesApi := api.Group("/object-store/policies")
|
||||
{
|
||||
objectStorePoliciesApi.GET("", h.policyHandlers.GetPolicies)
|
||||
objectStorePoliciesApi.POST("", h.policyHandlers.CreatePolicy)
|
||||
objectStorePoliciesApi.GET("/:name", h.policyHandlers.GetPolicy)
|
||||
objectStorePoliciesApi.PUT("/:name", h.policyHandlers.UpdatePolicy)
|
||||
objectStorePoliciesApi.DELETE("/:name", h.policyHandlers.DeletePolicy)
|
||||
objectStorePoliciesApi.POST("/validate", h.policyHandlers.ValidatePolicy)
|
||||
}
|
||||
|
||||
// S3 Tables API routes
|
||||
s3TablesApi := api.Group("/s3tables")
|
||||
{
|
||||
s3TablesApi.GET("/buckets", h.adminServer.ListS3TablesBucketsAPI)
|
||||
s3TablesApi.POST("/buckets", h.adminServer.CreateS3TablesBucket)
|
||||
s3TablesApi.DELETE("/buckets", h.adminServer.DeleteS3TablesBucket)
|
||||
s3TablesApi.GET("/namespaces", h.adminServer.ListS3TablesNamespacesAPI)
|
||||
s3TablesApi.POST("/namespaces", h.adminServer.CreateS3TablesNamespace)
|
||||
s3TablesApi.DELETE("/namespaces", h.adminServer.DeleteS3TablesNamespace)
|
||||
s3TablesApi.GET("/tables", h.adminServer.ListS3TablesTablesAPI)
|
||||
s3TablesApi.POST("/tables", h.adminServer.CreateS3TablesTable)
|
||||
s3TablesApi.DELETE("/tables", h.adminServer.DeleteS3TablesTable)
|
||||
s3TablesApi.PUT("/bucket-policy", h.adminServer.PutS3TablesBucketPolicy)
|
||||
s3TablesApi.GET("/bucket-policy", h.adminServer.GetS3TablesBucketPolicy)
|
||||
s3TablesApi.DELETE("/bucket-policy", h.adminServer.DeleteS3TablesBucketPolicy)
|
||||
s3TablesApi.PUT("/table-policy", h.adminServer.PutS3TablesTablePolicy)
|
||||
s3TablesApi.GET("/table-policy", h.adminServer.GetS3TablesTablePolicy)
|
||||
s3TablesApi.DELETE("/table-policy", h.adminServer.DeleteS3TablesTablePolicy)
|
||||
s3TablesApi.PUT("/tags", h.adminServer.TagS3TablesResource)
|
||||
s3TablesApi.GET("/tags", h.adminServer.ListS3TablesTags)
|
||||
s3TablesApi.DELETE("/tags", h.adminServer.UntagS3TablesResource)
|
||||
}
|
||||
|
||||
// File management API routes
|
||||
filesApi := api.Group("/files")
|
||||
{
|
||||
filesApi.DELETE("/delete", h.fileBrowserHandlers.DeleteFile)
|
||||
filesApi.DELETE("/delete-multiple", h.fileBrowserHandlers.DeleteMultipleFiles)
|
||||
filesApi.POST("/create-folder", h.fileBrowserHandlers.CreateFolder)
|
||||
filesApi.POST("/upload", h.fileBrowserHandlers.UploadFile)
|
||||
filesApi.GET("/download", h.fileBrowserHandlers.DownloadFile)
|
||||
filesApi.GET("/view", h.fileBrowserHandlers.ViewFile)
|
||||
filesApi.GET("/properties", h.fileBrowserHandlers.GetFileProperties)
|
||||
}
|
||||
|
||||
// Volume management API routes
|
||||
volumeApi := api.Group("/volumes")
|
||||
{
|
||||
volumeApi.POST("/:id/:server/vacuum", h.clusterHandlers.VacuumVolume)
|
||||
}
|
||||
|
||||
// Maintenance API routes
|
||||
maintenanceApi := api.Group("/maintenance")
|
||||
{
|
||||
maintenanceApi.POST("/scan", h.adminServer.TriggerMaintenanceScan)
|
||||
maintenanceApi.GET("/tasks", h.adminServer.GetMaintenanceTasks)
|
||||
maintenanceApi.GET("/tasks/:id", h.adminServer.GetMaintenanceTask)
|
||||
maintenanceApi.GET("/tasks/:id/detail", h.adminServer.GetMaintenanceTaskDetailAPI)
|
||||
maintenanceApi.POST("/tasks/:id/cancel", h.adminServer.CancelMaintenanceTask)
|
||||
maintenanceApi.GET("/workers", h.adminServer.GetMaintenanceWorkersAPI)
|
||||
maintenanceApi.GET("/workers/:id", h.adminServer.GetMaintenanceWorker)
|
||||
maintenanceApi.GET("/workers/:id/logs", h.adminServer.GetWorkerLogs)
|
||||
maintenanceApi.GET("/stats", h.adminServer.GetMaintenanceStats)
|
||||
maintenanceApi.GET("/config", h.adminServer.GetMaintenanceConfigAPI)
|
||||
maintenanceApi.PUT("/config", h.adminServer.UpdateMaintenanceConfigAPI)
|
||||
}
|
||||
|
||||
// Message Queue API routes
|
||||
mqApi := api.Group("/mq")
|
||||
{
|
||||
mqApi.GET("/topics/:namespace/:topic", h.mqHandlers.GetTopicDetailsAPI)
|
||||
mqApi.POST("/topics/create", h.mqHandlers.CreateTopicAPI)
|
||||
mqApi.POST("/topics/retention/update", h.mqHandlers.UpdateTopicRetentionAPI)
|
||||
mqApi.POST("/retention/purge", h.adminServer.TriggerTopicRetentionPurgeAPI)
|
||||
}
|
||||
}
|
||||
api := r.PathPrefix("/api").Subrouter()
|
||||
api.Use(dash.RequireAuthAPI(h.sessionStore))
|
||||
h.registerAPIRoutes(api, true)
|
||||
return
|
||||
}
|
||||
|
||||
// No authentication required - all routes are public
|
||||
h.registerUIRoutes(r)
|
||||
api := r.PathPrefix("/api").Subrouter()
|
||||
h.registerAPIRoutes(api, false)
|
||||
}
|
||||
|
||||
func (h *AdminHandlers) registerUIRoutes(r *mux.Router) {
|
||||
// Main admin interface routes
|
||||
r.HandleFunc("/", h.ShowDashboard).Methods(http.MethodGet)
|
||||
r.HandleFunc("/admin", h.ShowDashboard).Methods(http.MethodGet)
|
||||
|
||||
// Object Store management routes
|
||||
r.HandleFunc("/object-store/buckets", h.ShowS3Buckets).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/buckets/{bucket}", h.ShowBucketDetails).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/users", h.userHandlers.ShowObjectStoreUsers).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/policies", h.policyHandlers.ShowPolicies).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/service-accounts", h.serviceAccountHandlers.ShowServiceAccounts).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/s3tables/buckets", h.ShowS3TablesBuckets).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/s3tables/buckets/{bucket}/namespaces", h.ShowS3TablesNamespaces).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/s3tables/buckets/{bucket}/namespaces/{namespace}/tables", h.ShowS3TablesTables).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/s3tables/buckets/{bucket}/namespaces/{namespace}/tables/{table}", h.ShowS3TablesTableDetails).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/iceberg", h.ShowIcebergCatalog).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/iceberg/{catalog}/namespaces", h.ShowIcebergNamespaces).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/iceberg/{catalog}/namespaces/{namespace}/tables", h.ShowIcebergTables).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/iceberg/{catalog}/namespaces/{namespace}/tables/{table}", h.ShowIcebergTableDetails).Methods(http.MethodGet)
|
||||
|
||||
// File browser routes
|
||||
r.HandleFunc("/files", h.fileBrowserHandlers.ShowFileBrowser).Methods(http.MethodGet)
|
||||
|
||||
// Cluster management routes
|
||||
r.HandleFunc("/cluster/masters", h.clusterHandlers.ShowClusterMasters).Methods(http.MethodGet)
|
||||
r.HandleFunc("/cluster/filers", h.clusterHandlers.ShowClusterFilers).Methods(http.MethodGet)
|
||||
r.HandleFunc("/cluster/volume-servers", h.clusterHandlers.ShowClusterVolumeServers).Methods(http.MethodGet)
|
||||
|
||||
// Storage management routes
|
||||
r.HandleFunc("/storage/volumes", h.clusterHandlers.ShowClusterVolumes).Methods(http.MethodGet)
|
||||
r.HandleFunc("/storage/volumes/{id}/{server}", h.clusterHandlers.ShowVolumeDetails).Methods(http.MethodGet)
|
||||
r.HandleFunc("/storage/collections", h.clusterHandlers.ShowClusterCollections).Methods(http.MethodGet)
|
||||
r.HandleFunc("/storage/collections/{name}", h.clusterHandlers.ShowCollectionDetails).Methods(http.MethodGet)
|
||||
r.HandleFunc("/storage/ec-shards", h.clusterHandlers.ShowClusterEcShards).Methods(http.MethodGet)
|
||||
r.HandleFunc("/storage/ec-volumes/{id}", h.clusterHandlers.ShowEcVolumeDetails).Methods(http.MethodGet)
|
||||
|
||||
// Message Queue management routes
|
||||
r.HandleFunc("/mq/brokers", h.mqHandlers.ShowBrokers).Methods(http.MethodGet)
|
||||
r.HandleFunc("/mq/topics", h.mqHandlers.ShowTopics).Methods(http.MethodGet)
|
||||
r.HandleFunc("/mq/topics/{namespace}/{topic}", h.mqHandlers.ShowTopicDetails).Methods(http.MethodGet)
|
||||
|
||||
// Plugin pages
|
||||
r.HandleFunc("/plugin", h.pluginHandlers.ShowPlugin).Methods(http.MethodGet)
|
||||
r.HandleFunc("/plugin/configuration", h.pluginHandlers.ShowPluginConfiguration).Methods(http.MethodGet)
|
||||
r.HandleFunc("/plugin/queue", h.pluginHandlers.ShowPluginQueue).Methods(http.MethodGet)
|
||||
r.HandleFunc("/plugin/detection", h.pluginHandlers.ShowPluginDetection).Methods(http.MethodGet)
|
||||
r.HandleFunc("/plugin/execution", h.pluginHandlers.ShowPluginExecution).Methods(http.MethodGet)
|
||||
r.HandleFunc("/plugin/monitoring", h.pluginHandlers.ShowPluginMonitoring).Methods(http.MethodGet)
|
||||
}
|
||||
|
||||
func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
|
||||
wrapWrite := func(handler http.HandlerFunc) http.Handler {
|
||||
if !enforceWrite {
|
||||
return handler
|
||||
}
|
||||
return dash.RequireWriteAccess()(handler)
|
||||
}
|
||||
|
||||
api.HandleFunc("/cluster/topology", h.clusterHandlers.GetClusterTopology).Methods(http.MethodGet)
|
||||
api.HandleFunc("/cluster/masters", h.clusterHandlers.GetMasters).Methods(http.MethodGet)
|
||||
api.HandleFunc("/cluster/volumes", h.clusterHandlers.GetVolumeServers).Methods(http.MethodGet)
|
||||
api.HandleFunc("/admin", h.adminServer.ShowAdmin).Methods(http.MethodGet)
|
||||
api.HandleFunc("/config", h.adminServer.GetConfigInfo).Methods(http.MethodGet)
|
||||
|
||||
s3Api := api.PathPrefix("/s3").Subrouter()
|
||||
s3Api.HandleFunc("/buckets", h.adminServer.ListBucketsAPI).Methods(http.MethodGet)
|
||||
s3Api.Handle("/buckets", wrapWrite(h.adminServer.CreateBucket)).Methods(http.MethodPost)
|
||||
s3Api.Handle("/buckets/{bucket}", wrapWrite(h.adminServer.DeleteBucket)).Methods(http.MethodDelete)
|
||||
s3Api.HandleFunc("/buckets/{bucket}", h.adminServer.ShowBucketDetails).Methods(http.MethodGet)
|
||||
s3Api.Handle("/buckets/{bucket}/quota", wrapWrite(h.adminServer.UpdateBucketQuota)).Methods(http.MethodPut)
|
||||
s3Api.Handle("/buckets/{bucket}/owner", wrapWrite(h.adminServer.UpdateBucketOwner)).Methods(http.MethodPut)
|
||||
|
||||
usersApi := api.PathPrefix("/users").Subrouter()
|
||||
usersApi.HandleFunc("", h.userHandlers.GetUsers).Methods(http.MethodGet)
|
||||
usersApi.Handle("", wrapWrite(h.userHandlers.CreateUser)).Methods(http.MethodPost)
|
||||
usersApi.HandleFunc("/{username}", h.userHandlers.GetUserDetails).Methods(http.MethodGet)
|
||||
usersApi.Handle("/{username}", wrapWrite(h.userHandlers.UpdateUser)).Methods(http.MethodPut)
|
||||
usersApi.Handle("/{username}", wrapWrite(h.userHandlers.DeleteUser)).Methods(http.MethodDelete)
|
||||
usersApi.Handle("/{username}/access-keys", wrapWrite(h.userHandlers.CreateAccessKey)).Methods(http.MethodPost)
|
||||
usersApi.Handle("/{username}/access-keys/{accessKeyId}", wrapWrite(h.userHandlers.DeleteAccessKey)).Methods(http.MethodDelete)
|
||||
usersApi.Handle("/{username}/access-keys/{accessKeyId}/status", wrapWrite(h.userHandlers.UpdateAccessKeyStatus)).Methods(http.MethodPut)
|
||||
usersApi.HandleFunc("/{username}/policies", h.userHandlers.GetUserPolicies).Methods(http.MethodGet)
|
||||
usersApi.Handle("/{username}/policies", wrapWrite(h.userHandlers.UpdateUserPolicies)).Methods(http.MethodPut)
|
||||
|
||||
saApi := api.PathPrefix("/service-accounts").Subrouter()
|
||||
saApi.HandleFunc("", h.serviceAccountHandlers.GetServiceAccounts).Methods(http.MethodGet)
|
||||
saApi.Handle("", wrapWrite(h.serviceAccountHandlers.CreateServiceAccount)).Methods(http.MethodPost)
|
||||
saApi.HandleFunc("/{id}", h.serviceAccountHandlers.GetServiceAccountDetails).Methods(http.MethodGet)
|
||||
saApi.Handle("/{id}", wrapWrite(h.serviceAccountHandlers.UpdateServiceAccount)).Methods(http.MethodPut)
|
||||
saApi.Handle("/{id}", wrapWrite(h.serviceAccountHandlers.DeleteServiceAccount)).Methods(http.MethodDelete)
|
||||
|
||||
policyApi := api.PathPrefix("/object-store/policies").Subrouter()
|
||||
policyApi.HandleFunc("", h.policyHandlers.GetPolicies).Methods(http.MethodGet)
|
||||
policyApi.Handle("", wrapWrite(h.policyHandlers.CreatePolicy)).Methods(http.MethodPost)
|
||||
policyApi.HandleFunc("/{name}", h.policyHandlers.GetPolicy).Methods(http.MethodGet)
|
||||
policyApi.Handle("/{name}", wrapWrite(h.policyHandlers.UpdatePolicy)).Methods(http.MethodPut)
|
||||
policyApi.Handle("/{name}", wrapWrite(h.policyHandlers.DeletePolicy)).Methods(http.MethodDelete)
|
||||
policyApi.HandleFunc("/validate", h.policyHandlers.ValidatePolicy).Methods(http.MethodPost)
|
||||
|
||||
s3TablesApi := api.PathPrefix("/s3tables").Subrouter()
|
||||
s3TablesApi.HandleFunc("/buckets", h.adminServer.ListS3TablesBucketsAPI).Methods(http.MethodGet)
|
||||
s3TablesApi.Handle("/buckets", wrapWrite(h.adminServer.CreateS3TablesBucket)).Methods(http.MethodPost)
|
||||
s3TablesApi.Handle("/buckets", wrapWrite(h.adminServer.DeleteS3TablesBucket)).Methods(http.MethodDelete)
|
||||
s3TablesApi.HandleFunc("/namespaces", h.adminServer.ListS3TablesNamespacesAPI).Methods(http.MethodGet)
|
||||
s3TablesApi.Handle("/namespaces", wrapWrite(h.adminServer.CreateS3TablesNamespace)).Methods(http.MethodPost)
|
||||
s3TablesApi.Handle("/namespaces", wrapWrite(h.adminServer.DeleteS3TablesNamespace)).Methods(http.MethodDelete)
|
||||
s3TablesApi.HandleFunc("/tables", h.adminServer.ListS3TablesTablesAPI).Methods(http.MethodGet)
|
||||
s3TablesApi.Handle("/tables", wrapWrite(h.adminServer.CreateS3TablesTable)).Methods(http.MethodPost)
|
||||
s3TablesApi.Handle("/tables", wrapWrite(h.adminServer.DeleteS3TablesTable)).Methods(http.MethodDelete)
|
||||
s3TablesApi.Handle("/bucket-policy", wrapWrite(h.adminServer.PutS3TablesBucketPolicy)).Methods(http.MethodPut)
|
||||
s3TablesApi.HandleFunc("/bucket-policy", h.adminServer.GetS3TablesBucketPolicy).Methods(http.MethodGet)
|
||||
s3TablesApi.Handle("/bucket-policy", wrapWrite(h.adminServer.DeleteS3TablesBucketPolicy)).Methods(http.MethodDelete)
|
||||
s3TablesApi.Handle("/table-policy", wrapWrite(h.adminServer.PutS3TablesTablePolicy)).Methods(http.MethodPut)
|
||||
s3TablesApi.HandleFunc("/table-policy", h.adminServer.GetS3TablesTablePolicy).Methods(http.MethodGet)
|
||||
s3TablesApi.Handle("/table-policy", wrapWrite(h.adminServer.DeleteS3TablesTablePolicy)).Methods(http.MethodDelete)
|
||||
s3TablesApi.Handle("/tags", wrapWrite(h.adminServer.TagS3TablesResource)).Methods(http.MethodPut)
|
||||
s3TablesApi.HandleFunc("/tags", h.adminServer.ListS3TablesTags).Methods(http.MethodGet)
|
||||
s3TablesApi.Handle("/tags", wrapWrite(h.adminServer.UntagS3TablesResource)).Methods(http.MethodDelete)
|
||||
|
||||
filesApi := api.PathPrefix("/files").Subrouter()
|
||||
filesApi.Handle("/delete", wrapWrite(h.fileBrowserHandlers.DeleteFile)).Methods(http.MethodDelete)
|
||||
filesApi.Handle("/delete-multiple", wrapWrite(h.fileBrowserHandlers.DeleteMultipleFiles)).Methods(http.MethodDelete)
|
||||
filesApi.Handle("/create-folder", wrapWrite(h.fileBrowserHandlers.CreateFolder)).Methods(http.MethodPost)
|
||||
filesApi.Handle("/upload", wrapWrite(h.fileBrowserHandlers.UploadFile)).Methods(http.MethodPost)
|
||||
filesApi.HandleFunc("/download", h.fileBrowserHandlers.DownloadFile).Methods(http.MethodGet)
|
||||
filesApi.HandleFunc("/view", h.fileBrowserHandlers.ViewFile).Methods(http.MethodGet)
|
||||
filesApi.HandleFunc("/properties", h.fileBrowserHandlers.GetFileProperties).Methods(http.MethodGet)
|
||||
|
||||
volumeApi := api.PathPrefix("/volumes").Subrouter()
|
||||
volumeApi.Handle("/{id}/{server}/vacuum", wrapWrite(h.clusterHandlers.VacuumVolume)).Methods(http.MethodPost)
|
||||
|
||||
pluginApi := api.PathPrefix("/plugin").Subrouter()
|
||||
pluginApi.HandleFunc("/status", h.adminServer.GetPluginStatusAPI).Methods(http.MethodGet)
|
||||
pluginApi.HandleFunc("/workers", h.adminServer.GetPluginWorkersAPI).Methods(http.MethodGet)
|
||||
pluginApi.HandleFunc("/job-types", h.adminServer.GetPluginJobTypesAPI).Methods(http.MethodGet)
|
||||
pluginApi.HandleFunc("/jobs", h.adminServer.GetPluginJobsAPI).Methods(http.MethodGet)
|
||||
pluginApi.HandleFunc("/jobs/{jobId}", h.adminServer.GetPluginJobAPI).Methods(http.MethodGet)
|
||||
pluginApi.HandleFunc("/jobs/{jobId}/detail", h.adminServer.GetPluginJobDetailAPI).Methods(http.MethodGet)
|
||||
pluginApi.HandleFunc("/activities", h.adminServer.GetPluginActivitiesAPI).Methods(http.MethodGet)
|
||||
pluginApi.HandleFunc("/scheduler-states", h.adminServer.GetPluginSchedulerStatesAPI).Methods(http.MethodGet)
|
||||
pluginApi.HandleFunc("/job-types/{jobType}/descriptor", h.adminServer.GetPluginJobTypeDescriptorAPI).Methods(http.MethodGet)
|
||||
pluginApi.HandleFunc("/job-types/{jobType}/schema", h.adminServer.RequestPluginJobTypeSchemaAPI).Methods(http.MethodPost)
|
||||
pluginApi.HandleFunc("/job-types/{jobType}/config", h.adminServer.GetPluginJobTypeConfigAPI).Methods(http.MethodGet)
|
||||
pluginApi.Handle("/job-types/{jobType}/config", wrapWrite(h.adminServer.UpdatePluginJobTypeConfigAPI)).Methods(http.MethodPut)
|
||||
pluginApi.HandleFunc("/job-types/{jobType}/runs", h.adminServer.GetPluginRunHistoryAPI).Methods(http.MethodGet)
|
||||
pluginApi.Handle("/job-types/{jobType}/detect", wrapWrite(h.adminServer.TriggerPluginDetectionAPI)).Methods(http.MethodPost)
|
||||
pluginApi.Handle("/job-types/{jobType}/run", wrapWrite(h.adminServer.RunPluginJobTypeAPI)).Methods(http.MethodPost)
|
||||
pluginApi.Handle("/jobs/execute", wrapWrite(h.adminServer.ExecutePluginJobAPI)).Methods(http.MethodPost)
|
||||
|
||||
mqApi := api.PathPrefix("/mq").Subrouter()
|
||||
mqApi.HandleFunc("/topics/{namespace}/{topic}", h.mqHandlers.GetTopicDetailsAPI).Methods(http.MethodGet)
|
||||
mqApi.Handle("/topics/create", wrapWrite(h.mqHandlers.CreateTopicAPI)).Methods(http.MethodPost)
|
||||
mqApi.Handle("/topics/retention/update", wrapWrite(h.mqHandlers.UpdateTopicRetentionAPI)).Methods(http.MethodPost)
|
||||
mqApi.Handle("/retention/purge", wrapWrite(h.adminServer.TriggerTopicRetentionPurgeAPI)).Methods(http.MethodPost)
|
||||
}
|
||||
|
||||
// HealthCheck returns the health status of the admin interface
|
||||
func (h *AdminHandlers) HealthCheck(c *gin.Context) {
|
||||
c.JSON(200, gin.H{"health": "ok"})
|
||||
func (h *AdminHandlers) HealthCheck(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]string{"health": "ok"})
|
||||
}
|
||||
|
||||
// ShowDashboard renders the main admin dashboard
|
||||
func (h *AdminHandlers) ShowDashboard(c *gin.Context) {
|
||||
func (h *AdminHandlers) ShowDashboard(w http.ResponseWriter, r *http.Request) {
|
||||
// Get admin data from the server
|
||||
adminData := h.getAdminData(c)
|
||||
adminData := h.getAdminData(r)
|
||||
username := h.getUsername(r)
|
||||
|
||||
// Render HTML template
|
||||
c.Header("Content-Type", "text/html")
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
adminComponent := app.Admin(adminData)
|
||||
layoutComponent := layout.Layout(c, adminComponent)
|
||||
err := layoutComponent.Render(c.Request.Context(), c.Writer)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, adminComponent)
|
||||
if err := layoutComponent.Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// ShowS3Buckets renders the Object Store buckets management page
|
||||
func (h *AdminHandlers) ShowS3Buckets(c *gin.Context) {
|
||||
func (h *AdminHandlers) ShowS3Buckets(w http.ResponseWriter, r *http.Request) {
|
||||
// Get Object Store buckets data from the server
|
||||
s3Data := h.getS3BucketsData(c)
|
||||
s3Data := h.getS3BucketsData(r)
|
||||
username := h.getUsername(r)
|
||||
|
||||
// Render HTML template
|
||||
c.Header("Content-Type", "text/html")
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
s3Component := app.S3Buckets(s3Data)
|
||||
layoutComponent := layout.Layout(c, s3Component)
|
||||
err := layoutComponent.Render(c.Request.Context(), c.Writer)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, s3Component)
|
||||
if err := layoutComponent.Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// ShowS3TablesBuckets renders the S3 Tables buckets page
|
||||
func (h *AdminHandlers) ShowS3TablesBuckets(c *gin.Context) {
|
||||
username := h.getUsername(c)
|
||||
func (h *AdminHandlers) ShowS3TablesBuckets(w http.ResponseWriter, r *http.Request) {
|
||||
username := h.getUsername(r)
|
||||
|
||||
data, err := h.adminServer.GetS3TablesBucketsData(c.Request.Context())
|
||||
data, err := h.adminServer.GetS3TablesBucketsData(r.Context())
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get S3 Tables buckets: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get S3 Tables buckets: "+err.Error())
|
||||
return
|
||||
}
|
||||
data.Username = username
|
||||
|
||||
c.Header("Content-Type", "text/html")
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
component := app.S3TablesBuckets(data)
|
||||
layoutComponent := layout.Layout(c, component)
|
||||
if err := layoutComponent.Render(c.Request.Context(), c.Writer); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, component)
|
||||
if err := layoutComponent.Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// ShowS3TablesNamespaces renders namespaces for a table bucket
|
||||
func (h *AdminHandlers) ShowS3TablesNamespaces(c *gin.Context) {
|
||||
username := h.getUsername(c)
|
||||
func (h *AdminHandlers) ShowS3TablesNamespaces(w http.ResponseWriter, r *http.Request) {
|
||||
username := h.getUsername(r)
|
||||
|
||||
bucketName := c.Param("bucket")
|
||||
bucketName := mux.Vars(r)["bucket"]
|
||||
arn, err := buildS3TablesBucketArn(bucketName)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
writeJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
data, err := h.adminServer.GetS3TablesNamespacesData(c.Request.Context(), arn)
|
||||
data, err := h.adminServer.GetS3TablesNamespacesData(r.Context(), arn)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get S3 Tables namespaces: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get S3 Tables namespaces: "+err.Error())
|
||||
return
|
||||
}
|
||||
data.Username = username
|
||||
|
||||
c.Header("Content-Type", "text/html")
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
component := app.S3TablesNamespaces(data)
|
||||
layoutComponent := layout.Layout(c, component)
|
||||
if err := layoutComponent.Render(c.Request.Context(), c.Writer); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, component)
|
||||
if err := layoutComponent.Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// ShowS3TablesTables renders tables for a namespace
|
||||
func (h *AdminHandlers) ShowS3TablesTables(c *gin.Context) {
|
||||
username := h.getUsername(c)
|
||||
func (h *AdminHandlers) ShowS3TablesTables(w http.ResponseWriter, r *http.Request) {
|
||||
username := h.getUsername(r)
|
||||
|
||||
bucketName := c.Param("bucket")
|
||||
namespace := c.Param("namespace")
|
||||
bucketName := mux.Vars(r)["bucket"]
|
||||
namespace := mux.Vars(r)["namespace"]
|
||||
arn, err := buildS3TablesBucketArn(bucketName)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
writeJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
data, err := h.adminServer.GetS3TablesTablesData(c.Request.Context(), arn, namespace)
|
||||
data, err := h.adminServer.GetS3TablesTablesData(r.Context(), arn, namespace)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get S3 Tables tables: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get S3 Tables tables: "+err.Error())
|
||||
return
|
||||
}
|
||||
data.Username = username
|
||||
|
||||
c.Header("Content-Type", "text/html")
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
component := app.S3TablesTables(data)
|
||||
layoutComponent := layout.Layout(c, component)
|
||||
if err := layoutComponent.Render(c.Request.Context(), c.Writer); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, component)
|
||||
if err := layoutComponent.Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// ShowS3TablesTableDetails renders Iceberg table metadata and snapshot details on the merged S3 Tables path.
|
||||
func (h *AdminHandlers) ShowS3TablesTableDetails(c *gin.Context) {
|
||||
bucketName := c.Param("bucket")
|
||||
namespace := c.Param("namespace")
|
||||
tableName := c.Param("table")
|
||||
func (h *AdminHandlers) ShowS3TablesTableDetails(w http.ResponseWriter, r *http.Request) {
|
||||
bucketName := mux.Vars(r)["bucket"]
|
||||
namespace := mux.Vars(r)["namespace"]
|
||||
tableName := mux.Vars(r)["table"]
|
||||
arn, err := buildS3TablesBucketArn(bucketName)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
writeJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
data, err := h.adminServer.GetIcebergTableDetailsData(c.Request.Context(), bucketName, arn, namespace, tableName)
|
||||
username := h.getUsername(r)
|
||||
data, err := h.adminServer.GetIcebergTableDetailsData(r.Context(), bucketName, arn, namespace, tableName)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get table details: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get table details: "+err.Error())
|
||||
return
|
||||
}
|
||||
data.Username = h.getUsername(c)
|
||||
data.Username = username
|
||||
|
||||
c.Header("Content-Type", "text/html")
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
component := app.IcebergTableDetails(data)
|
||||
layoutComponent := layout.Layout(c, component)
|
||||
if err := layoutComponent.Render(c.Request.Context(), c.Writer); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, component)
|
||||
if err := layoutComponent.Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -566,8 +397,8 @@ func buildS3TablesBucketArn(bucketName string) (string, error) {
|
||||
}
|
||||
|
||||
// getUsername returns the username from context, defaulting to "admin" if not set
|
||||
func (h *AdminHandlers) getUsername(c *gin.Context) string {
|
||||
username := c.GetString("username")
|
||||
func (h *AdminHandlers) getUsername(r *http.Request) string {
|
||||
username := dash.UsernameFromContext(r.Context())
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
@@ -575,45 +406,45 @@ func (h *AdminHandlers) getUsername(c *gin.Context) string {
|
||||
}
|
||||
|
||||
// ShowIcebergCatalog redirects legacy Iceberg catalog URL to the merged S3 Tables buckets page.
|
||||
func (h *AdminHandlers) ShowIcebergCatalog(c *gin.Context) {
|
||||
c.Redirect(http.StatusMovedPermanently, "/object-store/s3tables/buckets")
|
||||
func (h *AdminHandlers) ShowIcebergCatalog(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, "/object-store/s3tables/buckets", http.StatusMovedPermanently)
|
||||
}
|
||||
|
||||
// ShowIcebergNamespaces redirects legacy Iceberg namespaces URL to the merged S3 Tables namespaces page.
|
||||
func (h *AdminHandlers) ShowIcebergNamespaces(c *gin.Context) {
|
||||
catalogName := c.Param("catalog")
|
||||
c.Redirect(http.StatusMovedPermanently, "/object-store/s3tables/buckets/"+url.PathEscape(catalogName)+"/namespaces")
|
||||
func (h *AdminHandlers) ShowIcebergNamespaces(w http.ResponseWriter, r *http.Request) {
|
||||
catalogName := mux.Vars(r)["catalog"]
|
||||
http.Redirect(w, r, "/object-store/s3tables/buckets/"+url.PathEscape(catalogName)+"/namespaces", http.StatusMovedPermanently)
|
||||
}
|
||||
|
||||
// ShowIcebergTables redirects legacy Iceberg tables URL to the merged S3 Tables tables page.
|
||||
func (h *AdminHandlers) ShowIcebergTables(c *gin.Context) {
|
||||
catalogName := c.Param("catalog")
|
||||
namespace := c.Param("namespace")
|
||||
c.Redirect(http.StatusMovedPermanently, "/object-store/s3tables/buckets/"+url.PathEscape(catalogName)+"/namespaces/"+url.PathEscape(namespace)+"/tables")
|
||||
func (h *AdminHandlers) ShowIcebergTables(w http.ResponseWriter, r *http.Request) {
|
||||
catalogName := mux.Vars(r)["catalog"]
|
||||
namespace := mux.Vars(r)["namespace"]
|
||||
http.Redirect(w, r, "/object-store/s3tables/buckets/"+url.PathEscape(catalogName)+"/namespaces/"+url.PathEscape(namespace)+"/tables", http.StatusMovedPermanently)
|
||||
}
|
||||
|
||||
// ShowIcebergTableDetails redirects legacy Iceberg table details URL to the merged S3 Tables details page.
|
||||
func (h *AdminHandlers) ShowIcebergTableDetails(c *gin.Context) {
|
||||
catalogName := c.Param("catalog")
|
||||
namespace := c.Param("namespace")
|
||||
tableName := c.Param("table")
|
||||
c.Redirect(http.StatusMovedPermanently, "/object-store/s3tables/buckets/"+url.PathEscape(catalogName)+"/namespaces/"+url.PathEscape(namespace)+"/tables/"+url.PathEscape(tableName))
|
||||
func (h *AdminHandlers) ShowIcebergTableDetails(w http.ResponseWriter, r *http.Request) {
|
||||
catalogName := mux.Vars(r)["catalog"]
|
||||
namespace := mux.Vars(r)["namespace"]
|
||||
tableName := mux.Vars(r)["table"]
|
||||
http.Redirect(w, r, "/object-store/s3tables/buckets/"+url.PathEscape(catalogName)+"/namespaces/"+url.PathEscape(namespace)+"/tables/"+url.PathEscape(tableName), http.StatusMovedPermanently)
|
||||
}
|
||||
|
||||
// ShowBucketDetails returns detailed information about a specific bucket
|
||||
func (h *AdminHandlers) ShowBucketDetails(c *gin.Context) {
|
||||
bucketName := c.Param("bucket")
|
||||
func (h *AdminHandlers) ShowBucketDetails(w http.ResponseWriter, r *http.Request) {
|
||||
bucketName := mux.Vars(r)["bucket"]
|
||||
details, err := h.adminServer.GetBucketDetails(bucketName)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get bucket details: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get bucket details: "+err.Error())
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, details)
|
||||
writeJSON(w, http.StatusOK, details)
|
||||
}
|
||||
|
||||
// getS3BucketsData retrieves Object Store buckets data from the server
|
||||
func (h *AdminHandlers) getS3BucketsData(c *gin.Context) dash.S3BucketsData {
|
||||
username := c.GetString("username")
|
||||
func (h *AdminHandlers) getS3BucketsData(r *http.Request) dash.S3BucketsData {
|
||||
username := dash.UsernameFromContext(r.Context())
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
@@ -636,8 +467,8 @@ func (h *AdminHandlers) getS3BucketsData(c *gin.Context) dash.S3BucketsData {
|
||||
}
|
||||
|
||||
// getAdminData retrieves admin data from the server (now uses consolidated method)
|
||||
func (h *AdminHandlers) getAdminData(c *gin.Context) dash.AdminData {
|
||||
username := c.GetString("username")
|
||||
func (h *AdminHandlers) getAdminData(r *http.Request) dash.AdminData {
|
||||
username := dash.UsernameFromContext(r.Context())
|
||||
|
||||
// Use the consolidated GetAdminData method from AdminServer
|
||||
adminData, err := h.adminServer.GetAdminData(username)
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/gorilla/sessions"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
)
|
||||
|
||||
func TestSetupRoutes_RegistersPluginSchedulerStatesAPI_NoAuth(t *testing.T) {
|
||||
router := mux.NewRouter()
|
||||
|
||||
newRouteTestAdminHandlers().SetupRoutes(router, false, "", "", "", "", true)
|
||||
|
||||
if !hasRoute(router, http.MethodGet, "/api/plugin/scheduler-states") {
|
||||
t.Fatalf("expected GET /api/plugin/scheduler-states to be registered in no-auth mode")
|
||||
}
|
||||
if !hasRoute(router, http.MethodGet, "/api/plugin/jobs/example/detail") {
|
||||
t.Fatalf("expected GET /api/plugin/jobs/:jobId/detail to be registered in no-auth mode")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetupRoutes_RegistersPluginSchedulerStatesAPI_WithAuth(t *testing.T) {
|
||||
router := mux.NewRouter()
|
||||
|
||||
newRouteTestAdminHandlers().SetupRoutes(router, true, "admin", "password", "", "", true)
|
||||
|
||||
if !hasRoute(router, http.MethodGet, "/api/plugin/scheduler-states") {
|
||||
t.Fatalf("expected GET /api/plugin/scheduler-states to be registered in auth mode")
|
||||
}
|
||||
if !hasRoute(router, http.MethodGet, "/api/plugin/jobs/example/detail") {
|
||||
t.Fatalf("expected GET /api/plugin/jobs/:jobId/detail to be registered in auth mode")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetupRoutes_RegistersPluginPages_NoAuth(t *testing.T) {
|
||||
router := mux.NewRouter()
|
||||
|
||||
newRouteTestAdminHandlers().SetupRoutes(router, false, "", "", "", "", true)
|
||||
|
||||
assertHasRoute(t, router, http.MethodGet, "/plugin")
|
||||
assertHasRoute(t, router, http.MethodGet, "/plugin/configuration")
|
||||
assertHasRoute(t, router, http.MethodGet, "/plugin/queue")
|
||||
assertHasRoute(t, router, http.MethodGet, "/plugin/detection")
|
||||
assertHasRoute(t, router, http.MethodGet, "/plugin/execution")
|
||||
assertHasRoute(t, router, http.MethodGet, "/plugin/monitoring")
|
||||
}
|
||||
|
||||
func TestSetupRoutes_RegistersPluginPages_WithAuth(t *testing.T) {
|
||||
router := mux.NewRouter()
|
||||
|
||||
newRouteTestAdminHandlers().SetupRoutes(router, true, "admin", "password", "", "", true)
|
||||
|
||||
assertHasRoute(t, router, http.MethodGet, "/plugin")
|
||||
assertHasRoute(t, router, http.MethodGet, "/plugin/configuration")
|
||||
assertHasRoute(t, router, http.MethodGet, "/plugin/queue")
|
||||
assertHasRoute(t, router, http.MethodGet, "/plugin/detection")
|
||||
assertHasRoute(t, router, http.MethodGet, "/plugin/execution")
|
||||
assertHasRoute(t, router, http.MethodGet, "/plugin/monitoring")
|
||||
}
|
||||
|
||||
func newRouteTestAdminHandlers() *AdminHandlers {
|
||||
adminServer := &dash.AdminServer{}
|
||||
store := sessions.NewCookieStore([]byte("test-session-key"))
|
||||
return &AdminHandlers{
|
||||
adminServer: adminServer,
|
||||
sessionStore: store,
|
||||
authHandlers: &AuthHandlers{adminServer: adminServer, sessionStore: store},
|
||||
clusterHandlers: &ClusterHandlers{adminServer: adminServer},
|
||||
fileBrowserHandlers: &FileBrowserHandlers{adminServer: adminServer},
|
||||
userHandlers: &UserHandlers{adminServer: adminServer},
|
||||
policyHandlers: &PolicyHandlers{adminServer: adminServer},
|
||||
pluginHandlers: &PluginHandlers{adminServer: adminServer},
|
||||
mqHandlers: &MessageQueueHandlers{adminServer: adminServer},
|
||||
serviceAccountHandlers: &ServiceAccountHandlers{adminServer: adminServer},
|
||||
}
|
||||
}
|
||||
|
||||
func hasRoute(router *mux.Router, method string, path string) bool {
|
||||
req := httptest.NewRequest(method, path, nil)
|
||||
var match mux.RouteMatch
|
||||
return router.Match(req, &match)
|
||||
}
|
||||
|
||||
func assertHasRoute(t *testing.T, router *mux.Router, method string, path string) {
|
||||
t.Helper()
|
||||
if !hasRoute(router, method, path) {
|
||||
t.Fatalf("expected %s %s to be registered", method, path)
|
||||
}
|
||||
}
|
||||
@@ -3,52 +3,65 @@ package handlers
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-contrib/sessions"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/gorilla/sessions"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
)
|
||||
|
||||
// AuthHandlers contains authentication-related HTTP handlers
|
||||
type AuthHandlers struct {
|
||||
adminServer *dash.AdminServer
|
||||
adminServer *dash.AdminServer
|
||||
sessionStore sessions.Store
|
||||
}
|
||||
|
||||
// NewAuthHandlers creates a new instance of AuthHandlers
|
||||
func NewAuthHandlers(adminServer *dash.AdminServer) *AuthHandlers {
|
||||
func NewAuthHandlers(adminServer *dash.AdminServer, store sessions.Store) *AuthHandlers {
|
||||
return &AuthHandlers{
|
||||
adminServer: adminServer,
|
||||
adminServer: adminServer,
|
||||
sessionStore: store,
|
||||
}
|
||||
}
|
||||
|
||||
// ShowLogin displays the login page
|
||||
func (a *AuthHandlers) ShowLogin(c *gin.Context) {
|
||||
session := sessions.Default(c)
|
||||
|
||||
// If already authenticated, redirect to admin
|
||||
if session.Get("authenticated") == true {
|
||||
c.Redirect(http.StatusSeeOther, "/admin")
|
||||
return
|
||||
func (a *AuthHandlers) ShowLogin(w http.ResponseWriter, r *http.Request) {
|
||||
session, err := a.sessionStore.Get(r, dash.SessionName())
|
||||
var csrfToken string
|
||||
if err == nil {
|
||||
if authenticated, _ := session.Values["authenticated"].(bool); authenticated {
|
||||
http.Redirect(w, r, "/admin", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
glog.V(1).Infof("Failed to load session for login page: %v", err)
|
||||
}
|
||||
|
||||
errorMessage := c.Query("error")
|
||||
if session != nil {
|
||||
token, tokenErr := dash.EnsureSessionCSRFToken(session, r, w)
|
||||
if tokenErr != nil {
|
||||
glog.V(1).Infof("Failed to ensure CSRF token for login page: %v", tokenErr)
|
||||
} else {
|
||||
csrfToken = token
|
||||
}
|
||||
}
|
||||
|
||||
errorMessage := r.URL.Query().Get("error")
|
||||
|
||||
// Render login template
|
||||
c.Header("Content-Type", "text/html")
|
||||
loginComponent := layout.LoginForm(c, "SeaweedFS Admin", errorMessage)
|
||||
err := loginComponent.Render(c.Request.Context(), c.Writer)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render login template: " + err.Error()})
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
loginComponent := layout.LoginForm("SeaweedFS Admin", errorMessage, csrfToken)
|
||||
if err := loginComponent.Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render login template: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// HandleLogin handles login form submission
|
||||
func (a *AuthHandlers) HandleLogin(adminUser, adminPassword, readOnlyUser, readOnlyPassword string) gin.HandlerFunc {
|
||||
return a.adminServer.HandleLogin(adminUser, adminPassword, readOnlyUser, readOnlyPassword)
|
||||
func (a *AuthHandlers) HandleLogin(adminUser, adminPassword, readOnlyUser, readOnlyPassword string) http.HandlerFunc {
|
||||
return a.adminServer.HandleLogin(a.sessionStore, adminUser, adminPassword, readOnlyUser, readOnlyPassword)
|
||||
}
|
||||
|
||||
// HandleLogout handles user logout
|
||||
func (a *AuthHandlers) HandleLogout(c *gin.Context) {
|
||||
a.adminServer.HandleLogout(c)
|
||||
func (a *AuthHandlers) HandleLogout(w http.ResponseWriter, r *http.Request) {
|
||||
a.adminServer.HandleLogout(a.sessionStore, w, r)
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/view/app"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
|
||||
@@ -24,402 +24,387 @@ func NewClusterHandlers(adminServer *dash.AdminServer) *ClusterHandlers {
|
||||
}
|
||||
|
||||
// ShowClusterVolumeServers renders the cluster volume servers page
|
||||
func (h *ClusterHandlers) ShowClusterVolumeServers(c *gin.Context) {
|
||||
func (h *ClusterHandlers) ShowClusterVolumeServers(w http.ResponseWriter, r *http.Request) {
|
||||
// Get cluster volume servers data
|
||||
volumeServersData, err := h.adminServer.GetClusterVolumeServers()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get cluster volume servers: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get cluster volume servers: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Set username
|
||||
username := c.GetString("username")
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
username := usernameOrDefault(r)
|
||||
volumeServersData.Username = username
|
||||
|
||||
// Render HTML template
|
||||
c.Header("Content-Type", "text/html")
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
volumeServersComponent := app.ClusterVolumeServers(*volumeServersData)
|
||||
layoutComponent := layout.Layout(c, volumeServersComponent)
|
||||
err = layoutComponent.Render(c.Request.Context(), c.Writer)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, volumeServersComponent)
|
||||
if err := layoutComponent.Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// ShowClusterVolumes renders the cluster volumes page
|
||||
func (h *ClusterHandlers) ShowClusterVolumes(c *gin.Context) {
|
||||
func (h *ClusterHandlers) ShowClusterVolumes(w http.ResponseWriter, r *http.Request) {
|
||||
// Get pagination and sorting parameters from query string
|
||||
page := 1
|
||||
if p := c.Query("page"); p != "" {
|
||||
if p := r.URL.Query().Get("page"); p != "" {
|
||||
if parsed, err := strconv.Atoi(p); err == nil && parsed > 0 {
|
||||
page = parsed
|
||||
}
|
||||
}
|
||||
|
||||
pageSize := 100
|
||||
if ps := c.Query("pageSize"); ps != "" {
|
||||
if ps := r.URL.Query().Get("pageSize"); ps != "" {
|
||||
if parsed, err := strconv.Atoi(ps); err == nil && parsed > 0 && parsed <= 1000 {
|
||||
pageSize = parsed
|
||||
}
|
||||
}
|
||||
|
||||
sortBy := c.DefaultQuery("sortBy", "id")
|
||||
sortOrder := c.DefaultQuery("sortOrder", "asc")
|
||||
collection := c.Query("collection") // Optional collection filter
|
||||
sortBy := defaultQuery(r.URL.Query().Get("sortBy"), "id")
|
||||
sortOrder := defaultQuery(r.URL.Query().Get("sortOrder"), "asc")
|
||||
collection := r.URL.Query().Get("collection") // Optional collection filter
|
||||
|
||||
// Get cluster volumes data
|
||||
volumesData, err := h.adminServer.GetClusterVolumes(page, pageSize, sortBy, sortOrder, collection)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get cluster volumes: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get cluster volumes: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Set username
|
||||
username := c.GetString("username")
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
username := usernameOrDefault(r)
|
||||
volumesData.Username = username
|
||||
|
||||
// Render HTML template
|
||||
c.Header("Content-Type", "text/html")
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
volumesComponent := app.ClusterVolumes(*volumesData)
|
||||
layoutComponent := layout.Layout(c, volumesComponent)
|
||||
err = layoutComponent.Render(c.Request.Context(), c.Writer)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, volumesComponent)
|
||||
if err := layoutComponent.Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// ShowVolumeDetails renders the volume details page
|
||||
func (h *ClusterHandlers) ShowVolumeDetails(c *gin.Context) {
|
||||
volumeIDStr := c.Param("id")
|
||||
server := c.Param("server")
|
||||
func (h *ClusterHandlers) ShowVolumeDetails(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
volumeIDStr := vars["id"]
|
||||
server := vars["server"]
|
||||
|
||||
if volumeIDStr == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Volume ID is required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Volume ID is required")
|
||||
return
|
||||
}
|
||||
|
||||
if server == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Server is required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Server is required")
|
||||
return
|
||||
}
|
||||
|
||||
volumeID, err := strconv.Atoi(volumeIDStr)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid volume ID"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid volume ID")
|
||||
return
|
||||
}
|
||||
|
||||
// Get volume details
|
||||
volumeDetails, err := h.adminServer.GetVolumeDetails(volumeID, server)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get volume details: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get volume details: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
username := usernameOrDefault(r)
|
||||
// Render HTML template
|
||||
c.Header("Content-Type", "text/html")
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
volumeDetailsComponent := app.VolumeDetails(*volumeDetails)
|
||||
layoutComponent := layout.Layout(c, volumeDetailsComponent)
|
||||
err = layoutComponent.Render(c.Request.Context(), c.Writer)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, volumeDetailsComponent)
|
||||
if err := layoutComponent.Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// ShowClusterCollections renders the cluster collections page
|
||||
func (h *ClusterHandlers) ShowClusterCollections(c *gin.Context) {
|
||||
func (h *ClusterHandlers) ShowClusterCollections(w http.ResponseWriter, r *http.Request) {
|
||||
// Get cluster collections data
|
||||
collectionsData, err := h.adminServer.GetClusterCollections()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get cluster collections: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get cluster collections: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Set username
|
||||
username := c.GetString("username")
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
username := usernameOrDefault(r)
|
||||
collectionsData.Username = username
|
||||
|
||||
// Render HTML template
|
||||
c.Header("Content-Type", "text/html")
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
collectionsComponent := app.ClusterCollections(*collectionsData)
|
||||
layoutComponent := layout.Layout(c, collectionsComponent)
|
||||
err = layoutComponent.Render(c.Request.Context(), c.Writer)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, collectionsComponent)
|
||||
if err := layoutComponent.Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// ShowCollectionDetails renders the collection detail page
|
||||
func (h *ClusterHandlers) ShowCollectionDetails(c *gin.Context) {
|
||||
collectionName := c.Param("name")
|
||||
func (h *ClusterHandlers) ShowCollectionDetails(w http.ResponseWriter, r *http.Request) {
|
||||
collectionName := mux.Vars(r)["name"]
|
||||
if collectionName == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Collection name is required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Collection name is required")
|
||||
return
|
||||
}
|
||||
|
||||
// Parse query parameters
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "25"))
|
||||
sortBy := c.DefaultQuery("sort_by", "volume_id")
|
||||
sortOrder := c.DefaultQuery("sort_order", "asc")
|
||||
query := r.URL.Query()
|
||||
page, _ := strconv.Atoi(defaultQuery(query.Get("page"), "1"))
|
||||
pageSize, _ := strconv.Atoi(defaultQuery(query.Get("page_size"), "25"))
|
||||
sortBy := defaultQuery(query.Get("sort_by"), "volume_id")
|
||||
sortOrder := defaultQuery(query.Get("sort_order"), "asc")
|
||||
|
||||
// Get collection details data (volumes and EC volumes)
|
||||
collectionDetailsData, err := h.adminServer.GetCollectionDetails(collectionName, page, pageSize, sortBy, sortOrder)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get collection details: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get collection details: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Set username
|
||||
username := c.GetString("username")
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
username := usernameOrDefault(r)
|
||||
collectionDetailsData.Username = username
|
||||
|
||||
// Render HTML template
|
||||
c.Header("Content-Type", "text/html")
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
collectionDetailsComponent := app.CollectionDetails(*collectionDetailsData)
|
||||
layoutComponent := layout.Layout(c, collectionDetailsComponent)
|
||||
err = layoutComponent.Render(c.Request.Context(), c.Writer)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, collectionDetailsComponent)
|
||||
if err := layoutComponent.Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// ShowClusterEcShards handles the cluster EC shards page (individual shards view)
|
||||
func (h *ClusterHandlers) ShowClusterEcShards(c *gin.Context) {
|
||||
func (h *ClusterHandlers) ShowClusterEcShards(w http.ResponseWriter, r *http.Request) {
|
||||
// Parse query parameters
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "100"))
|
||||
sortBy := c.DefaultQuery("sort_by", "volume_id")
|
||||
sortOrder := c.DefaultQuery("sort_order", "asc")
|
||||
collection := c.DefaultQuery("collection", "")
|
||||
query := r.URL.Query()
|
||||
page, _ := strconv.Atoi(defaultQuery(query.Get("page"), "1"))
|
||||
pageSize, _ := strconv.Atoi(defaultQuery(query.Get("page_size"), "100"))
|
||||
sortBy := defaultQuery(query.Get("sort_by"), "volume_id")
|
||||
sortOrder := defaultQuery(query.Get("sort_order"), "asc")
|
||||
collection := defaultQuery(query.Get("collection"), "")
|
||||
|
||||
// Get data from admin server
|
||||
data, err := h.adminServer.GetClusterEcVolumes(page, pageSize, sortBy, sortOrder, collection)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Set username
|
||||
username := c.GetString("username")
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
username := usernameOrDefault(r)
|
||||
data.Username = username
|
||||
|
||||
// Render template
|
||||
c.Header("Content-Type", "text/html")
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
ecVolumesComponent := app.ClusterEcVolumes(*data)
|
||||
layoutComponent := layout.Layout(c, ecVolumesComponent)
|
||||
err = layoutComponent.Render(c.Request.Context(), c.Writer)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, ecVolumesComponent)
|
||||
if err := layoutComponent.Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// ShowEcVolumeDetails renders the EC volume details page
|
||||
func (h *ClusterHandlers) ShowEcVolumeDetails(c *gin.Context) {
|
||||
volumeIDStr := c.Param("id")
|
||||
func (h *ClusterHandlers) ShowEcVolumeDetails(w http.ResponseWriter, r *http.Request) {
|
||||
volumeIDStr := mux.Vars(r)["id"]
|
||||
|
||||
if volumeIDStr == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Volume ID is required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Volume ID is required")
|
||||
return
|
||||
}
|
||||
|
||||
volumeID, err := strconv.Atoi(volumeIDStr)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid volume ID"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid volume ID")
|
||||
return
|
||||
}
|
||||
|
||||
// Check that volumeID is within uint32 range
|
||||
if volumeID < 0 || uint64(volumeID) > math.MaxUint32 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Volume ID out of range"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Volume ID out of range")
|
||||
return
|
||||
}
|
||||
|
||||
// Parse sorting parameters
|
||||
sortBy := c.DefaultQuery("sort_by", "shard_id")
|
||||
sortOrder := c.DefaultQuery("sort_order", "asc")
|
||||
query := r.URL.Query()
|
||||
sortBy := defaultQuery(query.Get("sort_by"), "shard_id")
|
||||
sortOrder := defaultQuery(query.Get("sort_order"), "asc")
|
||||
|
||||
// Get EC volume details
|
||||
ecVolumeDetails, err := h.adminServer.GetEcVolumeDetails(uint32(volumeID), sortBy, sortOrder)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get EC volume details: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get EC volume details: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Set username
|
||||
username := c.GetString("username")
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
username := usernameOrDefault(r)
|
||||
ecVolumeDetails.Username = username
|
||||
|
||||
// Render HTML template
|
||||
c.Header("Content-Type", "text/html")
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
ecVolumeDetailsComponent := app.EcVolumeDetails(*ecVolumeDetails)
|
||||
layoutComponent := layout.Layout(c, ecVolumeDetailsComponent)
|
||||
err = layoutComponent.Render(c.Request.Context(), c.Writer)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, ecVolumeDetailsComponent)
|
||||
if err := layoutComponent.Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// ShowClusterMasters renders the cluster masters page
|
||||
func (h *ClusterHandlers) ShowClusterMasters(c *gin.Context) {
|
||||
func (h *ClusterHandlers) ShowClusterMasters(w http.ResponseWriter, r *http.Request) {
|
||||
// Get cluster masters data
|
||||
mastersData, err := h.adminServer.GetClusterMasters()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get cluster masters: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get cluster masters: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Set username
|
||||
username := c.GetString("username")
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
username := usernameOrDefault(r)
|
||||
mastersData.Username = username
|
||||
|
||||
// Render HTML template
|
||||
c.Header("Content-Type", "text/html")
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
mastersComponent := app.ClusterMasters(*mastersData)
|
||||
layoutComponent := layout.Layout(c, mastersComponent)
|
||||
err = layoutComponent.Render(c.Request.Context(), c.Writer)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, mastersComponent)
|
||||
if err := layoutComponent.Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// ShowClusterFilers renders the cluster filers page
|
||||
func (h *ClusterHandlers) ShowClusterFilers(c *gin.Context) {
|
||||
func (h *ClusterHandlers) ShowClusterFilers(w http.ResponseWriter, r *http.Request) {
|
||||
// Get cluster filers data
|
||||
filersData, err := h.adminServer.GetClusterFilers()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get cluster filers: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get cluster filers: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Set username
|
||||
username := c.GetString("username")
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
username := usernameOrDefault(r)
|
||||
filersData.Username = username
|
||||
|
||||
// Render HTML template
|
||||
c.Header("Content-Type", "text/html")
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
filersComponent := app.ClusterFilers(*filersData)
|
||||
layoutComponent := layout.Layout(c, filersComponent)
|
||||
err = layoutComponent.Render(c.Request.Context(), c.Writer)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, filersComponent)
|
||||
if err := layoutComponent.Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// ShowClusterBrokers renders the cluster message brokers page
|
||||
func (h *ClusterHandlers) ShowClusterBrokers(c *gin.Context) {
|
||||
func (h *ClusterHandlers) ShowClusterBrokers(w http.ResponseWriter, r *http.Request) {
|
||||
// Get cluster brokers data
|
||||
brokersData, err := h.adminServer.GetClusterBrokers()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get cluster brokers: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get cluster brokers: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Set username
|
||||
username := c.GetString("username")
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
username := usernameOrDefault(r)
|
||||
brokersData.Username = username
|
||||
|
||||
// Render HTML template
|
||||
c.Header("Content-Type", "text/html")
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
brokersComponent := app.ClusterBrokers(*brokersData)
|
||||
layoutComponent := layout.Layout(c, brokersComponent)
|
||||
err = layoutComponent.Render(c.Request.Context(), c.Writer)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, brokersComponent)
|
||||
if err := layoutComponent.Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// GetClusterTopology returns the cluster topology as JSON
|
||||
func (h *ClusterHandlers) GetClusterTopology(c *gin.Context) {
|
||||
func (h *ClusterHandlers) GetClusterTopology(w http.ResponseWriter, r *http.Request) {
|
||||
topology, err := h.adminServer.GetClusterTopology()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, topology)
|
||||
writeJSON(w, http.StatusOK, topology)
|
||||
}
|
||||
|
||||
// GetMasters returns master node information
|
||||
func (h *ClusterHandlers) GetMasters(c *gin.Context) {
|
||||
// Simple master info
|
||||
c.JSON(http.StatusOK, gin.H{"masters": []gin.H{{"address": "localhost:9333"}}})
|
||||
func (h *ClusterHandlers) GetMasters(w http.ResponseWriter, r *http.Request) {
|
||||
mastersData, err := h.adminServer.GetClusterMasters()
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get cluster masters: "+err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, mastersData)
|
||||
}
|
||||
|
||||
// GetVolumeServers returns volume server information
|
||||
func (h *ClusterHandlers) GetVolumeServers(c *gin.Context) {
|
||||
func (h *ClusterHandlers) GetVolumeServers(w http.ResponseWriter, r *http.Request) {
|
||||
topology, err := h.adminServer.GetClusterTopology()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"volume_servers": topology.VolumeServers})
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"volume_servers": topology.VolumeServers})
|
||||
}
|
||||
|
||||
// VacuumVolume handles volume vacuum requests via API
|
||||
func (h *ClusterHandlers) VacuumVolume(c *gin.Context) {
|
||||
volumeIDStr := c.Param("id")
|
||||
server := c.Param("server")
|
||||
func (h *ClusterHandlers) VacuumVolume(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
volumeIDStr := vars["id"]
|
||||
server := vars["server"]
|
||||
|
||||
if volumeIDStr == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Volume ID is required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Volume ID is required")
|
||||
return
|
||||
}
|
||||
|
||||
volumeID, err := strconv.Atoi(volumeIDStr)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid volume ID"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid volume ID")
|
||||
return
|
||||
}
|
||||
|
||||
if server == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "Server is required")
|
||||
return
|
||||
}
|
||||
|
||||
// Perform vacuum operation
|
||||
err = h.adminServer.VacuumVolume(volumeID, server)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{
|
||||
"error": "Failed to vacuum volume: " + err.Error(),
|
||||
})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to vacuum volume: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"message": "Volume vacuum started successfully",
|
||||
"volume_id": volumeID,
|
||||
"server": server,
|
||||
})
|
||||
}
|
||||
|
||||
func usernameOrDefault(r *http.Request) string {
|
||||
username := dash.UsernameFromContext(r.Context())
|
||||
if username == "" {
|
||||
return "admin"
|
||||
}
|
||||
return username
|
||||
}
|
||||
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/view/app"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
|
||||
@@ -59,16 +58,16 @@ func (h *FileBrowserHandlers) newClientWithTimeout(timeout time.Duration) http.C
|
||||
}
|
||||
|
||||
// ShowFileBrowser renders the file browser page
|
||||
func (h *FileBrowserHandlers) ShowFileBrowser(c *gin.Context) {
|
||||
func (h *FileBrowserHandlers) ShowFileBrowser(w http.ResponseWriter, r *http.Request) {
|
||||
// Get path from query parameter, default to root
|
||||
path := c.DefaultQuery("path", "/")
|
||||
path := defaultQuery(r.URL.Query().Get("path"), "/")
|
||||
// Normalize Windows-style paths for consistency
|
||||
path = util.CleanWindowsPath(path)
|
||||
|
||||
// Get pagination parameters
|
||||
lastFileName := c.DefaultQuery("lastFileName", "")
|
||||
lastFileName := r.URL.Query().Get("lastFileName")
|
||||
|
||||
pageSize, err := strconv.Atoi(c.DefaultQuery("limit", "20"))
|
||||
pageSize, err := strconv.Atoi(defaultQuery(r.URL.Query().Get("limit"), "20"))
|
||||
if err != nil || pageSize < 1 {
|
||||
pageSize = 20
|
||||
}
|
||||
@@ -79,36 +78,42 @@ func (h *FileBrowserHandlers) ShowFileBrowser(c *gin.Context) {
|
||||
// Get file browser data with cursor-based pagination
|
||||
browserData, err := h.adminServer.GetFileBrowser(path, lastFileName, pageSize)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get file browser data: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get file browser data: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Set username
|
||||
username := c.GetString("username")
|
||||
username := dash.UsernameFromContext(r.Context())
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
browserData.Username = username
|
||||
|
||||
// Render HTML template
|
||||
c.Header("Content-Type", "text/html")
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
browserComponent := app.FileBrowser(*browserData)
|
||||
layoutComponent := layout.Layout(c, browserComponent)
|
||||
err = layoutComponent.Render(c.Request.Context(), c.Writer)
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, browserComponent)
|
||||
err = layoutComponent.Render(r.Context(), w)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteFile handles file deletion API requests
|
||||
func (h *FileBrowserHandlers) DeleteFile(c *gin.Context) {
|
||||
func (h *FileBrowserHandlers) DeleteFile(w http.ResponseWriter, r *http.Request) {
|
||||
var request struct {
|
||||
Path string `json:"path" binding:"required"`
|
||||
}
|
||||
|
||||
if err := c.ShouldBindJSON(&request); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &request); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if strings.TrimSpace(request.Path) == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "path is required")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -124,29 +129,36 @@ func (h *FileBrowserHandlers) DeleteFile(c *gin.Context) {
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to delete file: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to delete file: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"message": "File deleted successfully"})
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"message": "File deleted successfully"})
|
||||
}
|
||||
|
||||
// DeleteMultipleFiles handles multiple file deletion API requests
|
||||
func (h *FileBrowserHandlers) DeleteMultipleFiles(c *gin.Context) {
|
||||
func (h *FileBrowserHandlers) DeleteMultipleFiles(w http.ResponseWriter, r *http.Request) {
|
||||
var request struct {
|
||||
Paths []string `json:"paths" binding:"required"`
|
||||
}
|
||||
|
||||
if err := c.ShouldBindJSON(&request); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &request); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if len(request.Paths) == 0 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "No paths provided"})
|
||||
writeJSONError(w, http.StatusBadRequest, "No paths provided")
|
||||
return
|
||||
}
|
||||
|
||||
for _, path := range request.Paths {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "path is required")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
var deletedCount int
|
||||
var failedCount int
|
||||
var errors []string
|
||||
@@ -189,37 +201,40 @@ func (h *FileBrowserHandlers) DeleteMultipleFiles(c *gin.Context) {
|
||||
} else {
|
||||
response["message"] = fmt.Sprintf("Deleted %d item(s), failed to delete %d item(s)", deletedCount, failedCount)
|
||||
}
|
||||
c.JSON(http.StatusOK, response)
|
||||
writeJSON(w, http.StatusOK, response)
|
||||
} else {
|
||||
response["message"] = "Failed to delete all selected items"
|
||||
c.JSON(http.StatusInternalServerError, response)
|
||||
writeJSON(w, http.StatusInternalServerError, response)
|
||||
}
|
||||
}
|
||||
|
||||
// CreateFolder handles folder creation requests
|
||||
func (h *FileBrowserHandlers) CreateFolder(c *gin.Context) {
|
||||
func (h *FileBrowserHandlers) CreateFolder(w http.ResponseWriter, r *http.Request) {
|
||||
var request struct {
|
||||
Path string `json:"path" binding:"required"`
|
||||
FolderName string `json:"folder_name" binding:"required"`
|
||||
}
|
||||
|
||||
if err := c.ShouldBindJSON(&request); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &request); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if strings.TrimSpace(request.Path) == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "path is required")
|
||||
return
|
||||
}
|
||||
|
||||
// Clean and validate folder name
|
||||
folderName := strings.TrimSpace(request.FolderName)
|
||||
if folderName == "" || strings.Contains(folderName, "/") || strings.Contains(folderName, "\\") {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid folder name"})
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid folder name")
|
||||
return
|
||||
}
|
||||
|
||||
// Create full path for new folder
|
||||
fullPath := filepath.Join(request.Path, folderName)
|
||||
if !strings.HasPrefix(fullPath, "/") {
|
||||
fullPath = "/" + fullPath
|
||||
}
|
||||
base := "/" + strings.TrimPrefix(request.Path, "/")
|
||||
fullPath := path.Join(base, folderName)
|
||||
|
||||
// Create folder via filer
|
||||
err := h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
@@ -241,32 +256,32 @@ func (h *FileBrowserHandlers) CreateFolder(c *gin.Context) {
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create folder: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to create folder: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"message": "Folder created successfully"})
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"message": "Folder created successfully"})
|
||||
}
|
||||
|
||||
// UploadFile handles file upload requests
|
||||
func (h *FileBrowserHandlers) UploadFile(c *gin.Context) {
|
||||
func (h *FileBrowserHandlers) UploadFile(w http.ResponseWriter, r *http.Request) {
|
||||
// Get the current path
|
||||
currentPath := c.PostForm("path")
|
||||
currentPath := r.FormValue("path")
|
||||
if currentPath == "" {
|
||||
currentPath = "/"
|
||||
}
|
||||
|
||||
// Parse multipart form
|
||||
err := c.Request.ParseMultipartForm(1 << 30) // 1GB max memory for large file uploads
|
||||
err := r.ParseMultipartForm(1 << 30) // 1GB max memory for large file uploads
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to parse multipart form: " + err.Error()})
|
||||
writeJSONError(w, http.StatusBadRequest, "Failed to parse multipart form: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Get uploaded files (supports multiple files)
|
||||
files := c.Request.MultipartForm.File["files"]
|
||||
files := r.MultipartForm.File["files"]
|
||||
if len(files) == 0 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "No files uploaded"})
|
||||
writeJSONError(w, http.StatusBadRequest, "No files uploaded")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -292,16 +307,8 @@ func (h *FileBrowserHandlers) UploadFile(c *gin.Context) {
|
||||
fullPath = "/" + fullPath
|
||||
}
|
||||
|
||||
// Open the file
|
||||
file, err := fileHeader.Open()
|
||||
if err != nil {
|
||||
failedUploads = append(failedUploads, fmt.Sprintf("%s: %v", fileName, err))
|
||||
continue
|
||||
}
|
||||
|
||||
// Upload file to filer
|
||||
err = h.uploadFileToFiler(fullPath, fileHeader)
|
||||
file.Close()
|
||||
|
||||
if err != nil {
|
||||
failedUploads = append(failedUploads, fmt.Sprintf("%s: %v", fileName, err))
|
||||
@@ -331,10 +338,10 @@ func (h *FileBrowserHandlers) UploadFile(c *gin.Context) {
|
||||
} else {
|
||||
response["message"] = fmt.Sprintf("Uploaded %d file(s), %d failed", len(uploadResults), len(failedUploads))
|
||||
}
|
||||
c.JSON(http.StatusOK, response)
|
||||
writeJSON(w, http.StatusOK, response)
|
||||
} else {
|
||||
response["message"] = "All file uploads failed"
|
||||
c.JSON(http.StatusInternalServerError, response)
|
||||
writeJSON(w, http.StatusInternalServerError, response)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -561,23 +568,23 @@ func (h *FileBrowserHandlers) fetchFileContent(filePath string, timeout time.Dur
|
||||
|
||||
// DownloadFile handles file download requests by proxying through the Admin UI server
|
||||
// This ensures mTLS works correctly since the Admin UI server has the client certificates
|
||||
func (h *FileBrowserHandlers) DownloadFile(c *gin.Context) {
|
||||
filePath := c.Query("path")
|
||||
func (h *FileBrowserHandlers) DownloadFile(w http.ResponseWriter, r *http.Request) {
|
||||
filePath := r.URL.Query().Get("path")
|
||||
if filePath == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "File path is required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "File path is required")
|
||||
return
|
||||
}
|
||||
|
||||
// Get filer address
|
||||
filerAddress := h.adminServer.GetFilerAddress()
|
||||
if filerAddress == "" {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Filer address not configured"})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Filer address not configured")
|
||||
return
|
||||
}
|
||||
|
||||
// Validate filer address to prevent SSRF
|
||||
if err := h.validateFilerAddress(filerAddress); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Invalid filer address configuration"})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Invalid filer address configuration")
|
||||
return
|
||||
}
|
||||
filerHttpAddress := pb.ServerAddress(filerAddress).ToHttpAddress()
|
||||
@@ -585,7 +592,7 @@ func (h *FileBrowserHandlers) DownloadFile(c *gin.Context) {
|
||||
// Validate and sanitize the file path
|
||||
cleanFilePath, err := h.validateAndCleanFilePath(filePath)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file path: " + err.Error()})
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid file path: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
@@ -593,7 +600,7 @@ func (h *FileBrowserHandlers) DownloadFile(c *gin.Context) {
|
||||
downloadURL := fmt.Sprintf("%s%s", filerHttpAddress, cleanFilePath)
|
||||
downloadURL, err = h.httpClient.NormalizeHttpScheme(downloadURL)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to construct download URL: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to construct download URL: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
@@ -602,9 +609,9 @@ func (h *FileBrowserHandlers) DownloadFile(c *gin.Context) {
|
||||
// Safe: filerAddress validated by validateFilerAddress() to match configured filer
|
||||
// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
|
||||
// Use request context so download is cancelled when client disconnects
|
||||
req, err := http.NewRequestWithContext(c.Request.Context(), "GET", downloadURL, nil)
|
||||
req, err := http.NewRequestWithContext(r.Context(), "GET", downloadURL, nil)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create request: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to create request: "+err.Error())
|
||||
return
|
||||
}
|
||||
client := h.newClientWithTimeout(5 * time.Minute) // Longer timeout for large file downloads
|
||||
@@ -613,7 +620,7 @@ func (h *FileBrowserHandlers) DownloadFile(c *gin.Context) {
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": "Failed to fetch file from filer: " + err.Error()})
|
||||
writeJSONError(w, http.StatusBadGateway, "Failed to fetch file from filer: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
@@ -621,10 +628,10 @@ func (h *FileBrowserHandlers) DownloadFile(c *gin.Context) {
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
c.JSON(resp.StatusCode, gin.H{"error": fmt.Sprintf("Filer returned status %d but failed to read response body: %v", resp.StatusCode, err)})
|
||||
writeJSONError(w, resp.StatusCode, fmt.Sprintf("Filer returned status %d but failed to read response body: %v", resp.StatusCode, err))
|
||||
return
|
||||
}
|
||||
c.JSON(resp.StatusCode, gin.H{"error": fmt.Sprintf("Filer returned status %d: %s", resp.StatusCode, string(body))})
|
||||
writeJSONError(w, resp.StatusCode, fmt.Sprintf("Filer returned status %d: %s", resp.StatusCode, string(body)))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -632,33 +639,33 @@ func (h *FileBrowserHandlers) DownloadFile(c *gin.Context) {
|
||||
fileName := filepath.Base(cleanFilePath)
|
||||
// Use mime.FormatMediaType for RFC 6266 compliant Content-Disposition,
|
||||
// properly handling non-ASCII characters and special characters
|
||||
c.Header("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": fileName}))
|
||||
w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": fileName}))
|
||||
|
||||
// Use content type from filer response, or default to octet-stream
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
c.Header("Content-Type", contentType)
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
|
||||
// Set content length if available
|
||||
if resp.ContentLength > 0 {
|
||||
c.Header("Content-Length", fmt.Sprintf("%d", resp.ContentLength))
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", resp.ContentLength))
|
||||
}
|
||||
|
||||
// Stream the response body to the client
|
||||
c.Status(http.StatusOK)
|
||||
_, err = io.Copy(c.Writer, resp.Body)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, err = io.Copy(w, resp.Body)
|
||||
if err != nil {
|
||||
glog.Errorf("Error streaming file download: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ViewFile handles file viewing requests (for text files, images, etc.)
|
||||
func (h *FileBrowserHandlers) ViewFile(c *gin.Context) {
|
||||
filePath := c.Query("path")
|
||||
func (h *FileBrowserHandlers) ViewFile(w http.ResponseWriter, r *http.Request) {
|
||||
filePath := r.URL.Query().Get("path")
|
||||
if filePath == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "File path is required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "File path is required")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -704,7 +711,7 @@ func (h *FileBrowserHandlers) ViewFile(c *gin.Context) {
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get file metadata: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get file metadata: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
@@ -752,7 +759,7 @@ func (h *FileBrowserHandlers) ViewFile(c *gin.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"file": fileEntry,
|
||||
"content": content,
|
||||
"viewable": viewable,
|
||||
@@ -761,10 +768,10 @@ func (h *FileBrowserHandlers) ViewFile(c *gin.Context) {
|
||||
}
|
||||
|
||||
// GetFileProperties handles file properties requests
|
||||
func (h *FileBrowserHandlers) GetFileProperties(c *gin.Context) {
|
||||
filePath := c.Query("path")
|
||||
func (h *FileBrowserHandlers) GetFileProperties(w http.ResponseWriter, r *http.Request) {
|
||||
filePath := r.URL.Query().Get("path")
|
||||
if filePath == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "File path is required"})
|
||||
writeJSONError(w, http.StatusBadRequest, "File path is required")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -853,11 +860,11 @@ func (h *FileBrowserHandlers) GetFileProperties(c *gin.Context) {
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get file properties: " + err.Error()})
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get file properties: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, properties)
|
||||
writeJSON(w, http.StatusOK, properties)
|
||||
}
|
||||
|
||||
// Helper function to format bytes
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/internal/httputil"
|
||||
)
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, payload interface{}) {
|
||||
httputil.WriteJSON(w, status, payload)
|
||||
}
|
||||
|
||||
func writeJSONError(w http.ResponseWriter, status int, message string) {
|
||||
httputil.WriteJSONError(w, status, message)
|
||||
}
|
||||
|
||||
func decodeJSONBody(r io.Reader, v interface{}) error {
|
||||
return httputil.DecodeJSONBody(r, v)
|
||||
}
|
||||
|
||||
func newJSONMaxReader(w http.ResponseWriter, r *http.Request) io.Reader {
|
||||
return httputil.NewJSONMaxReader(w, r)
|
||||
}
|
||||
|
||||
func defaultQuery(value, fallback string) string {
|
||||
return httputil.DefaultQuery(value, fallback)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user