Compare commits

..
Author SHA1 Message Date
chrislu 15fcc03b04 fix comment 2025-10-26 22:44:10 -07:00
chrislu 502c183763 adjust to warning 2025-10-26 21:18:55 -07:00
chrislu a348381a51 refactor 2025-10-26 21:14:10 -07:00
chrislu 859a30bbd8 refactor 2025-10-26 21:09:03 -07:00
chrislu f20d7c3bd5 fmt 2025-10-26 20:29:13 -07:00
chrislu 40f82aa806 Avoid misclassifying local EC as distributed when .dat stat errors occur; also standardize unload-before-remove. 2025-10-26 20:28:47 -07:00
chrislu b5a6fa0cdf minor 2025-10-26 20:26:58 -07:00
chrislu d07b24e42c Eliminated Redundant Parsing in checkOrphanedShards 2025-10-26 20:18:08 -07:00
chrislu f85dd4df39 address comments 2025-10-26 19:56:34 -07:00
chrislu e1b448100e The check for orphaned shards should be performed for the previous volume before resetting sameVolumeShards for the new volume. 2025-10-26 19:51:04 -07:00
chrislu ab5f91ea3b handle exceptions in tests; use constants 2025-10-26 19:41:43 -07:00
chrislu 4605a30446 unexpected error 2025-10-26 19:21:39 -07:00
chrislu c6968ffd96 grouping logic should be updated to use both collection and volumeId to ensure correctness 2025-10-26 19:21:27 -07:00
chrislu 2bcf65741f 10GB sparse files can be slow or fail on non-sparse FS. Use 10MB to hit SmallBlockSize math (1MB shards) deterministically. 2025-10-26 19:13:33 -07:00
chrislu 46ce3b9831 fix shard directory 2025-10-26 13:58:38 -07:00
chrislu fbb459ca4c minor 2025-10-26 13:57:56 -07:00
chrislu 6fff6269c8 refactoring 2025-10-26 13:53:10 -07:00
chrislu 7b6f364e7f calculate expected shard size 2025-10-26 13:48:11 -07:00
chrislu 96caa76101 validate shard size 2025-10-26 13:32:03 -07:00
chrislu ed3788eb80 check ec shard sizes 2025-10-26 11:53:18 -07:00
chrislu e3d0f9e4ce Added Shard Size Validation 2025-10-26 11:46:22 -07:00
chrislu d2822ae7e6 refactor 2025-10-26 10:45:49 -07:00
chrislu 6f3cb7bebe Update disk_location_ec.go 2025-10-26 09:57:05 -07:00
chrislu 3843a75ac6 in memory cleanup 2025-10-26 09:44:43 -07:00
chrislu 1a124d1fd8 address comments
* Performance: Avoid Double os.Stat() Call
* Platform Compatibility: Use filepath.Join
2025-10-26 09:42:40 -07:00
chrislu 31f905ce2a Update disk_location_ec.go
When loadEcShards() fails partway through, some EC shards may already be loaded into the l.ecVolumes map in memory. The previous code only cleaned up filesystem files but left orphaned in-memory state, which could cause memory leaks and inconsistent state.
2025-10-26 01:32:50 -07:00
chrislu 51edda3356 simplify, and better logs 2025-10-26 01:28:18 -07:00
chrislu 6f7ceb63eb unit tests 2025-10-26 01:25:48 -07:00
chrislu 829dc8e092 handle incomplete ec encoding 2025-10-26 01:21:45 -07:00
508 changed files with 9214 additions and 40096 deletions
+2 -2
View File
@@ -38,7 +38,7 @@ jobs:
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
- name: Set BUILD_TIME env
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
@@ -87,7 +87,7 @@ jobs:
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
- name: Set BUILD_TIME env
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
with:
+1 -1
View File
@@ -18,7 +18,7 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
+3 -11
View File
@@ -16,11 +16,11 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # v3
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v3
with:
images: |
chrislusf/seaweedfs
@@ -33,20 +33,12 @@ jobs:
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v1
-
name: Create BuildKit config
run: |
cat > /tmp/buildkitd.toml <<EOF
[registry."docker.io"]
mirrors = ["https://mirror.gcr.io"]
EOF
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
with:
buildkitd-flags: "--debug"
buildkitd-config: /tmp/buildkitd.toml
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
@@ -1,169 +0,0 @@
name: "docker: build foundationdb image by version"
on:
pull_request:
branches: [ master, main ]
paths:
- 'weed/filer/foundationdb/**'
- 'test/foundationdb/**'
- 'docker/Dockerfile.foundationdb_large'
- 'docker/filer_foundationdb.toml'
- '.github/workflows/container_foundationdb_version.yml'
workflow_dispatch:
inputs:
fdb_version:
description: 'FoundationDB version to build (e.g. 7.4.5)'
required: true
default: '7.4.5'
seaweedfs_ref:
description: 'SeaweedFS git tag, branch, or commit to build'
required: true
default: 'master'
image_tag:
description: 'Optional Docker tag suffix (defaults to foundationdb_<fdb>_seaweedfs_<ref>)'
required: false
default: ''
permissions:
contents: read
jobs:
build-foundationdb-image:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Install FoundationDB client libraries
run: |
set -euo pipefail
sudo apt-get update
sudo apt-get install -y ca-certificates wget
FDB_VERSION="${{ inputs.fdb_version || '7.4.5' }}"
case "${FDB_VERSION}_amd64" in
"7.4.5_amd64") EXPECTED_SHA256="eea6b98cf386a0848655b2e196d18633662a7440a7ee061c10e32153c7e7e112" ;;
"7.3.43_amd64") EXPECTED_SHA256="c3fa0a59c7355b914a1455dac909238d5ea3b6c6bc7b530af8597e6487c1651a" ;;
*)
echo "Unsupported FoundationDB version ${FDB_VERSION} for CI client install" >&2
exit 1 ;;
esac
PACKAGE="foundationdb-clients_${FDB_VERSION}-1_amd64.deb"
wget --timeout=30 --tries=3 -O "${PACKAGE}" "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/${PACKAGE}"
echo "${EXPECTED_SHA256} ${PACKAGE}" | sha256sum -c -
sudo dpkg -i "${PACKAGE}"
rm "${PACKAGE}"
sudo ldconfig
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: go.mod
- name: Run FoundationDB tagged tests
env:
CGO_ENABLED: 1
run: |
go test ./weed/filer/foundationdb -tags foundationdb -count=1
- name: Prepare Docker tag
id: tag
env:
FDB_VERSION_INPUT: ${{ inputs.fdb_version }}
SEAWEEDFS_REF_INPUT: ${{ inputs.seaweedfs_ref }}
CUSTOM_TAG_INPUT: ${{ inputs.image_tag }}
EVENT_NAME: ${{ github.event_name }}
HEAD_REF: ${{ github.head_ref }}
REF_NAME: ${{ github.ref_name }}
run: |
set -euo pipefail
sanitize() {
local value="$1"
value="${value,,}"
value="${value// /-}"
value="${value//[^a-z0-9_.-]/-}"
value="${value#-}"
value="${value%-}"
printf '%s' "$value"
}
version="${FDB_VERSION_INPUT}"
seaweed="${SEAWEEDFS_REF_INPUT}"
tag="${CUSTOM_TAG_INPUT}"
# Use defaults for PR builds
if [ -z "$version" ]; then
version="7.4.5"
fi
if [ -z "$seaweed" ]; then
if [ "$EVENT_NAME" = "pull_request" ]; then
seaweed="${HEAD_REF}"
else
seaweed="${REF_NAME}"
fi
fi
sanitized_version="$(sanitize "$version")"
if [ -z "$sanitized_version" ]; then
echo "Unable to sanitize FoundationDB version '$version'." >&2
exit 1
fi
sanitized_seaweed="$(sanitize "$seaweed")"
if [ -z "$sanitized_seaweed" ]; then
echo "Unable to sanitize SeaweedFS ref '$seaweed'." >&2
exit 1
fi
if [ -z "$tag" ]; then
tag="foundationdb_${sanitized_version}_seaweedfs_${sanitized_seaweed}"
else
tag="$(sanitize "$tag")"
fi
if [ -z "$tag" ]; then
echo "Resulting Docker tag is empty." >&2
exit 1
fi
echo "docker_tag=$tag" >> "$GITHUB_OUTPUT"
echo "full_image=chrislusf/seaweedfs:$tag" >> "$GITHUB_OUTPUT"
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Determine branch to build
id: branch
run: |
if [ -n "${{ inputs.seaweedfs_ref }}" ]; then
echo "branch=${{ inputs.seaweedfs_ref }}" >> "$GITHUB_OUTPUT"
elif [ "${{ github.event_name }}" = "pull_request" ]; then
echo "branch=${{ github.head_ref }}" >> "$GITHUB_OUTPUT"
else
echo "branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
fi
- name: Build and push image
uses: docker/build-push-action@v6
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.foundationdb_large
build-args: |
FDB_VERSION=${{ inputs.fdb_version || '7.4.5' }}
BRANCH=${{ steps.branch.outputs.branch }}
# Note: ARM64 support requires FoundationDB ARM64 packages which are not available for all versions
# Currently only building for amd64. To enable ARM64, verify package availability and add checksums.
platforms: linux/amd64
tags: ${{ steps.tag.outputs.full_image || 'seaweedfs:foundationdb-test' }}
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
+29 -161
View File
@@ -10,190 +10,58 @@ permissions:
contents: read
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
platform: [amd64, arm64, arm, 386]
include:
- platform: amd64
qemu: false
- platform: arm64
qemu: true
- platform: arm
qemu: true
- platform: 386
qemu: true
build-latest-container:
runs-on: [ubuntu-latest]
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Free Disk Space
run: |
echo "Available disk space before cleanup:"
df -h
# Remove pre-installed tools
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
# Clean package managers
sudo apt-get clean
sudo rm -rf /var/lib/apt/lists/*
# Clean Docker aggressively
sudo docker system prune -af --volumes
# Clean Go cache if it exists
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
[ -d /go/pkg ] && rm -rf /go/pkg || true
echo "Available disk space after cleanup:"
df -h
- name: Docker meta
-
name: Checkout
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v3
with:
images: |
chrislusf/seaweedfs
ghcr.io/chrislusf/seaweedfs
tags: type=raw,value=latest
tags: |
type=raw,value=latest
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
- name: Set up QEMU
if: matrix.qemu
uses: docker/setup-qemu-action@v3
- name: Create BuildKit config
run: |
cat > /tmp/buildkitd.toml <<EOF
[registry."docker.io"]
mirrors = ["https://mirror.gcr.io"]
EOF
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
-
name: Set up QEMU
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
with:
buildkitd-flags: "--debug"
buildkitd-config: /tmp/buildkitd.toml
- name: Login to Docker Hub
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
-
name: Login to GHCR
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Build ${{ matrix.platform }}
uses: docker/build-push-action@v6
env:
DOCKER_BUILDKIT: 1
-
name: Build
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
platforms: linux/${{ matrix.platform }}
# Push to GHCR only during build to avoid Docker Hub rate limits
tags: ghcr.io/chrislusf/seaweedfs:latest-${{ matrix.platform }}
platforms: linux/amd64, linux/arm, linux/arm64, linux/386
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
build-args: |
BUILDKIT_INLINE_CACHE=1
BRANCH=${{ github.sha }}
- name: Clean up build artifacts
if: always()
run: |
# Clean up Docker build cache and temporary files
sudo docker system prune -f
# Remove Go build cache
sudo rm -rf /tmp/go-build*
create-manifest:
runs-on: ubuntu-latest
needs: [build]
if: github.event_name != 'pull_request'
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
with:
images: |
chrislusf/seaweedfs
ghcr.io/chrislusf/seaweedfs
tags: type=raw,value=latest
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Install crane
run: |
# Install crane for efficient multi-arch image copying
cd $(mktemp -d)
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
sudo mv crane /usr/local/bin/
crane version
- name: Create and push manifest
run: |
# Create manifest on GHCR first (no rate limits)
echo "Creating GHCR manifest (no rate limits)..."
docker buildx imagetools create -t ghcr.io/chrislusf/seaweedfs:latest \
ghcr.io/chrislusf/seaweedfs:latest-amd64 \
ghcr.io/chrislusf/seaweedfs:latest-arm64 \
ghcr.io/chrislusf/seaweedfs:latest-arm \
ghcr.io/chrislusf/seaweedfs:latest-386
# Copy the complete multi-arch image from GHCR to Docker Hub
# This only requires one pull from GHCR (no rate limit) and one push to Docker Hub
echo "Copying manifest from GHCR to Docker Hub..."
# Function to retry with exponential backoff for Docker Hub operations
retry_with_backoff() {
local max_attempts=5
local timeout=1
local attempt=1
local exit_code=0
while [ $attempt -le $max_attempts ]; do
if "$@"; then
return 0
else
exit_code=$?
fi
if [ $attempt -lt $max_attempts ]; then
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
sleep $timeout
timeout=$((timeout * 2))
fi
attempt=$((attempt + 1))
done
echo "Command failed after $max_attempts attempts" >&2
return $exit_code
}
# Use crane or skopeo to copy, fallback to docker if not available
if command -v crane &> /dev/null; then
echo "Using crane to copy..."
retry_with_backoff crane copy ghcr.io/chrislusf/seaweedfs:latest chrislusf/seaweedfs:latest
elif command -v skopeo &> /dev/null; then
echo "Using skopeo to copy..."
retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs:latest docker://chrislusf/seaweedfs:latest
else
echo "Using docker buildx imagetools (pulling 4 images from Docker Hub)..."
# Fallback: create manifest directly on Docker Hub (pulls from Docker Hub - rate limited)
retry_with_backoff docker buildx imagetools create -t chrislusf/seaweedfs:latest \
ghcr.io/chrislusf/seaweedfs:latest-amd64 \
ghcr.io/chrislusf/seaweedfs:latest-arm64 \
ghcr.io/chrislusf/seaweedfs:latest-arm \
ghcr.io/chrislusf/seaweedfs:latest-386
fi
+37 -132
View File
@@ -1,152 +1,57 @@
name: "docker: build release containers for normal volume"
# DISABLED: Merged into container_release_unified.yml
on:
workflow_dispatch:
inputs:
force_run:
description: 'This workflow is disabled. Use container_release_unified.yml instead'
required: true
default: 'disabled'
push:
tags:
- '*'
workflow_dispatch: {}
permissions:
contents: read
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
platform: [amd64, arm64, arm, 386]
include:
- platform: amd64
qemu: false
- platform: arm64
qemu: true
- platform: arm
qemu: true
- platform: 386
qemu: true
build-default-release-container:
runs-on: [ubuntu-latest]
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Free Disk Space
run: |
echo "Available disk space before cleanup:"
df -h
# Remove pre-installed tools
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
# Clean package managers
sudo apt-get clean
sudo rm -rf /var/lib/apt/lists/*
# Clean Docker aggressively
sudo docker system prune -af --volumes
# Clean Go cache if it exists
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
[ -d /go/pkg ] && rm -rf /go/pkg || true
echo "Available disk space after cleanup:"
df -h
- name: Docker meta
-
name: Checkout
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v3
with:
images: chrislusf/seaweedfs
tags: type=ref,event=tag
flavor: latest=false
- name: Set up QEMU
if: matrix.qemu
uses: docker/setup-qemu-action@v3
- name: Create BuildKit config
run: |
cat > /tmp/buildkitd.toml <<EOF
[registry."docker.io"]
mirrors = ["https://mirror.gcr.io"]
EOF
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
buildkitd-config: /tmp/buildkitd.toml
- name: Login to Docker Hub
images: |
chrislusf/seaweedfs
tags: |
type=ref,event=tag
flavor: |
latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build ${{ matrix.platform }}
uses: docker/build-push-action@v6
env:
DOCKER_BUILDKIT: 1
-
name: Build
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
platforms: linux/${{ matrix.platform }}
tags: ${{ steps.docker_meta.outputs.tags }}-${{ matrix.platform }}
platforms: linux/amd64, linux/arm, linux/arm64, linux/386
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
build-args: |
BUILDKIT_INLINE_CACHE=1
BRANCH=${{ github.sha }}
- name: Clean up build artifacts
if: always()
run: |
# Clean up Docker build cache and temporary files
sudo docker system prune -f
# Remove Go build cache
sudo rm -rf /tmp/go-build*
create-manifest:
runs-on: ubuntu-latest
needs: [build]
if: github.event_name != 'pull_request'
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
with:
images: chrislusf/seaweedfs
tags: type=ref,event=tag
flavor: latest=false
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Create and push manifest
run: |
# Function to retry command with exponential backoff
retry_with_backoff() {
local max_attempts=5
local timeout=1
local attempt=1
local exit_code=0
while [ $attempt -le $max_attempts ]; do
if "$@"; then
return 0
else
exit_code=$?
fi
if [ $attempt -lt $max_attempts ]; then
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
sleep $timeout
timeout=$((timeout * 2))
fi
attempt=$((attempt + 1))
done
echo "Command failed after $max_attempts attempts" >&2
return $exit_code
}
# Create manifest with retry
retry_with_backoff docker buildx imagetools create -t ${{ steps.docker_meta.outputs.tags }} \
${{ steps.docker_meta.outputs.tags }}-amd64 \
${{ steps.docker_meta.outputs.tags }}-arm64 \
${{ steps.docker_meta.outputs.tags }}-arm \
${{ steps.docker_meta.outputs.tags }}-386
+39 -133
View File
@@ -1,153 +1,59 @@
name: "docker: build release containers for large volume"
# DISABLED: Merged into container_release_unified.yml
on:
workflow_dispatch:
inputs:
force_run:
description: 'This workflow is disabled. Use container_release_unified.yml instead'
required: true
default: 'disabled'
push:
tags:
- '*'
workflow_dispatch: {}
permissions:
contents: read
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
platform: [amd64, arm64, arm, 386]
include:
- platform: amd64
qemu: false
- platform: arm64
qemu: true
- platform: arm
qemu: true
- platform: 386
qemu: true
build-large-release-container:
runs-on: [ubuntu-latest]
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Free Disk Space
run: |
echo "Available disk space before cleanup:"
df -h
# Remove pre-installed tools
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
# Clean package managers
sudo apt-get clean
sudo rm -rf /var/lib/apt/lists/*
# Clean Docker aggressively
sudo docker system prune -af --volumes
# Clean Go cache if it exists
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
[ -d /go/pkg ] && rm -rf /go/pkg || true
echo "Available disk space after cleanup:"
df -h
- name: Docker meta
-
name: Checkout
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v3
with:
images: chrislusf/seaweedfs
tags: type=ref,event=tag,suffix=_large_disk
flavor: latest=false
- name: Set up QEMU
if: matrix.qemu
uses: docker/setup-qemu-action@v3
- name: Create BuildKit config
run: |
cat > /tmp/buildkitd.toml <<EOF
[registry."docker.io"]
mirrors = ["https://mirror.gcr.io"]
EOF
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
buildkitd-config: /tmp/buildkitd.toml
- name: Login to Docker Hub
images: |
chrislusf/seaweedfs
tags: |
type=ref,event=tag,suffix=_large_disk
flavor: |
latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build ${{ matrix.platform }}
uses: docker/build-push-action@v6
env:
DOCKER_BUILDKIT: 1
-
name: Build
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
build-args: |
TAGS=5BytesOffset
BUILDKIT_INLINE_CACHE=1
BRANCH=${{ github.sha }}
platforms: linux/${{ matrix.platform }}
tags: ${{ steps.docker_meta.outputs.tags }}-${{ matrix.platform }}
build-args: TAGS=5BytesOffset
platforms: linux/amd64, linux/arm, linux/arm64, linux/386
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Clean up build artifacts
if: always()
run: |
# Clean up Docker build cache and temporary files
sudo docker system prune -f
# Remove Go build cache
sudo rm -rf /tmp/go-build*
create-manifest:
runs-on: ubuntu-latest
needs: [build]
if: github.event_name != 'pull_request'
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
with:
images: chrislusf/seaweedfs
tags: type=ref,event=tag,suffix=_large_disk
flavor: latest=false
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Create and push manifest
run: |
# Function to retry command with exponential backoff
retry_with_backoff() {
local max_attempts=5
local timeout=1
local attempt=1
local exit_code=0
while [ $attempt -le $max_attempts ]; do
if "$@"; then
return 0
else
exit_code=$?
fi
if [ $attempt -lt $max_attempts ]; then
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
sleep $timeout
timeout=$((timeout * 2))
fi
attempt=$((attempt + 1))
done
echo "Command failed after $max_attempts attempts" >&2
return $exit_code
}
# Create manifest with retry
retry_with_backoff docker buildx imagetools create -t ${{ steps.docker_meta.outputs.tags }} \
${{ steps.docker_meta.outputs.tags }}-amd64 \
${{ steps.docker_meta.outputs.tags }}-arm64 \
${{ steps.docker_meta.outputs.tags }}-arm \
${{ steps.docker_meta.outputs.tags }}-386
+17 -30
View File
@@ -1,13 +1,10 @@
name: "docker: build release containers for rocksdb"
# DISABLED: Merged into container_release_unified.yml
on:
workflow_dispatch:
inputs:
force_run:
description: 'This workflow is disabled. Use container_release_unified.yml instead'
required: true
default: 'disabled'
push:
tags:
- '*'
workflow_dispatch: {}
permissions:
contents: read
@@ -20,46 +17,38 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@v6
-
name: Free Disk Space
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
sudo docker system prune -af
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v3
with:
images: chrislusf/seaweedfs
tags: type=ref,event=tag,suffix=_large_disk_rocksdb
flavor: latest=false
images: |
chrislusf/seaweedfs
tags: |
type=ref,event=tag,suffix=_large_disk_rocksdb
flavor: |
latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Create BuildKit config
run: |
cat > /tmp/buildkitd.toml <<EOF
[registry."docker.io"]
mirrors = ["https://mirror.gcr.io"]
EOF
name: Set up QEMU
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
buildkitd-config: /tmp/buildkitd.toml
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@v6
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
@@ -69,5 +58,3 @@ jobs:
platforms: linux/amd64
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
+12 -26
View File
@@ -1,13 +1,10 @@
name: "docker: build release containers for all tags"
# DISABLED: Merged into container_release_unified.yml
on:
workflow_dispatch:
inputs:
force_run:
description: 'This workflow is disabled. Use container_release_unified.yml instead'
required: true
default: 'disabled'
push:
tags:
- '*'
workflow_dispatch: {}
permissions:
contents: read
@@ -19,23 +16,11 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@v6
-
name: Free Disk Space
run: |
echo "Before cleanup:"
df -h
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
sudo docker system prune -af
echo "After cleanup:"
df -h
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v3
with:
images: |
chrislusf/seaweedfs
@@ -47,19 +32,22 @@ jobs:
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@v6
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
@@ -68,5 +56,3 @@ jobs:
platforms: linux/amd64
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
+18 -22
View File
@@ -1,13 +1,10 @@
name: "docker: build release containers for all tags and large volume"
# DISABLED: Merged into container_release_unified.yml
on:
workflow_dispatch:
inputs:
force_run:
description: 'This workflow is disabled. Use container_release_unified.yml instead'
required: true
default: 'disabled'
push:
tags:
- '*'
workflow_dispatch: {}
permissions:
contents: read
@@ -19,37 +16,38 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@v6
-
name: Free Disk Space
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
sudo docker system prune -af
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v3
with:
images: chrislusf/seaweedfs
tags: type=ref,event=tag,suffix=_large_disk_full
flavor: latest=false
images: |
chrislusf/seaweedfs
tags: |
type=ref,event=tag,suffix=_large_disk_full
flavor: |
latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@v6
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
@@ -58,5 +56,3 @@ jobs:
platforms: linux/amd64
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
@@ -1,71 +0,0 @@
name: "docker: build release containers for foundationdb"
on:
push:
tags:
- '*'
workflow_dispatch: {}
permissions:
contents: read
jobs:
build-large-release-container_foundationdb:
runs-on: [ubuntu-latest]
steps:
-
name: Checkout
uses: actions/checkout@v6
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
with:
images: |
chrislusf/seaweedfs
tags: |
type=ref,event=tag,suffix=_large_disk_foundationdb
flavor: |
latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@v3
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Determine branch to build
id: branch
run: |
if [ "${{ github.event_name }}" = "push" ] && [ -n "${{ github.ref_name }}" ]; then
echo "branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
else
echo "branch=master" >> "$GITHUB_OUTPUT"
fi
-
name: Build
uses: docker/build-push-action@v6
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.foundationdb_large
build-args: |
BRANCH=${{ steps.branch.outputs.branch }}
# Note: ARM64 support requires FoundationDB ARM64 packages which are not available for all versions
platforms: linux/amd64
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
@@ -1,225 +0,0 @@
name: "docker: build all release containers (unified)"
on:
push:
tags:
- '*'
workflow_dispatch: {}
permissions:
contents: read
# Limit concurrent builds to avoid rate limits
concurrency:
group: release-${{ github.ref }}
cancel-in-progress: false
jobs:
build:
runs-on: ubuntu-latest
strategy:
# Build sequentially to avoid rate limits
max-parallel: 2
matrix:
include:
# Normal volume - multi-arch
- variant: normal
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
dockerfile: ./docker/Dockerfile.go_build
build_args: ""
tag_suffix: ""
# Large disk - multi-arch
- variant: large_disk
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
dockerfile: ./docker/Dockerfile.go_build
build_args: TAGS=5BytesOffset
tag_suffix: _large_disk
# Full tags - amd64 only
- variant: full
platforms: linux/amd64
dockerfile: ./docker/Dockerfile.go_build
build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
tag_suffix: _full
# Large disk + full tags - amd64 only
- variant: large_disk_full
platforms: linux/amd64
dockerfile: ./docker/Dockerfile.go_build
build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
tag_suffix: _large_disk_full
# RocksDB large disk - amd64 only
- variant: rocksdb
platforms: linux/amd64
dockerfile: ./docker/Dockerfile.rocksdb_large
build_args: ""
tag_suffix: _large_disk_rocksdb
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Free Disk Space
run: |
echo "Available disk space before cleanup:"
df -h
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
sudo apt-get clean
sudo rm -rf /var/lib/apt/lists/*
sudo docker system prune -af --volumes
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
[ -d /go/pkg ] && rm -rf /go/pkg || true
echo "Available disk space after cleanup:"
df -h
- name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
with:
images: |
chrislusf/seaweedfs
ghcr.io/chrislusf/seaweedfs
tags: type=ref,event=tag,suffix=${{ matrix.tag_suffix }}
flavor: latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
- name: Set up QEMU
if: contains(matrix.platforms, 'arm')
uses: docker/setup-qemu-action@v3
- name: Create BuildKit config
run: |
cat > /tmp/buildkitd.toml <<EOF
[registry."docker.io"]
mirrors = ["https://mirror.gcr.io"]
EOF
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
buildkitd-config: /tmp/buildkitd.toml
- name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Build and push ${{ matrix.variant }}
uses: docker/build-push-action@v6
env:
DOCKER_BUILDKIT: 1
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ${{ matrix.dockerfile }}
platforms: ${{ matrix.platforms }}
# Push to GHCR to avoid Docker Hub rate limits on pulls
tags: |
ghcr.io/chrislusf/seaweedfs:${{ github.ref_name }}${{ matrix.tag_suffix }}
labels: ${{ steps.docker_meta.outputs.labels }}
cache-from: type=gha,scope=${{ matrix.variant }}
cache-to: type=gha,mode=max,scope=${{ matrix.variant }}
build-args: |
${{ matrix.build_args }}
BUILDKIT_INLINE_CACHE=1
BRANCH=${{ github.sha }}
- name: Clean up build artifacts
if: always()
run: |
sudo docker system prune -f
sudo rm -rf /tmp/go-build*
copy-to-dockerhub:
runs-on: ubuntu-latest
needs: [build]
if: github.event_name != 'pull_request'
strategy:
matrix:
variant: [normal, large_disk, full, large_disk_full, rocksdb]
include:
- variant: normal
tag_suffix: ""
- variant: large_disk
tag_suffix: _large_disk
- variant: full
tag_suffix: _full
- variant: large_disk_full
tag_suffix: _large_disk_full
- variant: rocksdb
tag_suffix: _large_disk_rocksdb
steps:
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Install crane
run: |
cd $(mktemp -d)
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
sudo mv crane /usr/local/bin/
crane version
- name: Copy ${{ matrix.variant }} from GHCR to Docker Hub
run: |
# Function to retry with exponential backoff
retry_with_backoff() {
local max_attempts=5
local timeout=1
local attempt=1
local exit_code=0
while [ $attempt -le $max_attempts ]; do
if "$@"; then
return 0
else
exit_code=$?
fi
if [ $attempt -lt $max_attempts ]; then
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
sleep $timeout
timeout=$((timeout * 2))
fi
attempt=$((attempt + 1))
done
echo "Command failed after $max_attempts attempts" >&2
return $exit_code
}
# Copy multi-arch image from GHCR to Docker Hub with retry
# This is much more efficient than pulling/pushing individual arch images
echo "Copying ${{ matrix.variant }} from GHCR to Docker Hub..."
retry_with_backoff crane copy \
ghcr.io/chrislusf/seaweedfs:${{ github.ref_name }}${{ matrix.tag_suffix }} \
chrislusf/seaweedfs:${{ github.ref_name }}${{ matrix.tag_suffix }}
echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"
@@ -25,7 +25,7 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
- name: Prepare Docker tag
id: tag
@@ -82,7 +82,7 @@ jobs:
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
- name: Set up QEMU
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v1
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
+1 -1
View File
@@ -21,7 +21,7 @@ jobs:
deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
+2 -2
View File
@@ -9,6 +9,6 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: 'Checkout Repository'
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
- name: 'Dependency Review'
uses: actions/dependency-review-action@3c4e3dcb1aa7874d2c16be7d79418e9b7efd6261
uses: actions/dependency-review-action@40c09b7dc99638e5ddb0bfd91c1673effc064d8a
+3 -3
View File
@@ -24,13 +24,13 @@ jobs:
timeout-minutes: 30
steps:
- name: Set up Go 1.x
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v2
uses: actions/setup-go@c0137caad775660c0844396c52da96e560aba63d # v2
with:
go-version: ^1.13
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
@@ -134,7 +134,7 @@ jobs:
- name: Archive logs
if: always()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: output-logs
path: docker/output.log
+2 -2
View File
@@ -33,7 +33,7 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go ${{ env.GO_VERSION }}
uses: actions/setup-go@v6
@@ -183,7 +183,7 @@ jobs:
- name: Upload Test Artifacts
if: always()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: fuse-integration-test-results
path: |
+2 -2
View File
@@ -21,13 +21,13 @@ jobs:
steps:
- name: Set up Go 1.x
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v2
uses: actions/setup-go@c0137caad775660c0844396c52da96e560aba63d # v2
with:
go-version: ^1.13
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v2
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v2
- name: Get dependencies
run: |
+1 -1
View File
@@ -12,7 +12,7 @@ jobs:
release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
- name: Publish Helm charts
uses: stefanprodan/helm-gh-pages@v1.7.0
with:
+3 -3
View File
@@ -16,7 +16,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
with:
fetch-depth: 0
@@ -31,7 +31,7 @@ jobs:
check-latest: true
- name: Set up chart-testing
uses: helm/chart-testing-action@v2.8.0
uses: helm/chart-testing-action@v2.7.0
- name: Run chart-testing (list-changed)
id: list-changed
@@ -45,7 +45,7 @@ jobs:
run: ct lint --target-branch ${{ github.event.repository.default_branch }} --all --validate-maintainers=false --chart-dirs k8s/charts
- name: Create kind cluster
uses: helm/kind-action@v1.13.0
uses: helm/kind-action@v1.12.0
- name: Run chart-testing (install)
run: ct install --target-branch ${{ github.event.repository.default_branch }} --all --chart-dirs k8s/charts
@@ -1,165 +0,0 @@
name: Java Client Integration Tests
on:
push:
branches: [ master ]
paths:
- 'other/java/**'
- 'weed/**'
- '.github/workflows/java_integration_tests.yml'
pull_request:
branches: [ master ]
paths:
- 'other/java/**'
- 'weed/**'
- '.github/workflows/java_integration_tests.yml'
jobs:
test:
name: Java Integration Tests
runs-on: ubuntu-latest
strategy:
matrix:
java: ['11', '17']
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Java
uses: actions/setup-java@v5
with:
java-version: ${{ matrix.java }}
distribution: 'temurin'
cache: 'maven'
- name: Build SeaweedFS
run: |
cd weed
go install -buildvcs=false
weed version
- name: Start SeaweedFS Server
run: |
# Create clean data directory
export WEED_DATA_DIR="/tmp/seaweedfs-java-tests-$(date +%s)"
mkdir -p "$WEED_DATA_DIR"
# Start SeaweedFS with optimized settings for CI
weed server -dir="$WEED_DATA_DIR" \
-master.raftHashicorp \
-master.electionTimeout=1s \
-master.volumeSizeLimitMB=100 \
-volume.max=100 \
-volume.preStopSeconds=1 \
-master.peers=none \
-filer -filer.maxMB=64 \
-master.port=9333 \
-volume.port=8080 \
-filer.port=8888 \
-metricsPort=9324 > seaweedfs.log 2>&1 &
SERVER_PID=$!
echo "SERVER_PID=$SERVER_PID" >> $GITHUB_ENV
echo "WEED_DATA_DIR=$WEED_DATA_DIR" >> $GITHUB_ENV
echo "SeaweedFS server started with PID: $SERVER_PID"
- name: Wait for SeaweedFS Components
run: |
echo "Waiting for SeaweedFS components to start..."
# Wait for master
for i in {1..30}; do
if curl -s http://localhost:9333/cluster/status > /dev/null 2>&1; then
echo "✓ Master server is ready"
break
fi
echo "Waiting for master server... ($i/30)"
sleep 2
done
# Wait for volume
for i in {1..30}; do
if curl -s http://localhost:8080/status > /dev/null 2>&1; then
echo "✓ Volume server is ready"
break
fi
echo "Waiting for volume server... ($i/30)"
sleep 2
done
# Wait for filer
for i in {1..30}; do
if curl -s http://localhost:8888/ > /dev/null 2>&1; then
echo "✓ Filer is ready"
break
fi
echo "Waiting for filer... ($i/30)"
sleep 2
done
echo "✓ All SeaweedFS components are ready!"
# Display cluster status
echo "Cluster status:"
curl -s http://localhost:9333/cluster/status | head -20
- name: Build and Install SeaweedFS Client
working-directory: other/java/client
run: |
mvn clean install -DskipTests -Dmaven.javadoc.skip=true -Dgpg.skip=true
- name: Run Client Unit Tests
working-directory: other/java/client
run: |
mvn test -Dtest=SeaweedReadTest,SeaweedCipherTest
- name: Run Client Integration Tests
working-directory: other/java/client
env:
SEAWEEDFS_TEST_ENABLED: true
run: |
mvn test -Dtest=*IntegrationTest
- name: Run HDFS3 Configuration Tests
working-directory: other/java/hdfs3
run: |
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true
- name: Display logs on failure
if: failure()
run: |
echo "=== SeaweedFS Server Log ==="
tail -100 seaweedfs.log || echo "No server log"
echo ""
echo "=== Cluster Status ==="
curl -s http://localhost:9333/cluster/status || echo "Cannot reach cluster"
echo ""
echo "=== Process Status ==="
ps aux | grep weed || echo "No weed processes"
- name: Cleanup
if: always()
run: |
# Stop server using stored PID
if [ -n "$SERVER_PID" ]; then
echo "Stopping SeaweedFS server (PID: $SERVER_PID)"
kill -9 $SERVER_PID 2>/dev/null || true
fi
# Fallback: kill any remaining weed processes
pkill -f "weed server" || true
# Clean up data directory
if [ -n "$WEED_DATA_DIR" ]; then
echo "Cleaning up data directory: $WEED_DATA_DIR"
rm -rf "$WEED_DATA_DIR" || true
fi
-58
View File
@@ -1,58 +0,0 @@
name: Java Client Unit Tests
on:
push:
branches: [ master ]
paths:
- 'other/java/**'
- '.github/workflows/java_unit_tests.yml'
pull_request:
branches: [ master ]
paths:
- 'other/java/**'
- '.github/workflows/java_unit_tests.yml'
jobs:
test:
name: Java Unit Tests
runs-on: ubuntu-latest
strategy:
matrix:
java: ['8', '11', '17', '21']
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Java
uses: actions/setup-java@v5
with:
java-version: ${{ matrix.java }}
distribution: 'temurin'
cache: 'maven'
- name: Build and Install SeaweedFS Client
working-directory: other/java/client
run: |
mvn clean install -DskipTests -Dmaven.javadoc.skip=true -Dgpg.skip=true
- name: Run Client Unit Tests
working-directory: other/java/client
run: |
mvn test -Dtest=SeaweedReadTest,SeaweedCipherTest
- name: Run HDFS3 Configuration Tests
working-directory: other/java/hdfs3
run: |
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true
- name: Upload Test Reports
if: always()
uses: actions/upload-artifact@v5
with:
name: test-reports-java-${{ matrix.java }}
path: |
other/java/client/target/surefire-reports/
other/java/hdfs3/target/surefire-reports/
+1 -1
View File
@@ -21,7 +21,7 @@ jobs:
timeout-minutes: 15
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go 1.x
uses: actions/setup-go@v6
+7 -11
View File
@@ -41,7 +41,7 @@ jobs:
id: go
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Setup Container Environment
run: |
@@ -85,7 +85,7 @@ jobs:
id: go
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Setup Integration Container Environment
run: |
@@ -125,7 +125,7 @@ jobs:
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go 1.x
uses: actions/setup-go@v6
@@ -176,7 +176,6 @@ jobs:
-filer.port=8888 \
-filer=true \
-metricsPort=9325 \
-master.peers=none \
> /tmp/weed-server.log 2>&1 &
# Wait for master to be ready
@@ -303,7 +302,7 @@ jobs:
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go 1.x
uses: actions/setup-go@v6
@@ -354,7 +353,6 @@ jobs:
-filer.port=8888 \
-filer=true \
-metricsPort=9325 \
-master.peers=none \
> /tmp/weed-server.log 2>&1 &
# Wait for master to be ready
@@ -464,7 +462,7 @@ jobs:
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go 1.x
uses: actions/setup-go@v6
@@ -514,7 +512,6 @@ jobs:
-filer.port=8888 \
-filer=true \
-metricsPort=9325 \
-master.peers=none \
> /tmp/weed-server.log 2>&1 &
# Wait for master to be ready
@@ -621,7 +618,7 @@ jobs:
CONTAINER_ID: ${{ matrix.container-id }}
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go 1.x
uses: actions/setup-go@v6
@@ -671,7 +668,6 @@ jobs:
-filer.port=8888 \
-filer=true \
-metricsPort=9325 \
-master.peers=none \
> /tmp/weed-server.log 2>&1 &
# Wait for master to be ready
@@ -787,7 +783,7 @@ jobs:
id: go
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Setup Protocol Container Environment
run: |
+2 -2
View File
@@ -29,7 +29,7 @@ jobs:
id: go
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
@@ -62,7 +62,7 @@ jobs:
- name: Archive logs
if: always()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: postgres-logs
path: test/postgres/postgres-output.log
+14 -14
View File
@@ -25,7 +25,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
@@ -76,7 +76,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: s3-versioning-test-logs-${{ matrix.test-type }}
path: test/s3/versioning/weed-test*.log
@@ -89,7 +89,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
@@ -124,7 +124,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: s3-versioning-compatibility-logs
path: test/s3/versioning/weed-test*.log
@@ -137,7 +137,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
@@ -172,7 +172,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: s3-cors-compatibility-logs
path: test/s3/cors/weed-test*.log
@@ -188,7 +188,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
@@ -239,7 +239,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: s3-retention-test-logs-${{ matrix.test-type }}
path: test/s3/retention/weed-test*.log
@@ -255,7 +255,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
@@ -306,7 +306,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: s3-cors-test-logs-${{ matrix.test-type }}
path: test/s3/cors/weed-test*.log
@@ -319,7 +319,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
@@ -355,7 +355,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: s3-retention-worm-logs
path: test/s3/retention/weed-test*.log
@@ -370,7 +370,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
@@ -405,7 +405,7 @@ jobs:
- name: Upload stress test logs
if: always()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: s3-versioning-stress-logs
path: test/s3/versioning/weed-test*.log
+8 -8
View File
@@ -35,7 +35,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
@@ -65,7 +65,7 @@ jobs:
- name: Upload test results on failure
if: failure()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: iam-unit-test-results
path: |
@@ -84,7 +84,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
@@ -162,7 +162,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: s3-iam-integration-logs-${{ matrix.test-type }}
path: test/s3/iam/weed-*.log
@@ -176,7 +176,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
@@ -222,7 +222,7 @@ jobs:
- name: Upload distributed test logs
if: always()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: s3-iam-distributed-logs
path: test/s3/iam/weed-*.log
@@ -236,7 +236,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
@@ -274,7 +274,7 @@ jobs:
- name: Upload performance test results
if: always()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: s3-iam-performance-results
path: |
+2 -2
View File
@@ -35,7 +35,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
@@ -152,7 +152,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: s3-keycloak-test-logs
path: |
-163
View File
@@ -1,163 +0,0 @@
name: "S3 PyArrow Parquet Tests"
on:
push:
branches: [master]
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'test/s3/parquet/**'
- '.github/workflows/s3-parquet-tests.yml'
pull_request:
branches: [master]
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'test/s3/parquet/**'
- '.github/workflows/s3-parquet-tests.yml'
workflow_dispatch:
env:
S3_ACCESS_KEY: some_access_key1
S3_SECRET_KEY: some_secret_key1
S3_ENDPOINT_URL: http://localhost:8333
BUCKET_NAME: test-parquet-bucket
jobs:
parquet-integration-tests:
name: PyArrow Parquet Tests (Python ${{ matrix.python-version }})
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
python-version: ['3.9', '3.11', '3.12']
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: ^1.24
cache: true
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
cache-dependency-path: 'test/s3/parquet/requirements.txt'
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y lsof netcat-openbsd
- name: Build SeaweedFS
run: |
cd weed
go build -v
sudo cp weed /usr/local/bin/
weed version
- name: Run PyArrow Parquet integration tests
run: |
cd test/s3/parquet
make test-with-server
env:
SEAWEEDFS_BINARY: weed
S3_PORT: 8333
FILER_PORT: 8888
VOLUME_PORT: 8080
MASTER_PORT: 9333
VOLUME_MAX_SIZE_MB: 50
- name: Run implicit directory fix tests
run: |
cd test/s3/parquet
make test-implicit-dir-with-server
env:
SEAWEEDFS_BINARY: weed
S3_PORT: 8333
FILER_PORT: 8888
VOLUME_PORT: 8080
MASTER_PORT: 9333
- name: Run PyArrow native S3 filesystem tests
run: |
cd test/s3/parquet
make test-native-s3-with-server
env:
SEAWEEDFS_BINARY: weed
S3_PORT: 8333
FILER_PORT: 8888
VOLUME_PORT: 8080
MASTER_PORT: 9333
- name: Run cross-filesystem compatibility tests
run: |
cd test/s3/parquet
TEST_QUICK=1 make test-cross-fs-with-server
env:
SEAWEEDFS_BINARY: weed
S3_PORT: 8333
FILER_PORT: 8888
VOLUME_PORT: 8080
MASTER_PORT: 9333
- name: Run SSE-S3 encryption compatibility tests
run: |
cd test/s3/parquet
make test-sse-s3-compat
env:
SEAWEEDFS_BINARY: weed
S3_PORT: 8333
FILER_PORT: 8888
VOLUME_PORT: 8080
MASTER_PORT: 9333
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v5
with:
name: test-logs-python-${{ matrix.python-version }}
path: |
/tmp/seaweedfs-parquet-*.log
test/s3/parquet/*.log
retention-days: 7
- name: Cleanup
if: always()
run: |
cd test/s3/parquet
make stop-seaweedfs-safe || true
make clean || true
unit-tests:
name: Go Unit Tests (Implicit Directory)
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: ^1.24
cache: true
- name: Run Go unit tests
run: |
cd weed/s3api
go test -v -run TestImplicitDirectory
- name: Run all S3 API tests
run: |
cd weed/s3api
go test -v -timeout 5m
+17 -19
View File
@@ -4,7 +4,6 @@ on:
pull_request:
paths:
- 'weed/s3api/s3_sse_*.go'
- 'weed/s3api/s3api_object_handlers.go'
- 'weed/s3api/s3api_object_handlers_put.go'
- 'weed/s3api/s3api_object_handlers_copy*.go'
- 'weed/server/filer_server_handlers_*.go'
@@ -15,7 +14,6 @@ on:
branches: [ master, main ]
paths:
- 'weed/s3api/s3_sse_*.go'
- 'weed/s3api/s3api_object_handlers.go'
- 'weed/s3api/s3api_object_handlers_put.go'
- 'weed/s3api/s3api_object_handlers_copy*.go'
- 'weed/server/filer_server_handlers_*.go'
@@ -44,7 +42,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
@@ -70,11 +68,11 @@ jobs:
# Run tests with automatic server management
# The test-with-server target handles server startup/shutdown automatically
if [ "${{ matrix.test-type }}" = "quick" ]; then
# Quick tests - basic SSE-C and SSE-KMS functionality + Range requests
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
# Quick tests - basic SSE-C and SSE-KMS functionality
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|TestSimpleSSECIntegration"
else
# Comprehensive tests - SSE-C/KMS functionality, excluding copy operations (pre-existing SSE-C issues)
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration"
fi
- name: Show server logs on failure
@@ -95,7 +93,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: s3-sse-test-logs-${{ matrix.test-type }}
path: test/s3/sse/weed-test*.log
@@ -108,7 +106,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
@@ -129,8 +127,8 @@ jobs:
uname -a
free -h
# Run the specific tests that validate AWS S3 SSE compatibility - both SSE-C and SSE-KMS basic functionality plus Range requests
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|.*RangeRequestsServerBehavior" || {
# Run the specific tests that validate AWS S3 SSE compatibility - both SSE-C and SSE-KMS basic functionality
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic" || {
echo "❌ SSE compatibility test failed, checking logs..."
if [ -f weed-test.log ]; then
echo "=== Server logs ==="
@@ -143,7 +141,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: s3-sse-compatibility-logs
path: test/s3/sse/weed-test*.log
@@ -156,7 +154,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
@@ -192,7 +190,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: s3-sse-metadata-persistence-logs
path: test/s3/sse/weed-test*.log
@@ -205,7 +203,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
@@ -241,7 +239,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: s3-sse-copy-operations-logs
path: test/s3/sse/weed-test*.log
@@ -254,7 +252,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
@@ -290,7 +288,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: s3-sse-multipart-logs
path: test/s3/sse/weed-test*.log
@@ -305,7 +303,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go
uses: actions/setup-go@v6
@@ -340,7 +338,7 @@ jobs:
- name: Upload performance test logs
if: always()
uses: actions/upload-artifact@v5
uses: actions/upload-artifact@v4
with:
name: s3-sse-performance-logs
path: test/s3/sse/weed-test*.log
+14 -21
View File
@@ -20,7 +20,7 @@ jobs:
timeout-minutes: 15
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go 1.x
uses: actions/setup-go@v6
@@ -54,17 +54,17 @@ jobs:
shell: bash
run: |
cd weed
go install -tags s3tests -buildvcs=false
go install -buildvcs=false
set -x
# Create clean data directory for this test run
export WEED_DATA_DIR="/tmp/seaweedfs-s3tests-$(date +%s)"
mkdir -p "$WEED_DATA_DIR"
weed -v 3 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
-dir="$WEED_DATA_DIR" \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=100 -volume.preStopSeconds=1 \
-master.port=9333 -volume.port=8080 -filer.port=8888 -s3.port=8000 -metricsPort=9324 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none &
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" &
pid=$!
# Wait for all SeaweedFS components to be ready
@@ -308,10 +308,7 @@ jobs:
s3tests/functional/test_s3.py::test_copy_object_ifnonematch_good \
s3tests/functional/test_s3.py::test_lifecycle_set \
s3tests/functional/test_s3.py::test_lifecycle_get \
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
s3tests/functional/test_s3.py::test_lifecycle_expiration \
s3tests/functional/test_s3.py::test_lifecyclev2_expiration \
s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
s3tests/functional/test_s3.py::test_lifecycle_set_filter
kill -9 $pid || true
# Clean up data directory
rm -rf "$WEED_DATA_DIR" || true
@@ -322,7 +319,7 @@ jobs:
timeout-minutes: 15
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go 1.x
uses: actions/setup-go@v6
@@ -368,7 +365,7 @@ jobs:
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=100 -volume.preStopSeconds=1 \
-master.port=9334 -volume.port=8081 -filer.port=8889 -s3.port=8001 -metricsPort=9325 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none &
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" &
pid=$!
# Wait for all SeaweedFS components to be ready
@@ -490,7 +487,7 @@ jobs:
timeout-minutes: 10
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go 1.x
uses: actions/setup-go@v6
@@ -526,7 +523,7 @@ jobs:
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=100 -volume.preStopSeconds=1 \
-master.port=9335 -volume.port=8082 -filer.port=8890 -s3.port=8002 -metricsPort=9326 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none &
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" &
pid=$!
# Wait for all SeaweedFS components to be ready
@@ -613,7 +610,7 @@ jobs:
timeout-minutes: 10
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go 1.x
uses: actions/setup-go@v6
@@ -636,7 +633,7 @@ jobs:
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=100 -volume.preStopSeconds=1 \
-master.port=9336 -volume.port=8083 -filer.port=8891 -s3.port=8003 -metricsPort=9327 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none &
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" &
pid=$!
# Wait for all SeaweedFS components to be ready
@@ -713,7 +710,7 @@ jobs:
timeout-minutes: 15
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go 1.x
uses: actions/setup-go@v6
@@ -794,7 +791,7 @@ jobs:
exit 1
fi
go install -tags "sqlite s3tests" -buildvcs=false
go install -tags "sqlite" -buildvcs=false
# Create clean data directory for this test run with unique timestamp and process ID
export WEED_DATA_DIR="/tmp/seaweedfs-sql-test-$(date +%s)-$$"
mkdir -p "$WEED_DATA_DIR"
@@ -818,7 +815,6 @@ jobs:
-volume.max=100 -volume.preStopSeconds=1 \
-master.port=9337 -volume.port=8085 -filer.port=8892 -s3.port=8004 -metricsPort=9328 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" \
-master.peers=none \
> /tmp/seaweedfs-sql-server.log 2>&1 &
pid=$!
@@ -1127,10 +1123,7 @@ jobs:
s3tests/functional/test_s3.py::test_copy_object_ifnonematch_good \
s3tests/functional/test_s3.py::test_lifecycle_set \
s3tests/functional/test_s3.py::test_lifecycle_get \
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
s3tests/functional/test_s3.py::test_lifecycle_expiration \
s3tests/functional/test_s3.py::test_lifecyclev2_expiration \
s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
s3tests/functional/test_s3.py::test_lifecycle_set_filter
kill -9 $pid || true
# Clean up data directory
rm -rf "$WEED_DATA_DIR" || true
@@ -1,263 +0,0 @@
name: Spark Integration Tests
on:
push:
paths:
- 'test/java/spark/**'
- 'other/java/hdfs3/**'
- 'other/java/client/**'
- '.github/workflows/spark-integration-tests.yml'
pull_request:
paths:
- 'test/java/spark/**'
- 'other/java/hdfs3/**'
- 'other/java/client/**'
- '.github/workflows/spark-integration-tests.yml'
workflow_dispatch:
permissions:
contents: read
checks: write
pull-requests: write
jobs:
spark-integration-tests:
name: Spark Integration Tests
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
# ========================================
# SETUP & BUILD
# ========================================
- name: Checkout code
uses: actions/checkout@v6
- name: Set up JDK 11
uses: actions/setup-java@v5
with:
java-version: '11'
distribution: 'temurin'
cache: maven
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: '1.24'
- name: Build SeaweedFS binary
run: |
echo "Building SeaweedFS binary (statically linked for Alpine)..."
cd weed
CGO_ENABLED=0 go build -o ../docker/weed
cd ../docker
ls -la weed filer.toml entrypoint.sh
file weed
echo "OK SeaweedFS binary built"
- name: Build SeaweedFS Java dependencies
run: |
echo "Building Java client..."
cd other/java/client
mvn clean install -U -DskipTests -Dgpg.skip=true -Dcentral.publishing.skip=true
echo "OK Java client built"
cd ../../..
echo "Building HDFS3 client..."
cd other/java/hdfs3
mvn clean install -U -DskipTests -Dgpg.skip=true -Dcentral.publishing.skip=true
echo "OK HDFS3 client built"
echo ""
echo "All Java dependencies installed to ~/.m2/repository"
# ========================================
# SPARK INTEGRATION TESTS (DOCKER)
# ========================================
- name: Start SeaweedFS services for tests
working-directory: test/java/spark
run: |
echo "=== Starting SeaweedFS Services for Tests ==="
docker compose down -v || true
docker compose build --no-cache
docker compose up -d seaweedfs-master seaweedfs-volume seaweedfs-filer
echo "Waiting for services..."
for i in {1..30}; do
if curl -f http://localhost:8888/ > /dev/null 2>&1; then
echo "OK SeaweedFS filer is ready!"
break
fi
if [ $i -eq 30 ]; then
echo "FAILED Services failed to start"
docker compose ps -a
docker compose logs
exit 1
fi
echo "Waiting... ($i/30)"
sleep 2
done
curl -f http://localhost:9333/cluster/status || exit 1
echo "OK All services healthy"
- name: Prepare Maven repository for Docker
working-directory: test/java/spark
run: |
echo "Copying Maven artifacts for Docker container..."
mkdir -p .m2/repository/com
cp -r ~/.m2/repository/com/seaweedfs .m2/repository/com/
echo "OK Maven artifacts copied"
- name: Run Spark integration tests
working-directory: test/java/spark
run: |
echo "=== Running Spark Integration Tests ==="
docker compose up --abort-on-container-exit spark-tests
echo ""
echo "=== Test Logs ==="
docker compose logs spark-tests | tail -100
- name: Stop test services
if: always()
working-directory: test/java/spark
run: docker compose down -v
- name: Upload test results
if: always()
uses: actions/upload-artifact@v5
with:
name: spark-test-results
path: test/java/spark/target/surefire-reports/
retention-days: 30
- name: Publish test report
if: always()
uses: dorny/test-reporter@v2
with:
name: Spark Test Results
path: test/java/spark/target/surefire-reports/*.xml
reporter: java-junit
fail-on-error: true
# ========================================
# SPARK EXAMPLE (HOST-BASED) - DISABLED
# Note: Host-based example doesn't work with Docker networking
# because master returns Docker hostnames (seaweedfs-volume)
# which are not resolvable from the host.
# The Docker-based tests above are sufficient.
# ========================================
- name: Cache Apache Spark
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
id: cache-spark
uses: actions/cache@v4
with:
path: spark-3.5.0-bin-hadoop3
key: spark-3.5.0-hadoop3
- name: Download Apache Spark
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && steps.cache-spark.outputs.cache-hit != 'true'
run: |
echo "Downloading Apache Spark 3.5.0..."
wget -q https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz
tar xzf spark-3.5.0-bin-hadoop3.tgz
echo "OK Spark downloaded"
- name: Start SeaweedFS services for example
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
working-directory: test/java/spark
run: |
echo "=== Starting SeaweedFS Services for Example ==="
docker compose down -v || true
docker compose build --no-cache
docker compose up -d seaweedfs-master seaweedfs-volume seaweedfs-filer
echo "Waiting for services..."
for i in {1..30}; do
if curl -f http://localhost:8888/ > /dev/null 2>&1; then
echo "OK SeaweedFS filer is ready!"
break
fi
if [ $i -eq 30 ]; then
echo "FAILED Services failed to start"
docker compose ps -a
docker compose logs
exit 1
fi
echo "Waiting... ($i/30)"
sleep 2
done
curl -f http://localhost:9333/cluster/status || exit 1
echo "OK All services healthy"
- name: Clean target directory
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
working-directory: test/java/spark
run: |
# Force remove target directory to avoid permission issues
sudo rm -rf target || rm -rf target || true
echo "OK Target directory cleaned"
- name: Build project for example
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
working-directory: test/java/spark
run: mvn package -DskipTests
- name: Run Spark example application
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
working-directory: test/java/spark
run: |
echo "=== Running Spark Example Application ==="
export SPARK_HOME=$(pwd)/../../../spark-3.5.0-bin-hadoop3
$SPARK_HOME/bin/spark-submit \
--class seaweed.spark.SparkSeaweedFSExample \
--master local[2] \
--conf spark.hadoop.fs.seaweedfs.impl=seaweed.hdfs.SeaweedFileSystem \
--conf spark.hadoop.fs.seaweed.filer.host=localhost \
--conf spark.hadoop.fs.seaweed.filer.port=8888 \
--conf spark.hadoop.fs.seaweed.filer.port.grpc=18888 \
--conf spark.hadoop.fs.seaweed.replication="" \
target/seaweedfs-spark-integration-tests-1.0-SNAPSHOT.jar \
seaweedfs://localhost:8888/ci-spark-output
echo "OK Example completed"
- name: Verify example output
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
run: |
echo "Verifying output..."
curl -s http://localhost:8888/ci-spark-output/ || echo "Output listing unavailable"
- name: Stop example services
if: false && always() && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
working-directory: test/java/spark
run: docker compose down -v
# ========================================
# DIAGNOSTICS
# ========================================
- name: Display diagnostics on failure
if: failure()
working-directory: test/java/spark
run: |
echo "=== Container Status ==="
docker compose ps -a
echo ""
echo "=== Master Logs ==="
docker compose logs seaweedfs-master
echo ""
echo "=== Volume Logs ==="
docker compose logs seaweedfs-volume
echo ""
echo "=== Filer Logs ==="
docker compose logs seaweedfs-filer
echo ""
echo "=== Volume List ==="
docker compose exec -T seaweedfs-master weed shell <<EOF || echo "Failed"
volume.list
exit
EOF
echo ""
echo "=== Cluster Status ==="
curl -s http://localhost:9333/dir/status | jq '.' || curl -s http://localhost:9333/dir/status
@@ -20,7 +20,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v5
- uses: actions/setup-go@v6
with:
@@ -34,7 +34,7 @@ jobs:
run: |
set -e
mkdir -p /tmp/data
./weed -v=3 server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json -master.peers=none > weed.log 2>&1 &
./weed server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json &
until curl -s http://localhost:8333/ > /dev/null; do sleep 1; done
- name: Setup Caddy
@@ -77,35 +77,3 @@ jobs:
aws --no-verify-ssl s3 cp --no-progress s3://bucket/test-multipart downloaded
diff -q generated downloaded
rm -f generated downloaded
- name: Test GetObject with If-Match
run: |
set -e
dd if=/dev/urandom of=generated bs=1M count=32
ETAG=$(aws --no-verify-ssl s3api put-object --bucket bucket --key test-get-obj --body generated | jq -r .ETag)
# jq -r already removes quotes, so use ETAG directly (handles both simple and multipart ETags)
aws --no-verify-ssl s3api get-object --bucket bucket --key test-get-obj --if-match "$ETAG" downloaded
diff -q generated downloaded
rm -f generated downloaded
- name: Show server logs on failure
if: failure()
run: |
echo "========================================="
echo "SeaweedFS Server Logs"
echo "========================================="
# Note: weed.log is relative to working-directory (weed/)
if [ -f weed.log ]; then
cat weed.log
else
echo "No weed.log file found"
fi
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v5
with:
name: seaweedfs-logs
# Note: actions don't use defaults.run.working-directory, so path is relative to workspace root
path: weed/weed.log
retention-days: 3
-242
View File
@@ -1,242 +0,0 @@
# Bucket Policy Engine Integration - Complete
## Summary
Successfully integrated the `policy_engine` package to evaluate bucket policies for **all requests** (both anonymous and authenticated). This provides comprehensive AWS S3-compatible bucket policy support.
## What Changed
### 1. **New File: `s3api_bucket_policy_engine.go`**
Created a wrapper around `policy_engine.PolicyEngine` to:
- Load bucket policies from filer entries
- Sync policies from the bucket config cache
- Evaluate policies for any request (bucket, object, action, principal)
- Return structured results (allowed, evaluated, error)
### 2. **Modified: `s3api_server.go`**
- Added `policyEngine *BucketPolicyEngine` field to `S3ApiServer` struct
- Initialized the policy engine in `NewS3ApiServerWithStore()`
- Linked `IdentityAccessManagement` back to `S3ApiServer` for policy evaluation
### 3. **Modified: `auth_credentials.go`**
- Added `s3ApiServer *S3ApiServer` field to `IdentityAccessManagement` struct
- Added `buildPrincipalARN()` helper to convert identities to AWS ARN format
- **Integrated bucket policy evaluation into the authentication flow:**
- Policies are now checked **before** IAM/identity-based permissions
- Explicit `Deny` in bucket policy blocks access immediately
- Explicit `Allow` in bucket policy grants access and **bypasses IAM checks** (enables cross-account access)
- If no policy exists, falls through to normal IAM checks
- Policy evaluation errors result in access denial (fail-close security)
### 4. **Modified: `s3api_bucket_config.go`**
- Added policy engine sync when bucket configs are loaded
- Ensures policies are loaded into the engine for evaluation
### 5. **Modified: `auth_credentials_subscribe.go`**
- Added policy engine sync when bucket metadata changes
- Keeps the policy engine up-to-date via event-driven updates
## How It Works
### Anonymous Requests
```
1. Request comes in (no credentials)
2. Check ACL-based public access → if public, allow
3. Check bucket policy for anonymous ("*") access → if allowed, allow
4. Otherwise, deny
```
### Authenticated Requests (NEW!)
```
1. Request comes in (with credentials)
2. Authenticate user → get Identity
3. Build principal ARN (e.g., "arn:aws:iam::123456:user/bob")
4. Check bucket policy:
- If DENY → reject immediately
- If ALLOW → grant access immediately (bypasses IAM checks)
- If no policy or no matching statements → continue to step 5
5. Check IAM/identity-based permissions (only if not already allowed by bucket policy)
6. Allow or deny based on identity permissions
```
## Policy Evaluation Flow
```
┌─────────────────────────────────────────────────────────┐
│ Request (GET /bucket/file) │
└───────────────────────────┬─────────────────────────────┘
│
┌───────────▼──────────┐
│ Authenticate User │
│ (or Anonymous) │
└───────────┬──────────┘
│
┌───────────▼──────────────────────────────┐
│ Build Principal ARN │
│ - Anonymous: "*" │
│ - User: "arn:aws:iam::123456:user/bob" │
└───────────┬──────────────────────────────┘
│
┌───────────▼──────────────────────────────┐
│ Evaluate Bucket Policy (PolicyEngine) │
│ - Action: "s3:GetObject" │
│ - Resource: "arn:aws:s3:::bucket/file" │
│ - Principal: (from above) │
└───────────┬──────────────────────────────┘
│
┌─────────────┼─────────────┐
│ │ │
DENY │ ALLOW │ NO POLICY
│ │ │
▼ ▼ ▼
Reject Request Grant Access Continue
│
┌───────────────────┘
│
┌────────────▼─────────────┐
│ IAM/Identity Check │
│ (identity.canDo) │
└────────────┬─────────────┘
│
┌─────────┴─────────┐
│ │
ALLOW │ DENY │
▼ ▼
Grant Access Reject Request
```
## Example Policies That Now Work
### 1. **Public Read Access** (Anonymous)
```json
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": "*",
"Action": "s3:GetObject",
"Resource": "arn:aws:s3:::mybucket/*"
}]
}
```
- Anonymous users can read all objects
- Authenticated users are also evaluated against this policy. If they don't match an explicit `Allow` for this action, they will fall back to their own IAM permissions
### 2. **Grant Access to Specific User** (Authenticated)
```json
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {"AWS": "arn:aws:iam::123456789012:user/bob"},
"Action": ["s3:GetObject", "s3:PutObject"],
"Resource": "arn:aws:s3:::mybucket/shared/*"
}]
}
```
- User "bob" can read/write objects in `/shared/` prefix
- Other users cannot (unless granted by their IAM policies)
### 3. **Deny Access to Specific Path** (Both)
```json
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Deny",
"Principal": "*",
"Action": "s3:*",
"Resource": "arn:aws:s3:::mybucket/confidential/*"
}]
}
```
- **No one** can access `/confidential/` objects
- Denies override all other allows (AWS policy evaluation rules)
## Performance Characteristics
### Policy Loading
- **Cold start**: Policy loaded from filer → parsed → compiled → cached
- **Warm path**: Policy retrieved from `BucketConfigCache` (already parsed)
- **Updates**: Event-driven sync via metadata subscription (real-time)
### Policy Evaluation
- **Compiled policies**: Pre-compiled regex patterns and matchers
- **Pattern cache**: Regex patterns cached with LRU eviction (max 1000)
- **Fast path**: Common patterns (`*`, exact matches) optimized
- **Case sensitivity**: Actions case-insensitive, resources case-sensitive (AWS-compatible)
### Overhead
- **Anonymous requests**: Minimal (policy already checked, now using compiled engine)
- **Authenticated requests**: ~1-2ms added for policy evaluation (compiled patterns)
- **No policy**: Near-zero overhead (quick indeterminate check)
## Testing
All tests pass:
```bash
✅ TestBucketPolicyValidationBasics
✅ TestPrincipalMatchesAnonymous
✅ TestActionToS3Action
✅ TestResourceMatching
✅ TestMatchesPatternRegexEscaping (security tests)
✅ TestActionMatchingCaseInsensitive
✅ TestResourceMatchingCaseSensitive
✅ All policy_engine package tests (30+ tests)
```
## Security Improvements
1. **Regex Metacharacter Escaping**: Patterns like `*.json` properly match only files ending in `.json` (not `filexjson`)
2. **Case-Insensitive Actions**: S3 actions matched case-insensitively per AWS spec
3. **Case-Sensitive Resources**: Resource paths matched case-sensitively for security
4. **Pattern Cache Size Limit**: Prevents DoS attacks via unbounded cache growth
5. **Principal Validation**: Supports `[]string` for manually constructed policies
## AWS Compatibility
The implementation follows AWS S3 bucket policy evaluation rules:
1. **Explicit Deny** always wins (checked first)
2. **Explicit Allow** grants access (checked second)
3. **Default Deny** if no matching statements (implicit)
4. Bucket policies work alongside IAM policies (both are evaluated)
## Files Changed
```
Modified:
weed/s3api/auth_credentials.go (+47 lines)
weed/s3api/auth_credentials_subscribe.go (+8 lines)
weed/s3api/s3api_bucket_config.go (+8 lines)
weed/s3api/s3api_server.go (+5 lines)
New:
weed/s3api/s3api_bucket_policy_engine.go (115 lines)
```
## Migration Notes
- **Backward Compatible**: Existing setups without bucket policies work unchanged
- **No Breaking Changes**: All existing ACL and IAM-based authorization still works
- **Additive Feature**: Bucket policies are an additional layer of authorization
- **Performance**: Minimal impact on existing workloads
## Future Enhancements
Potential improvements (not implemented yet):
- [ ] Condition support (IP address, time-based, etc.) - already in policy_engine
- [ ] Cross-account policies (different AWS accounts)
- [ ] Policy validation API endpoint
- [ ] Policy simulation/testing tool
- [ ] Metrics for policy evaluations (allow/deny counts)
## Conclusion
Bucket policies now work for **all requests** in SeaweedFS S3 API:
- ✅ Anonymous requests (public access)
- ✅ Authenticated requests (user-specific policies)
- ✅ High performance (compiled policies, caching)
- ✅ AWS-compatible (follows AWS evaluation rules)
- ✅ Secure (proper escaping, case sensitivity)
The integration is complete, tested, and ready for use!
-2
View File
@@ -520,8 +520,6 @@ cd seaweedfs/weed && make install
Once this is done, you will find the executable "weed" in your `$GOPATH/bin` directory
For more installation options, including how to run with Docker, see the [Getting Started guide](https://github.com/seaweedfs/seaweedfs/wiki/Getting-Started).
[Back to TOC](#table-of-contents)
## Disk Related Topics ##
+1 -6
View File
@@ -3,10 +3,6 @@ FROM ubuntu:22.04
LABEL author="Chris Lu"
# Use faster mirrors and optimize package installation
# Note: This e2e test image intentionally runs as root for simplicity and compatibility.
# Production images (Dockerfile.go_build) use proper user isolation with su-exec.
# For testing purposes, running as root avoids permission complexities and dependency
# on Alpine-specific tools like su-exec (not available in Ubuntu repos).
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y \
--no-install-recommends \
@@ -14,7 +10,6 @@ RUN apt-get update && \
curl \
fio \
fuse \
ca-certificates \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& rm -rf /tmp/* \
@@ -23,7 +18,7 @@ RUN mkdir -p /etc/seaweedfs /data/filerldb2
COPY ./weed /usr/bin/
COPY ./filer.toml /etc/seaweedfs/filer.toml
COPY ./entrypoint_e2e.sh /entrypoint.sh
COPY ./entrypoint.sh /entrypoint.sh
# volume server grpc port
EXPOSE 18080
-131
View File
@@ -1,131 +0,0 @@
FROM golang:1.24 AS builder
RUN apt-get update && \
apt-get install -y build-essential wget ca-certificates && \
rm -rf /var/lib/apt/lists/*
ARG FDB_VERSION=7.4.5
ENV FDB_VERSION=${FDB_VERSION}
ARG TARGETARCH
# Install FoundationDB client libraries with SHA256 checksum verification
# Known SHA256 checksums for FoundationDB client packages (verified 2025-01-19)
# To add checksums for new versions: run docker/get_fdb_checksum.sh <version> <arch>
RUN cd /tmp && \
case "${TARGETARCH}" in \
"amd64") FDB_ARCH="amd64"; PACKAGE_ARCH="amd64" ;; \
"arm64") FDB_ARCH="arm64"; PACKAGE_ARCH="aarch64" ;; \
*) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \
esac && \
case "${FDB_VERSION}_${FDB_ARCH}" in \
"7.4.5_amd64") \
EXPECTED_SHA256="eea6b98cf386a0848655b2e196d18633662a7440a7ee061c10e32153c7e7e112" ;; \
"7.4.5_arm64") \
EXPECTED_SHA256="f2176b86b7e1b561c3632b4e6e7efb82e3b8f57c2ff0d0ac4671e742867508aa" ;; \
"7.3.43_amd64") \
EXPECTED_SHA256="c3fa0a59c7355b914a1455dac909238d5ea3b6c6bc7b530af8597e6487c1651a" ;; \
"7.3.43_arm64") \
echo "ERROR: FoundationDB ${FDB_VERSION} does not publish arm64 client packages." >&2; \
echo "Please upgrade to 7.4.5+ when targeting arm64." >&2; \
exit 1 ;; \
*) \
echo "ERROR: No checksum available for FDB version ${FDB_VERSION} on ${FDB_ARCH}" >&2; \
echo "This is a security requirement. To add verification:" >&2; \
echo " 1. Run: docker/get_fdb_checksum.sh ${FDB_VERSION} ${FDB_ARCH}" >&2; \
echo " 2. Add the checksum to this Dockerfile" >&2; \
echo "Refusing to proceed without checksum verification." >&2; \
exit 1 ;; \
esac && \
PACKAGE="foundationdb-clients_${FDB_VERSION}-1_${PACKAGE_ARCH}.deb" && \
wget --timeout=30 --tries=3 https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/${PACKAGE} && \
echo "${EXPECTED_SHA256} ${PACKAGE}" | sha256sum -c - || \
(echo "ERROR: Checksum verification failed for FoundationDB ${FDB_VERSION} (${FDB_ARCH})" >&2; \
echo "Expected: ${EXPECTED_SHA256}" >&2; \
echo "This indicates either a corrupted download or potential tampering." >&2; \
exit 1) && \
dpkg -i ${PACKAGE} && \
rm ${PACKAGE}
# Set up FoundationDB environment variables for CGO
ENV CGO_CFLAGS="-I/usr/include/foundationdb"
ENV CGO_LDFLAGS="-lfdb_c"
# build SeaweedFS sources; prefer local context but fall back to git clone if context only has docker files
ARG SOURCE_REF=master
WORKDIR /go/src/github.com/seaweedfs/seaweedfs
COPY . .
RUN set -euo pipefail && \
if [ ! -d weed ]; then \
echo "Local build context does not include SeaweedFS sources; cloning ${SOURCE_REF}" >&2; \
mkdir -p /tmp/local-context && cp -a /go/src/github.com/seaweedfs/seaweedfs/. /tmp/local-context && \
cd / && rm -rf /go/src/github.com/seaweedfs/seaweedfs && \
git clone --depth 1 --branch ${SOURCE_REF} https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs && \
cp -a /tmp/local-context/. /go/src/github.com/seaweedfs/seaweedfs/docker/ && \
rm -rf /tmp/local-context && \
cd /go/src/github.com/seaweedfs/seaweedfs; \
fi && \
cd weed \
&& COMMIT_SHA=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") \
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${COMMIT_SHA}" \
&& go install -tags "5BytesOffset foundationdb" -ldflags "${LDFLAGS}"
FROM debian:bookworm-slim AS final
LABEL author="Chris Lu"
# Install runtime dependencies first
RUN apt-get update && \
apt-get install -y --no-install-recommends \
ca-certificates \
fuse \
wget && \
rm -rf /var/lib/apt/lists/*
# Reuse FoundationDB artifacts installed during the build stage
COPY --from=builder /usr/lib/libfdb_c* /usr/lib/
COPY --from=builder /usr/lib/foundationdb /usr/lib/foundationdb
COPY --from=builder /usr/bin/fdb* /usr/bin/
RUN ldconfig
# Copy SeaweedFS binary and configuration
COPY --from=builder /go/bin/weed /usr/bin/
RUN mkdir -p /etc/seaweedfs
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_foundationdb.toml /etc/seaweedfs/filer.toml
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
# Create non-root user
RUN groupadd -g 1000 seaweed && \
useradd -u 1000 -g seaweed -s /bin/bash -m seaweed
# volume server gprc port
EXPOSE 18080
# volume server http port
EXPOSE 8080
# filer server gprc port
EXPOSE 18888
# filer server http port
EXPOSE 8888
# master server shared gprc port
EXPOSE 19333
# master server shared http port
EXPOSE 9333
# s3 server http port
EXPOSE 8333
# webdav server http port
EXPOSE 7333
# Create data directory and set proper ownership for seaweed user
RUN mkdir -p /data && \
chown -R seaweed:seaweed /data && \
chown -R seaweed:seaweed /etc/seaweedfs && \
chmod 755 /entrypoint.sh
VOLUME /data
WORKDIR /data
# Switch to non-root user
USER seaweed
ENTRYPOINT ["/entrypoint.sh"]
+6 -23
View File
@@ -1,17 +1,10 @@
FROM golang:1.24-alpine as builder
RUN apk add git g++ fuse
RUN mkdir -p /go/src/github.com/seaweedfs/
RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
ARG BRANCH=${BRANCH:-master}
# Clone with full history and all tags to ensure all commits are available
RUN git clone --no-single-branch --tags https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
ARG TAGS
RUN cd /go/src/github.com/seaweedfs/seaweedfs && \
(git checkout $BRANCH || \
(echo "Checkout failed, fetching all history..." && \
git fetch --all --tags --prune && \
git checkout $BRANCH) || \
(echo "ERROR: Branch/commit $BRANCH not found in repository" && \
echo "Available branches:" && git branch -a && exit 1))
RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
&& CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
@@ -22,14 +15,7 @@ COPY --from=builder /go/bin/weed /usr/bin/
RUN mkdir -p /etc/seaweedfs
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
# FIPS 140-3 mode is ON by default (Go 1.24+)
# To disable: docker run -e GODEBUG=fips140=off ...
# Install dependencies and create non-root user
RUN apk add --no-cache fuse su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
RUN apk add fuse # for weed mount
# volume server gprc port
EXPOSE 18080
@@ -48,14 +34,11 @@ EXPOSE 8333
# webdav server http port
EXPOSE 7333
# Create data directory and set proper ownership for seaweed user
RUN mkdir -p /data/filerldb2 && \
chown -R seaweed:seaweed /data && \
chown -R seaweed:seaweed /etc/seaweedfs && \
chmod 755 /entrypoint.sh
RUN mkdir -p /data/filerldb2
VOLUME /data
WORKDIR /data
# Entrypoint will handle permission fixes and user switching
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
+8 -13
View File
@@ -1,15 +1,13 @@
FROM alpine AS final
LABEL author="Chris Lu"
COPY ./weed /usr/bin/weed
RUN chmod +x /usr/bin/weed && ls -la /usr/bin/weed
COPY ./weed /usr/bin/
COPY ./weed_pub* /usr/bin/
COPY ./weed_sub* /usr/bin/
RUN mkdir -p /etc/seaweedfs
COPY ./filer.toml /etc/seaweedfs/filer.toml
COPY ./entrypoint.sh /entrypoint.sh
# Install dependencies and create non-root user
RUN apk add --no-cache fuse curl su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
RUN apk add fuse # for weed mount
RUN apk add curl # for health checks
# volume server grpc port
EXPOSE 18080
@@ -28,14 +26,11 @@ EXPOSE 8333
# webdav server http port
EXPOSE 7333
# Create data directory and set proper ownership for seaweed user
RUN mkdir -p /data/filerldb2 && \
chown -R seaweed:seaweed /data && \
chown -R seaweed:seaweed /etc/seaweedfs && \
chmod 755 /entrypoint.sh
RUN mkdir -p /data/filerldb2
VOLUME /data
WORKDIR /data
# Entrypoint will handle permission fixes and user switching
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
+4 -11
View File
@@ -32,11 +32,7 @@ COPY --from=builder /go/bin/weed /usr/bin/
RUN mkdir -p /etc/seaweedfs
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
# Install dependencies and create non-root user
RUN apk add --no-cache fuse snappy gflags su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
RUN apk add fuse snappy gflags
# volume server gprc port
EXPOSE 18080
@@ -55,15 +51,12 @@ EXPOSE 8333
# webdav server http port
EXPOSE 7333
# Create data directory and set proper ownership for seaweed user
RUN mkdir -p /data/filer_rocksdb && \
chown -R seaweed:seaweed /data && \
chown -R seaweed:seaweed /etc/seaweedfs && \
chmod 755 /entrypoint.sh
RUN mkdir -p /data/filer_rocksdb
VOLUME /data
WORKDIR /data
# Entrypoint will handle permission fixes and user switching
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
+4 -11
View File
@@ -15,11 +15,7 @@ COPY --from=builder /go/bin/weed /usr/bin/
RUN mkdir -p /etc/seaweedfs
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
# Install dependencies and create non-root user
RUN apk add --no-cache fuse snappy gflags tmux su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
RUN apk add fuse snappy gflags tmux
# volume server gprc port
EXPOSE 18080
@@ -38,15 +34,12 @@ EXPOSE 8333
# webdav server http port
EXPOSE 7333
# Create data directory and set proper ownership for seaweed user
RUN mkdir -p /data/filer_rocksdb && \
chown -R seaweed:seaweed /data && \
chown -R seaweed:seaweed /etc/seaweedfs && \
chmod 755 /entrypoint.sh
RUN mkdir -p /data/filer_rocksdb
VOLUME /data
WORKDIR /data
# Entrypoint will handle permission fixes and user switching
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
+1 -5
View File
@@ -5,19 +5,15 @@ all: gen
gen: dev
cgo ?= 0
ldflags_extra ?= -extldflags -static
binary:
export SWCOMMIT=$(shell git rev-parse --short HEAD)
export SWLDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(SWCOMMIT)"
cd ../weed && CGO_ENABLED=$(cgo) GOOS=linux go build $(options) -tags "$(tags)" -ldflags "-s -w $(ldflags_extra) $(SWLDFLAGS)" -o weed_binary && mv weed_binary ../docker/weed
cd ../weed && CGO_ENABLED=$(cgo) GOOS=linux go build $(options) -tags "$(tags)" -ldflags "-s -w -extldflags -static $(SWLDFLAGS)" -o weed_binary && mv weed_binary ../docker/weed
cd ../other/mq_client_example/agent_pub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv agent_pub_record ../../../docker/
cd ../other/mq_client_example/agent_sub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv agent_sub_record ../../../docker/
# Race detector requires CGO and dynamic linking - don't use -static
binary_race: options = -race
binary_race: cgo = 1
binary_race: ldflags_extra =
binary_race: binary
build: binary
+4 -4
View File
@@ -10,10 +10,10 @@
scripts = """
lock
ec.encode -fullPercent=95 -quietFor=1h
ec.rebuild -apply
ec.balance -apply
volume.balance -apply
volume.fix.replication -apply
ec.rebuild -force
ec.balance -force
volume.balance -force
volume.fix.replication -force
unlock
"""
sleep_minutes = 17 # sleep minutes between each script execution
+2 -34
View File
@@ -1,37 +1,5 @@
#!/bin/sh
# Enable FIPS 140-3 mode by default (Go 1.24+)
# To disable: docker run -e GODEBUG=fips140=off ...
export GODEBUG="${GODEBUG:+$GODEBUG,}fips140=on"
# Fix permissions for mounted volumes
# If /data is mounted from host, it might have different ownership
# Fix this by ensuring seaweed user owns the directory
if [ "$(id -u)" = "0" ]; then
# Running as root, check and fix permissions if needed
SEAWEED_UID=$(id -u seaweed)
SEAWEED_GID=$(id -g seaweed)
# Verify seaweed user and group exist
if [ -z "$SEAWEED_UID" ] || [ -z "$SEAWEED_GID" ]; then
echo "Error: 'seaweed' user or group not found. Cannot fix permissions." >&2
exit 1
fi
DATA_UID=$(stat -c '%u' /data 2>/dev/null)
DATA_GID=$(stat -c '%g' /data 2>/dev/null)
# Only run chown -R if ownership doesn't match (much faster for subsequent starts)
echo "Fixing /data ownership for seaweed user (uid=$SEAWEED_UID, gid=$SEAWEED_GID)"
if ! chown -R seaweed:seaweed /data; then
echo "Warning: Failed to change ownership of /data. This may cause permission errors." >&2
echo "If /data is read-only or has mount issues, the application may fail to start." >&2
fi
# Use su-exec to drop privileges and run as seaweed user
exec su-exec seaweed "$0" "$@"
fi
isArgPassed() {
arg="$1"
argWithEqualSign="$1="
@@ -40,10 +8,10 @@ isArgPassed() {
passedArg="$1"
shift
case $passedArg in
"$arg")
$arg)
return 0
;;
"$argWithEqualSign"*)
$argWithEqualSign*)
return 0
;;
esac
-86
View File
@@ -1,86 +0,0 @@
#!/bin/bash
set -e
# Simplified entrypoint for e2e testing
#
# This script intentionally runs as root for e2e test environments to:
# 1. Simplify test setup and avoid permission-related test failures
# 2. Eliminate dependency on Alpine-specific tools (su-exec) since we use Ubuntu base
# 3. Focus testing on application logic rather than container security
#
# IMPORTANT: Production deployments should use Dockerfile.go_build with proper
# user isolation via su-exec. This simplified approach is ONLY for testing.
isArgPassed() {
arg="$1"
argWithEqualSign="$1="
shift
while [ $# -gt 0 ]; do
passedArg="$1"
shift
case $passedArg in
"$arg")
return 0
;;
"$argWithEqualSign"*)
return 0
;;
esac
done
return 1
}
case "$1" in
'master')
ARGS="-mdir=/data -volumePreallocate -volumeSizeLimitMB=1024"
shift
exec /usr/bin/weed -logtostderr=true master $ARGS "$@"
;;
'volume')
ARGS="-dir=/data -max=0"
if isArgPassed "-max" "$@"; then
ARGS="-dir=/data"
fi
shift
exec /usr/bin/weed -logtostderr=true volume $ARGS "$@"
;;
'server')
ARGS="-dir=/data -volume.max=0 -master.volumePreallocate -master.volumeSizeLimitMB=1024"
if isArgPassed "-volume.max" "$@"; then
ARGS="-dir=/data -master.volumePreallocate -master.volumeSizeLimitMB=1024"
fi
shift
exec /usr/bin/weed -logtostderr=true server $ARGS "$@"
;;
'filer')
ARGS=""
shift
exec /usr/bin/weed -logtostderr=true filer $ARGS "$@"
;;
's3')
ARGS="-domainName=$S3_DOMAIN_NAME -key.file=$S3_KEY_FILE -cert.file=$S3_CERT_FILE"
shift
exec /usr/bin/weed -logtostderr=true s3 $ARGS "$@"
;;
'mount')
shift
exec /usr/bin/weed -logtostderr=true mount "$@"
;;
'shell')
ARGS="-cluster=$SHELL_CLUSTER -filer=$SHELL_FILER -filerGroup=$SHELL_FILER_GROUP -master=$SHELL_MASTER -options=$SHELL_OPTIONS"
shift
exec echo "$@" | /usr/bin/weed -logtostderr=true shell $ARGS
;;
*)
exec /usr/bin/weed "$@"
;;
esac
-19
View File
@@ -1,19 +0,0 @@
[filer.options]
# with http DELETE, by default the filer would check whether a folder is empty.
# recursive_delete will delete all sub folders and files, similar to "rm -Rf"
recursive_delete = false
####################################################
# FoundationDB store
####################################################
[foundationdb]
enabled = true
cluster_file = "/etc/foundationdb/fdb.cluster"
api_version = 740
# Optional: timeout for FDB operations (default: 10s)
# timeout = "10s"
# Optional: max retry delay for retryable errors (default: 1s)
# max_retry_delay = "1s"
# Optional: directory prefix for storing SeaweedFS data (default: "seaweedfs")
# directory_prefix = "seaweedfs"
-61
View File
@@ -1,61 +0,0 @@
#!/bin/bash
# Helper script to get SHA256 checksum for FoundationDB client package
# Usage: ./get_fdb_checksum.sh <version> [arch]
# Example: ./get_fdb_checksum.sh 7.4.5 amd64
# Example: ./get_fdb_checksum.sh 7.4.5 arm64
set -euo pipefail
if [ $# -lt 1 ] || [ $# -gt 2 ]; then
echo "Usage: $0 <fdb_version> [arch]" >&2
echo "Example: $0 7.4.5" >&2
echo "Example: $0 7.4.5 arm64" >&2
exit 1
fi
FDB_VERSION="$1"
FDB_ARCH="${2:-amd64}"
case "$FDB_ARCH" in
"amd64")
CANONICAL_ARCH="amd64"
PACKAGE_ARCH="amd64"
;;
"arm64"|"aarch64")
CANONICAL_ARCH="arm64"
PACKAGE_ARCH="aarch64"
;;
*)
echo "Error: Architecture must be 'amd64', 'arm64', or 'aarch64'" >&2
exit 1
;;
esac
PACKAGE="foundationdb-clients_${FDB_VERSION}-1_${PACKAGE_ARCH}.deb"
URL="https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/${PACKAGE}"
echo "Downloading FoundationDB ${FDB_VERSION} client package for ${FDB_ARCH}..."
echo "URL: ${URL}"
echo ""
# Download to temp directory
TEMP_DIR=$(mktemp -d)
trap 'rm -rf "${TEMP_DIR}"' EXIT
cd "${TEMP_DIR}"
if wget --timeout=30 --tries=3 -q "${URL}"; then
CHECKSUM=$(sha256sum "${PACKAGE}" | awk '{print $1}')
echo "✓ Download successful"
echo ""
echo "SHA256 Checksum:"
echo "${CHECKSUM}"
echo ""
echo "Add this to Dockerfile.foundationdb_large:"
echo " \"${FDB_VERSION}_${CANONICAL_ARCH}\") \\"
echo " EXPECTED_SHA256=\"${CHECKSUM}\" ;; \\"
else
echo "✗ Failed to download package from ${URL}" >&2
echo "Please verify the version number, architecture, and URL" >&2
exit 1
fi
+4 -4
View File
@@ -7,14 +7,14 @@ services:
- 9333:9333
- 19333:19333
- 9324:9324
command: 'master -ip=master -ip.bind=0.0.0.0 -metricsPort=9324'
command: "master -ip=master -ip.bind=0.0.0.0 -metricsPort=9324"
volume:
image: chrislusf/seaweedfs # use a remote image
ports:
- 8080:8080
- 18080:18080
- 9325:9325
command: 'volume -ip=volume -mserver="master:9333" -ip.bind=0.0.0.0 -port=8080 -metricsPort=9325'
command: 'volume -mserver="master:9333" -ip.bind=0.0.0.0 -port=8080 -metricsPort=9325'
depends_on:
- master
filer:
@@ -23,7 +23,7 @@ services:
- 8888:8888
- 18888:18888
- 9326:9326
command: 'filer -ip=filer -master="master:9333" -ip.bind=0.0.0.0 -metricsPort=9326'
command: 'filer -master="master:9333" -ip.bind=0.0.0.0 -metricsPort=9326'
tty: true
stdin_open: true
depends_on:
@@ -54,6 +54,6 @@ services:
- 9000:9090
volumes:
- ./prometheus:/etc/prometheus
command: '--web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml'
command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml
depends_on:
- s3
+3 -3
View File
@@ -6,13 +6,13 @@ services:
ports:
- 9333:9333
- 19333:19333
command: 'master -ip=master -ip.bind=0.0.0.0'
command: "master -ip=master"
volume:
image: chrislusf/seaweedfs:dev # use a remote dev image
ports:
- 8080:8080
- 18080:18080
command: 'volume -ip=volume -mserver="master:9333" -ip.bind=0.0.0.0 -port=8080'
command: 'volume -mserver="master:9333" -port=8080 -ip=volume'
depends_on:
- master
filer:
@@ -20,7 +20,7 @@ services:
ports:
- 8888:8888
- 18888:18888
command: 'filer -ip=filer -master="master:9333" -ip.bind=0.0.0.0'
command: 'filer -master="master:9333" -ip.bind=0.0.0.0'
depends_on:
- master
- volume
+76 -74
View File
@@ -7,7 +7,7 @@ toolchain go1.24.1
require (
cloud.google.com/go v0.121.6 // indirect
cloud.google.com/go/pubsub v1.50.1
cloud.google.com/go/storage v1.57.1
cloud.google.com/go/storage v1.57.0
github.com/Shopify/sarama v1.38.1
github.com/aws/aws-sdk-go v1.55.8
github.com/beorn7/perks v1.0.1 // indirect
@@ -50,10 +50,10 @@ require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.12
github.com/karlseguin/ccache/v2 v2.0.8
github.com/klauspost/compress v1.18.2
github.com/klauspost/compress v1.18.1
github.com/klauspost/reedsolomon v1.12.5
github.com/kurin/blazer v0.5.3
github.com/linxGnu/grocksdb v1.10.3
github.com/linxGnu/grocksdb v1.10.2
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -67,7 +67,7 @@ require (
github.com/prometheus/client_golang v1.23.2
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.19.2
github.com/prometheus/procfs v0.17.0
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/seaweedfs/goexif v1.0.3
@@ -90,25 +90,25 @@ require (
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
go.etcd.io/etcd/client/v3 v3.6.6
go.mongodb.org/mongo-driver v1.17.6
go.etcd.io/etcd/client/v3 v3.6.5
go.mongodb.org/mongo-driver v1.17.4
go.opencensus.io v0.24.0 // indirect
gocloud.dev v0.43.0
gocloud.dev/pubsub/natspubsub v0.43.0
gocloud.dev/pubsub/rabbitpubsub v0.43.0
golang.org/x/crypto v0.45.0
golang.org/x/crypto v0.43.0
golang.org/x/exp v0.0.0-20250811191247-51f88131bc50
golang.org/x/image v0.33.0
golang.org/x/net v0.47.0
golang.org/x/oauth2 v0.32.0 // indirect
golang.org/x/sys v0.38.0
golang.org/x/text v0.31.0 // indirect
golang.org/x/tools v0.38.0 // indirect
golang.org/x/image v0.32.0
golang.org/x/net v0.45.0
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/sys v0.37.0
golang.org/x/text v0.30.0 // indirect
golang.org/x/tools v0.37.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/api v0.247.0
google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79 // indirect
google.golang.org/grpc v1.77.0
google.golang.org/protobuf v1.36.10
google.golang.org/grpc v1.75.1
google.golang.org/protobuf v1.36.9
gopkg.in/inf.v0 v0.9.1 // indirect
modernc.org/b v1.0.0 // indirect
modernc.org/mathutil v1.7.1
@@ -123,46 +123,45 @@ require (
github.com/Jille/raft-grpc-transport v1.6.1
github.com/ThreeDotsLabs/watermill v1.5.1
github.com/a-h/templ v0.3.943
github.com/apple/foundationdb/bindings/go v0.0.0-20240515141816-262c6fe778ad
github.com/arangodb/go-driver v1.6.9
github.com/arangodb/go-driver v1.6.7
github.com/armon/go-metrics v0.4.1
github.com/aws/aws-sdk-go-v2 v1.40.0
github.com/aws/aws-sdk-go-v2 v1.39.2
github.com/aws/aws-sdk-go-v2/config v1.31.3
github.com/aws/aws-sdk-go-v2/credentials v1.18.20
github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1
github.com/aws/aws-sdk-go-v2/credentials v1.18.10
github.com/aws/aws-sdk-go-v2/service/s3 v1.88.3
github.com/cognusion/imaging v1.0.2
github.com/fluent/fluent-logger-golang v1.10.1
github.com/getsentry/sentry-go v0.38.0
github.com/getsentry/sentry-go v0.35.3
github.com/gin-contrib/sessions v1.0.4
github.com/gin-gonic/gin v1.11.0
github.com/golang-jwt/jwt/v5 v5.3.0
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
github.com/hanwen/go-fuse/v2 v2.9.0
github.com/hanwen/go-fuse/v2 v2.8.0
github.com/hashicorp/raft v1.7.3
github.com/hashicorp/raft-boltdb/v2 v2.3.1
github.com/hashicorp/vault/api v1.22.0
github.com/hashicorp/vault/api v1.20.0
github.com/jhump/protoreflect v1.17.0
github.com/lib/pq v1.10.9
github.com/linkedin/goavro/v2 v2.14.1
github.com/linkedin/goavro/v2 v2.14.0
github.com/mattn/go-sqlite3 v1.14.32
github.com/minio/crc64nvme v1.1.1
github.com/orcaman/concurrent-map/v2 v2.0.1
github.com/parquet-go/parquet-go v0.25.1
github.com/pkg/sftp v1.13.10
github.com/pkg/sftp v1.13.9
github.com/rabbitmq/amqp091-go v1.10.0
github.com/rclone/rclone v1.71.2
github.com/rclone/rclone v1.71.1
github.com/rdleal/intervalst v1.5.0
github.com/redis/go-redis/v9 v9.14.1
github.com/schollz/progressbar/v3 v3.18.0
github.com/shirou/gopsutil/v4 v4.25.11
github.com/tarantool/go-tarantool/v2 v2.4.1
github.com/shirou/gopsutil/v4 v4.25.9
github.com/tarantool/go-tarantool/v2 v2.4.0
github.com/tikv/client-go/v2 v2.0.7
github.com/xeipuuv/gojsonschema v1.2.0
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0
github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0
go.etcd.io/etcd/client/pkg/v3 v3.6.6
github.com/ydb-platform/ydb-go-sdk/v3 v3.113.5
go.etcd.io/etcd/client/pkg/v3 v3.6.5
go.uber.org/atomic v1.11.0
golang.org/x/sync v0.18.0
golang.org/x/sync v0.17.0
golang.org/x/tools/godoc v0.1.0-deprecated
google.golang.org/grpc/security/advancedtls v1.0.0
)
@@ -189,9 +188,9 @@ require (
github.com/google/go-cmp v0.7.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
@@ -221,7 +220,7 @@ require (
go.uber.org/mock v0.5.0 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/mod v0.29.0 // indirect
golang.org/x/mod v0.28.0 // indirect
gonum.org/v1/gonum v0.16.0 // indirect
)
@@ -229,19 +228,19 @@ require (
cel.dev/expr v0.24.0 // indirect
cloud.google.com/go/auth v0.16.5 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
cloud.google.com/go/compute/metadata v0.8.0 // indirect
cloud.google.com/go/iam v1.5.2 // indirect
cloud.google.com/go/monitoring v1.24.2 // indirect
filippo.io/edwards25519 v1.1.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.2 // indirect
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 // indirect
github.com/Files-com/files-sdk-go/v3 v3.2.218 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect
github.com/IBM/go-sdk-core/v5 v5.21.0 // indirect
@@ -260,23 +259,23 @@ require (
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.9 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.9 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.9 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 // indirect
github.com/aws/smithy-go v1.23.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.29.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.38.2 // indirect
github.com/aws/smithy-go v1.23.0 // indirect
github.com/boltdb/bolt v1.3.1 // indirect
github.com/bradenaw/juniper v0.15.3 // indirect
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
@@ -290,7 +289,7 @@ require (
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
github.com/cloudwego/base64x v0.1.6 // indirect
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
github.com/creasty/defaults v1.8.0 // indirect
github.com/cronokirby/saferith v0.33.0 // indirect
@@ -298,13 +297,13 @@ require (
github.com/d4l3k/messagediff v1.2.1 // indirect
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
github.com/ebitengine/purego v0.9.1 // indirect
github.com/ebitengine/purego v0.9.0 // indirect
github.com/elastic/gosigar v0.14.3 // indirect
github.com/emersion/go-message v0.18.2 // indirect
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/fatih/color v1.18.0 // indirect
github.com/fatih/color v1.16.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/flynn/noise v1.1.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.9 // indirect
@@ -312,7 +311,7 @@ require (
github.com/gin-contrib/sse v1.1.0 // indirect
github.com/go-chi/chi/v5 v5.2.2 // indirect
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
github.com/go-jose/go-jose/v4 v4.1.1 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
@@ -408,15 +407,15 @@ require (
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
github.com/spacemonkeygo/monkit/v3 v3.0.24 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/t3rm1n4l/go-mega v0.0.0-20250926104142-ccb8d3498e6c // indirect
github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 // indirect
github.com/tarantool/go-iproto v1.1.0 // indirect
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
github.com/tinylib/msgp v1.3.0 // indirect
github.com/tklauser/go-sysconf v0.3.16 // indirect
github.com/tklauser/numcpus v0.11.0 // indirect
github.com/tklauser/go-sysconf v0.3.15 // indirect
github.com/tklauser/numcpus v0.10.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/twmb/murmur3 v1.1.3 // indirect
github.com/ugorji/go/codec v1.3.0 // indirect
@@ -425,7 +424,7 @@ require (
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb // indirect
github.com/ydb-platform/ydb-go-genproto v0.0.0-20241112172322-ea1f63298f77 // indirect
github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
@@ -433,23 +432,23 @@ require (
github.com/zeebo/blake3 v0.2.4 // indirect
github.com/zeebo/errs v1.4.0 // indirect
go.etcd.io/bbolt v1.4.2 // indirect
go.etcd.io/etcd/api/v3 v3.6.6 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect
go.etcd.io/etcd/api/v3 v3.6.5 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.37.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect
go.opentelemetry.io/otel v1.38.0 // indirect
go.opentelemetry.io/otel/metric v1.38.0 // indirect
go.opentelemetry.io/otel/sdk v1.38.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect
go.opentelemetry.io/otel/trace v1.38.0 // indirect
go.opentelemetry.io/otel v1.37.0 // indirect
go.opentelemetry.io/otel/metric v1.37.0 // indirect
go.opentelemetry.io/otel/sdk v1.37.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect
go.opentelemetry.io/otel/trace v1.37.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/arch v0.20.0 // indirect
golang.org/x/term v0.37.0 // indirect
golang.org/x/term v0.36.0 // indirect
golang.org/x/time v0.12.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/validator.v2 v2.0.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
@@ -465,4 +464,7 @@ require (
storj.io/uplink v1.13.1 // indirect
)
// Use the seaweedfs fork of cockroachdb-parser to fix cross-platform build issues
replace github.com/cockroachdb/cockroachdb-parser => github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021182748-d0c58c67297e
// replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft
+164 -154
View File
@@ -158,8 +158,8 @@ cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZ
cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
cloud.google.com/go/compute/metadata v0.8.0 h1:HxMRIbao8w17ZX6wBnjhcDkW6lTFpgcaobyVfZWqRLA=
cloud.google.com/go/compute/metadata v0.8.0/go.mod h1:sYOGTp851OV9bOFJ9CH7elVvyzopvWQFNNghtDQ/Biw=
cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY=
cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck=
cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w=
@@ -477,8 +477,8 @@ cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq
cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc=
cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s=
cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y=
cloud.google.com/go/storage v1.57.1 h1:gzao6odNJ7dR3XXYvAgPK+Iw4fVPPznEPPyNjbaVkq8=
cloud.google.com/go/storage v1.57.1/go.mod h1:329cwlpzALLgJuu8beyJ/uvQznDHpa2U5lGjWednkzg=
cloud.google.com/go/storage v1.57.0 h1:4g7NB7Ta7KetVbOMpCqy89C+Vg5VE8scqlSHUPm7Rds=
cloud.google.com/go/storage v1.57.0/go.mod h1:329cwlpzALLgJuu8beyJ/uvQznDHpa2U5lGjWednkzg=
cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w=
cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I=
cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4=
@@ -541,10 +541,10 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8=
git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1 h1:5YTBM8QDVIBN3sxBil89WfdAAqDZbyJTgh688DSxX5w=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0 h1:KpMC6LFL7mqpExyMC9jVOYRiVhLmamjeZfRsUpB7l4s=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0/go.mod h1:J7MUC/wtRpfGVbQ5sIItY5/FuVWmvzlY21WAOfQnq/I=
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA=
@@ -555,8 +555,8 @@ github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGlu
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 h1:ZJJNFaQ86GVKQ9ehwqyAFE6pIfyicpuJ8IkVaPBc6/4=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATVKqHBH9/0nOiNKk0+YcwfQ3WkK5PqHKxc8=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZYLBR2kBz5C8Tg0fw5w5Y7meRXWI=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk=
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.2 h1:l3SabZmNuXCMCbQUIeR4W6/N4j8SeH/lwX+a6leZhHo=
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.2/go.mod h1:k+mEZ4f1pVqZTRqtSDW2AhZ/3wT5qLpsUA75C/k7dtE=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
@@ -564,8 +564,8 @@ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 h1:XkkQbfMyuH2jTSjQjSoihryI8GINRcs4xp8lNawg0FI=
github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/Codefor/geohash v0.0.0-20140723084247-1b41c28e3a9d h1:iG9B49Q218F/XxXNRM7k/vWf7MKmLIS8AcJV9cGN4nA=
@@ -576,8 +576,8 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3
github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
github.com/Files-com/files-sdk-go/v3 v3.2.218 h1:tIvcbHXNY/bq+Sno6vajOJOxhe5XbU59Fa1ohOybK+s=
github.com/Files-com/files-sdk-go/v3 v3.2.218/go.mod h1:E0BaGQbcMUcql+AfubCR/iasWKBxX5UZPivnQGC2z0M=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0 h1:4LP6hvB4I5ouTbGgWtixJhgED6xdf67twf9PoY96Tbg=
@@ -651,62 +651,61 @@ github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmg
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0=
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
github.com/apple/foundationdb/bindings/go v0.0.0-20240515141816-262c6fe778ad h1:fQBkhYv86zyW95PWhzBlkgz3NoY1ue0L+8oYBaoCMbg=
github.com/apple/foundationdb/bindings/go v0.0.0-20240515141816-262c6fe778ad/go.mod h1:OMVSB21p9+xQUIqlGizHPZfjK+SHws1ht+ZytVDoz9U=
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc h1:LoL75er+LKDHDUfU5tRvFwxH0LjPpZN8OoG8Ll+liGU=
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc/go.mod h1:w648aMHEgFYS6xb0KVMMtZ2uMeemhiKCuD2vj6gY52A=
github.com/arangodb/go-driver v1.6.9 h1:zckB+xuA16NmHUuYOX7INCJTIyIkoBQjAGqNpiyf2HQ=
github.com/arangodb/go-driver v1.6.9/go.mod h1:eAM/drVZw39hTGFdkxvbVv0uJsDGFaUpqQHVZMSoALc=
github.com/arangodb/go-driver v1.6.7 h1:9FBUsH60cKu7DjFGozTsaqWMy+3UeEplplqUn4yEcg4=
github.com/arangodb/go-driver v1.6.7/go.mod h1:H6uhiKUD/ki7fS9dNDK6xzMX/D5ibj5kGN1bGKd37Ho=
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g=
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:mq7Shfa/CaixoDxiyAAc5jZ6CVBAyPaNQCGS7mkj4Ho=
github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc=
github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2/go.mod h1:IusfVNTmiSN3t4rhxWFaBAqn+mcNdwKtPcV16eYdgko=
github.com/aws/aws-sdk-go-v2 v1.39.2 h1:EJLg8IdbzgeD7xgvZ+I8M1e0fL0ptn/M47lianzth0I=
github.com/aws/aws-sdk-go-v2 v1.39.2/go.mod h1:sDioUELIUO9Znk23YVmIk86/9DOpkbyyVb1i/gUNFXY=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 h1:i8p8P4diljCr60PpJp6qZXNlgX4m2yQFpYk+9ZT+J4E=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1/go.mod h1:ddqbooRZYNoJ2dsTwOty16rM+/Aqmk/GOXrK8cg7V00=
github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco=
github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE=
github.com/aws/aws-sdk-go-v2/credentials v1.18.20 h1:KFndAnHd9NUuzikHjQ8D5CfFVO+bgELkmcGY8yAw98Q=
github.com/aws/aws-sdk-go-v2/credentials v1.18.20/go.mod h1:9mCi28a+fmBHSQ0UM79omkz6JtN+PEsvLrnG36uoUv0=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 h1:VO3FIM2TDbm0kqp6sFNR0PbioXJb/HzCDW6NtIZpIWE=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12/go.mod h1:6C39gB8kg82tx3r72muZSrNhHia9rjGkX7ORaS2GKNE=
github.com/aws/aws-sdk-go-v2/credentials v1.18.10 h1:xdJnXCouCx8Y0NncgoptztUocIYLKeQxrCgN6x9sdhg=
github.com/aws/aws-sdk-go-v2/credentials v1.18.10/go.mod h1:7tQk08ntj914F/5i9jC4+2HQTAuJirq7m1vZVIhEkWs=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6 h1:wbjnrrMnKew78/juW7I2BtKQwa1qlf6EjQgS69uYY14=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6/go.mod h1:AtiqqNrDioJXuUgz3+3T0mBWN7Hro2n9wll2zRUc0ww=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 h1:p/9flfXdoAnwJnuW9xHEAFY22R3A6skYkW19JFF9F+8=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12/go.mod h1:ZTLHakoVCTtW8AaLGSwJ3LXqHD9uQKnOcv1TrpO6u2k=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 h1:2lTWFvRcnWFFLzHWmtddu5MTchc5Oj2OOey++99tPZ0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12/go.mod h1:hI92pK+ho8HVcWMHKHrK3Uml4pfG7wvL86FzO0LVtQQ=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.9 h1:se2vOWGD3dWQUtfn4wEjRQJb1HK1XsNIt825gskZ970=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.9/go.mod h1:hijCGH2VfbZQxqCDN7bwz/4dzxV+hkyhjawAtdPWKZA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.9 h1:6RBnKZLkJM4hQ+kN6E7yWFveOTg8NLPHAkqrs4ZPlTU=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.9/go.mod h1:V9rQKRmK7AWuEsOMnHzKj8WyrIir1yUJbZxDuZLFvXI=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 h1:itu4KHu8JK/N6NcLIISlf3LL1LccMqruLUXZ9y7yBZw=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12/go.mod h1:i+6vTU3xziikTY3vcox23X8pPGW5X3wVgd1VZ7ha+x8=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 h1:NEe7FaViguRQEm8zl8Ay/kC/QRsMtWUiCGZajQIsLdc=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3/go.mod h1:JLuCKu5VfiLBBBl/5IzZILU7rxS0koQpHzMOCzycOJU=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 h1:MM8imH7NZ0ovIVX7D2RxfMDv7Jt9OiUXkcQ+GqywA7M=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12/go.mod h1:gf4OGwdNkbEsb7elw2Sy76odfhwNktWII3WgvQgQQ6w=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 h1:R3uW0iKl8rgNEXNjVGliW/oMEh9fO/LlUEV8RvIFr1I=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12/go.mod h1:XEttbEr5yqsw8ebi7vlDoGJJjMXRez4/s9pibpJyL5s=
github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 h1:Dq82AV+Qxpno/fG162eAhnD8d48t9S+GZCfz7yv1VeA=
github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1/go.mod h1:MbKLznDKpf7PnSonNRUVYZzfP0CeLkRIUexeblgKcU4=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.9 h1:w9LnHqTq8MEdlnyhV4Bwfizd65lfNCNgdlNC6mM5paE=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.9/go.mod h1:LGEP6EK4nj+bwWNdrvX/FnDTFowdBNwcSPuZu/ouFys=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 h1:oegbebPEMA/1Jny7kvwejowCaHz1FWZAQ94WXFNCyTM=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1/go.mod h1:kemo5Myr9ac0U9JfSjMo9yHLtw+pECEHsFtJ9tqCEI8=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.9 h1:by3nYZLR9l8bUH7kgaMU4dJgYFjyRdFEfORlDpPILB4=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.9/go.mod h1:IWjQYlqw4EX9jw2g3qnEPPWvCE6bS8fKzhMed1OK7c8=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9 h1:5r34CgVOD4WZudeEKZ9/iKpiT6cM1JyEROpXjOcdWv8=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9/go.mod h1:dB12CEbNWPbzO2uC6QSWHteqOg4JfBVJOojbAoAUb5I=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.9 h1:wuZ5uW2uhJR63zwNlqWH2W4aL4ZjeJP3o92/W+odDY4=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.9/go.mod h1:/G58M2fGszCrOzvJUkDdY8O9kycodunH4VdT5oBAqls=
github.com/aws/aws-sdk-go-v2/service/s3 v1.88.3 h1:P18I4ipbk+b/3dZNq5YYh+Hq6XC0vp5RWkLp1tJldDA=
github.com/aws/aws-sdk-go-v2/service/s3 v1.88.3/go.mod h1:Rm3gw2Jov6e6kDuamDvyIlZJDMYk97VeCZ82wz/mVZ0=
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 h1:OBuZE9Wt8h2imuRktu+WfjiTGrnYdCIJg8IX92aalHE=
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7/go.mod h1:4WYoZAhHt+dWYpoOQUgkUKfuQbE6Gg/hW4oXE0pKS9U=
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 h1:80dpSqWMwx2dAm30Ib7J6ucz1ZHfiv5OCRwN/EnCOXQ=
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8/go.mod h1:IzNt/udsXlETCdvBOL0nmyMe2t9cGmXmZgsdoZGYYhI=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 h1:xHXvxst78wBpJFgDW07xllOx0IAzbryrSdM4nMVQ4Dw=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0/go.mod h1:/e8m+AO6HNPPqMyfKRtzZ9+mBF5/x1Wk8QiDva4m07I=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 h1:tBw2Qhf0kj4ZwtsVpDiVRU3zKLvjvjgIjHMKirxXg8M=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4/go.mod h1:Deq4B7sRM6Awq/xyOBlxBdgW8/Z926KYNNaGMW2lrkA=
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 h1:C+BRMnasSYFcgDw8o9H5hzehKzXyAb9GY5v/8bP9DUY=
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0/go.mod h1:4EjU+4mIx6+JqKQkruye+CaigV7alL3thVPfDd9VlMs=
github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
github.com/aws/aws-sdk-go-v2/service/sso v1.29.1 h1:8OLZnVJPvjnrxEwHFg9hVUof/P4sibH+Ea4KKuqAGSg=
github.com/aws/aws-sdk-go-v2/service/sso v1.29.1/go.mod h1:27M3BpVi0C02UiQh1w9nsBEit6pLhlaH3NHna6WUbDE=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2 h1:gKWSTnqudpo8dAxqBqZnDoDWCiEh/40FziUjr/mo6uA=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2/go.mod h1:x7+rkNmRoEN1U13A6JE2fXne9EWyJy54o3n6d4mGaXQ=
github.com/aws/aws-sdk-go-v2/service/sts v1.38.2 h1:YZPjhyaGzhDQEvsffDEcpycq49nl7fiGcfJTIo8BszI=
github.com/aws/aws-sdk-go-v2/service/sts v1.38.2/go.mod h1:2dIN8qhQfv37BdUYGgEC8Q3tteM3zFxTI1MLO2O3J3c=
github.com/aws/smithy-go v1.23.0 h1:8n6I3gXzWJB2DxBDnfxgBaSX6oe0d/t10qGz7OKqMCE=
github.com/aws/smithy-go v1.23.0/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
github.com/bazelbuild/rules_go v0.46.0/go.mod h1:Dhcz716Kqg1RHNWos+N6MlXNkjNP2EwZQ0LukRKJfMs=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -714,6 +713,7 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f h1:+6okTAeUsUrdQr/qN7fIODzowrjjCrnJDg/gkYqcSXY=
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f/go.mod h1:z52shMwD6SGwRg2iYFjjDwX5Ene4ENTw6HfXraUy/08=
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY=
@@ -796,8 +796,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH
github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0=
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4=
github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls=
github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
github.com/cockroachdb/apd/v3 v3.1.0 h1:MK3Ow7LH0W8zkd5GMKA1PvS9qG3bWFI95WaVNfyZJ/w=
github.com/cockroachdb/apd/v3 v3.1.0/go.mod h1:6qgPBMXjATAdD/VefbRP9NoSLKjbB4LCoA7gN4LpHs4=
github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
@@ -859,8 +859,8 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0=
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/ebitengine/purego v0.9.0 h1:mh0zpKBIXDceC63hpvPuGLiJ8ZAa3DfrFTudmfi8A4k=
github.com/ebitengine/purego v0.9.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo=
github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
@@ -878,10 +878,10 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.
github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34=
github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI=
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM=
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs=
github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo=
github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs=
github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M=
github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA=
github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A=
github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw=
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI=
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
@@ -902,9 +902,10 @@ github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpm
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
github.com/fanixk/geohash v0.0.0-20150324002647-c1f9b5fa157a h1:Fyfh/dsHFrC6nkX7H7+nFdTd1wROlX/FxEIWVpKYf1U=
github.com/fanixk/geohash v0.0.0-20150324002647-c1f9b5fa157a/go.mod h1:UgNw+PTmmGN8rV7RvjvnBMsoTU8ZXXnaT3hYsDTBlgQ=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/fluent/fluent-logger-golang v1.10.1 h1:wu54iN1O2afll5oQrtTjhgZRwWcfOeFFzwRsEkABfFQ=
@@ -925,8 +926,8 @@ github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBv
github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok=
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
github.com/getsentry/sentry-go v0.38.0 h1:S8Xui7gLeAvXINVLMOaX94HnsDf1GexnfXGSNC4+KQs=
github.com/getsentry/sentry-go v0.38.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
github.com/getsentry/sentry-go v0.35.3 h1:u5IJaEqZyPdWqe/hKlBKBBnMTSxB/HenCqF3QLabeds=
github.com/getsentry/sentry-go v0.35.3/go.mod h1:mdL49ixwT2yi57k5eh7mpnDyPybixPzlzEJFu0Z76QA=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gin-contrib/sessions v1.0.4 h1:ha6CNdpYiTOK/hTp05miJLbpTSNfOnFg5Jm2kbcqy8U=
github.com/gin-contrib/sessions v1.0.4/go.mod h1:ccmkrb2z6iU2osiAHZG3x3J4suJK+OU27oqzlWOqQgs=
@@ -948,8 +949,8 @@ github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmn
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI=
github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -996,8 +997,8 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U=
github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw=
github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
@@ -1186,8 +1187,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+u
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90=
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
github.com/hanwen/go-fuse/v2 v2.9.0 h1:0AOGUkHtbOVeyGLr0tXupiid1Vg7QB7M6YUcdmVdC58=
github.com/hanwen/go-fuse/v2 v2.9.0/go.mod h1:yE6D2PqWwm3CbYRxFXV9xUd8Md5d6NG0WBs5spCswmI=
github.com/hanwen/go-fuse/v2 v2.8.0 h1:wV8rG7rmCz8XHSOwBZhG5YcVqcYjkzivjmbaMafPlAs=
github.com/hanwen/go-fuse/v2 v2.8.0/go.mod h1:yE6D2PqWwm3CbYRxFXV9xUd8Md5d6NG0WBs5spCswmI=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -1215,12 +1216,13 @@ github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVU
github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw=
github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 h1:U+kC2dOhMFQctRfhK0gRctKAPTloZdMU5ZJxaesJ/VM=
github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0/go.mod h1:Ll013mhdmsVDuoIXVfBtvgGJsXDYkTw1kooNcoCXuE0=
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 h1:om4Al8Oy7kCm/B86rLCLah4Dt5Aa0Fr5rYBG60OzwHQ=
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8=
github.com/hashicorp/go-secure-stdlib/strutil v0.1.1/go.mod h1:gKOamz3EwoIoJq7mlMIRBpVTAUn8qPCrEclOKKWhD3U=
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw=
github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw=
github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc=
github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
@@ -1239,8 +1241,8 @@ github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702 h1:RLKEcCuKc
github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702/go.mod h1:nTakvJ4XYq45UXtn0DbwR4aU9ZdjlnIenpbs6Cd+FM0=
github.com/hashicorp/raft-boltdb/v2 v2.3.1 h1:ackhdCNPKblmOhjEU9+4lHSJYFkJd6Jqyvj6eW9pwkc=
github.com/hashicorp/raft-boltdb/v2 v2.3.1/go.mod h1:n4S+g43dXF1tqDT+yzcXHhXM6y7MrlUd3TTwGRcUvQE=
github.com/hashicorp/vault/api v1.22.0 h1:+HYFquE35/B74fHoIeXlZIP2YADVboaPjaSicHEZiH0=
github.com/hashicorp/vault/api v1.22.0/go.mod h1:IUZA2cDvr4Ok3+NtK2Oq/r+lJeXkeCrHRmqdyWfpmGM=
github.com/hashicorp/vault/api v1.20.0 h1:KQMHElgudOsr+IbJgmbjHnCTxEpKs9LnozA1D3nozU4=
github.com/hashicorp/vault/api v1.20.0/go.mod h1:GZ4pcjfzoOWpkJ3ijHNpEoAxKEsBJnVljyTe3jM2Sms=
github.com/henrybear327/Proton-API-Bridge v1.0.0 h1:gjKAaWfKu++77WsZTHg6FUyPC5W0LTKWQciUm8PMZb0=
github.com/henrybear327/Proton-API-Bridge v1.0.0/go.mod h1:gunH16hf6U74W2b9CGDaWRadiLICsoJ6KRkSt53zLts=
github.com/henrybear327/go-proton-api v1.0.0 h1:zYi/IbjLwFAW7ltCeqXneUGJey0TN//Xo851a/BgLXw=
@@ -1324,8 +1326,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
@@ -1363,10 +1365,10 @@ github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTRe
github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/linkedin/goavro/v2 v2.14.1 h1:/8VjDpd38PRsy02JS0jflAu7JZPfJcGTwqWgMkFS2iI=
github.com/linkedin/goavro/v2 v2.14.1/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
github.com/linxGnu/grocksdb v1.10.3 h1:0laII9AQ6kFxo5SjhdTfSh9EgF20piD6TMHK6YuDm+4=
github.com/linxGnu/grocksdb v1.10.3/go.mod h1:OLQKZwiKwaJiAVCsOzWKvwiLwfZ5Vz8Md5TYR7t7pM8=
github.com/linkedin/goavro/v2 v2.14.0 h1:aNO/js65U+Mwq4yB5f1h01c3wiM458qtRad1DN0CMUI=
github.com/linkedin/goavro/v2 v2.14.0/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
github.com/linxGnu/grocksdb v1.10.2 h1:y0dXsWYULY15/BZMcwAZzLd13ZuyA470vyoNzWwmqG0=
github.com/linxGnu/grocksdb v1.10.2/go.mod h1:C3CNe9UYc9hlEM2pC82AqiGS3LRW537u9LFV4wIZuHk=
github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8=
github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts=
github.com/lpar/date v1.0.0 h1:bq/zVqFTUmsxvd/CylidY4Udqpr9BOFrParoP6p0x/I=
@@ -1378,10 +1380,12 @@ github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuz
github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -1400,11 +1404,14 @@ github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 h1:BpfhmLKZf+SjVanKKhCgf3bg+511DmU9eDQTen7LLbY=
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
@@ -1518,8 +1525,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU=
github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA=
github.com/pkg/sftp v1.13.9 h1:4NGkvGudBL7GteO3m6qnaQ4pC0Kvf0onSVc9gR3EWBw=
github.com/pkg/sftp v1.13.9/go.mod h1:OBN7bVXdstkFFN/gdnHPUb5TE8eb8G1Rp9wCItqjkkA=
github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM=
github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
@@ -1527,6 +1534,7 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo=
github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
@@ -1559,8 +1567,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0=
github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw=
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8=
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU=
github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
@@ -1569,8 +1577,8 @@ github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2e
github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY=
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/rclone/rclone v1.71.2 h1:3Jk5xNPFrZhVABRuN/OPvApuZQddpE2tkhYMuEn1Ud4=
github.com/rclone/rclone v1.71.2/go.mod h1:dCK9FzPDlpkbQJ9M7MmWsmv3X5nibfWe+ogJXu6gSgM=
github.com/rclone/rclone v1.71.1 h1:cpODfWTRz5i/WAzXsyW85tzfIKNsd1aq8CE8lUB+0zg=
github.com/rclone/rclone v1.71.1/go.mod h1:NLyX57FrnZ9nVLTY5TRdMmGelrGKbIRYGcgRkNdqqlA=
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
@@ -1602,6 +1610,7 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w=
github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk=
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI=
@@ -1624,8 +1633,8 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shirou/gopsutil/v4 v4.25.11 h1:X53gB7muL9Gnwwo2evPSE+SfOrltMoR6V3xJAXZILTY=
github.com/shirou/gopsutil/v4 v4.25.11/go.mod h1:EivAfP5x2EhLp2ovdpKSozecVXn1TmuG7SMzs/Wh4PU=
github.com/shirou/gopsutil/v4 v4.25.9 h1:JImNpf6gCVhKgZhtaAHJ0serfFGtlfIlSC08eaKdTrU=
github.com/shirou/gopsutil/v4 v4.25.9/go.mod h1:gxIxoC+7nQRwUl/xNhutXlD8lq+jxTgpIkEf3rADHL8=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -1662,8 +1671,8 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo=
github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs=
github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE=
github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@@ -1693,13 +1702,13 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw=
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
github.com/t3rm1n4l/go-mega v0.0.0-20250926104142-ccb8d3498e6c h1:BLopNCyqewbE8+BtlIp/Juzu8AJGxz0gHdGADnsblVc=
github.com/t3rm1n4l/go-mega v0.0.0-20250926104142-ccb8d3498e6c/go.mod h1:ykucQyiE9Q2qx1wLlEtZkkNn1IURib/2O+Mvd25i1Fo=
github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 h1:Sa+sR8aaAMFwxhXWENEnE6ZpqhZ9d7u1RT2722Rw6hc=
github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5/go.mod h1:UdZiFUFu6e2WjjtjxivwXWcwc1N/8zgbkBR9QNucUOY=
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
github.com/tarantool/go-iproto v1.1.0 h1:HULVOIHsiehI+FnHfM7wMDntuzUddO09DKqu2WnFQ5A=
github.com/tarantool/go-iproto v1.1.0/go.mod h1:LNCtdyZxojUed8SbOiYHoc3v9NvaZTB7p96hUySMlIo=
github.com/tarantool/go-tarantool/v2 v2.4.1 h1:Bk9mh+gMPVmHTSefHvVBpEkf6P2UZA/8xa5kqgyQtyo=
github.com/tarantool/go-tarantool/v2 v2.4.1/go.mod h1:MTbhdjFc3Jl63Lgi/UJr5D+QbT+QegqOzsNJGmaw7VM=
github.com/tarantool/go-tarantool/v2 v2.4.0 h1:cfGngxdknpVVbd/vF2LvaoWsKjsLV9i3xC859XgsJlI=
github.com/tarantool/go-tarantool/v2 v2.4.0/go.mod h1:MTbhdjFc3Jl63Lgi/UJr5D+QbT+QegqOzsNJGmaw7VM=
github.com/the42/cartconvert v0.0.0-20131203171324-aae784c392b8 h1:I4DY8wLxJXCrMYzDM6lKCGc3IQwJX0PlTLsd3nQqI3c=
github.com/the42/cartconvert v0.0.0-20131203171324-aae784c392b8/go.mod h1:fWO/msnJVhHqN1yX6OBoxSyfj7TEj1hHiL8bJSQsK30=
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4=
@@ -1718,10 +1727,10 @@ github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1/go.mod h1:3cTcfo8GR
github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww=
github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=
github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4=
github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4=
github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso=
github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ=
github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 h1:6iRwZdrFUzbcVYZwa8dXTIILGIxmmhjyUPJEcwzPGaU=
github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365/go.mod h1:zj0GJHGvyf1ed3Jm/Tb4830c/ZKDq+YoLsCt2rGQuT0=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
@@ -1775,14 +1784,14 @@ github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE=
github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb h1:LZ6dhVfWzhicf/P5Xh7fA0Jd7rfGduxmB2QZpD+Lz9Q=
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
github.com/ydb-platform/ydb-go-genproto v0.0.0-20241112172322-ea1f63298f77 h1:LY6cI8cP4B9rrpTleZk95+08kl2gF4rixG7+V/dwL6Q=
github.com/ydb-platform/ydb-go-genproto v0.0.0-20241112172322-ea1f63298f77/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0 h1:/NyPd9KnCJgzrEXCArqk1ThqCH2Dh31uUwl88o/VkuM=
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
github.com/ydb-platform/ydb-go-sdk/v3 v3.44.0/go.mod h1:oSLwnuilwIpaF5bJJMAofnGgzPJusoI3zWMNb8I+GnM=
github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0 h1:Ml4WMsaRY+/+Wh3DCp1A+/zjM1aMdQg3irjuYFwWVHQ=
github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0/go.mod h1:/LjMxb/rXmoGAAnImoqAFIlhO5ampHacbvDetQitCk4=
github.com/ydb-platform/ydb-go-sdk/v3 v3.113.5 h1:olAAZfpMnFYChJNgZJ16G4jqoelRNx7Kx4tW50XcMv0=
github.com/ydb-platform/ydb-go-sdk/v3 v3.113.5/go.mod h1:Pp1w2xxUoLQ3NCNAwV7pvDq0TVQOdtAqs+ZiC+i8r14=
github.com/ydb-platform/ydb-go-yc v0.12.1 h1:qw3Fa+T81+Kpu5Io2vYHJOwcrYrVjgJlT6t/0dOXJrA=
github.com/ydb-platform/ydb-go-yc v0.12.1/go.mod h1:t/ZA4ECdgPWjAb4jyDe8AzQZB5dhpGbi3iCahFaNwBY=
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 h1:9E5q8Nsy2RiJMZDNVy0A3KUrIMBPakJ2VgloeWbcI84=
@@ -1815,14 +1824,14 @@ go.einride.tech/aip v0.73.0 h1:bPo4oqBo2ZQeBKo4ZzLb1kxYXTY1ysJhpvQyfuGzvps=
go.einride.tech/aip v0.73.0/go.mod h1:Mj7rFbmXEgw0dq1dqJ7JGMvYCZZVxmGOR3S4ZcV5LvQ=
go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I=
go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM=
go.etcd.io/etcd/api/v3 v3.6.6 h1:mcaMp3+7JawWv69p6QShYWS8cIWUOl32bFLb6qf8pOQ=
go.etcd.io/etcd/api/v3 v3.6.6/go.mod h1:f/om26iXl2wSkcTA1zGQv8reJRSLVdoEBsi4JdfMrx4=
go.etcd.io/etcd/client/pkg/v3 v3.6.6 h1:uoqgzSOv2H9KlIF5O1Lsd8sW+eMLuV6wzE3q5GJGQNs=
go.etcd.io/etcd/client/pkg/v3 v3.6.6/go.mod h1:YngfUVmvsvOJ2rRgStIyHsKtOt9SZI2aBJrZiWJhCbI=
go.etcd.io/etcd/client/v3 v3.6.6 h1:G5z1wMf5B9SNexoxOHUGBaULurOZPIgGPsW6CN492ec=
go.etcd.io/etcd/client/v3 v3.6.6/go.mod h1:36Qv6baQ07znPR3+n7t+Rk5VHEzVYPvFfGmfF4wBHV8=
go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss=
go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
go.etcd.io/etcd/api/v3 v3.6.5 h1:pMMc42276sgR1j1raO/Qv3QI9Af/AuyQUW6CBAWuntA=
go.etcd.io/etcd/api/v3 v3.6.5/go.mod h1:ob0/oWA/UQQlT1BmaEkWQzI0sJ1M0Et0mMpaABxguOQ=
go.etcd.io/etcd/client/pkg/v3 v3.6.5 h1:Duz9fAzIZFhYWgRjp/FgNq2gO1jId9Yae/rLn3RrBP8=
go.etcd.io/etcd/client/pkg/v3 v3.6.5/go.mod h1:8Wx3eGRPiy0qOFMZT/hfvdos+DjEaPxdIDiCDUv/FQk=
go.etcd.io/etcd/client/v3 v3.6.5 h1:yRwZNFBx/35VKHTcLDeO7XVLbCBFbPi+XV4OC3QJf2U=
go.etcd.io/etcd/client/v3 v3.6.5/go.mod h1:ZqwG/7TAFZ0BJ0jXRPoJjKQJtbFo/9NIY8uoFFKcCyo=
go.mongodb.org/mongo-driver v1.17.4 h1:jUorfmVzljjr0FLzYQsGP8cgN/qzzxlY9Vh0C9KFXVw=
go.mongodb.org/mongo-driver v1.17.4/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -1832,16 +1841,16 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs=
go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/contrib/detectors/gcp v1.37.0 h1:B+WbN9RPsvobe6q4vP6KgM8/9plR/HNjgGBrfcOlweA=
go.opentelemetry.io/contrib/detectors/gcp v1.37.0/go.mod h1:K5zQ3TT7p2ru9Qkzk0bKtCql0RGkPj9pRjpXgZJZ+rU=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 h1:rbRJ8BBoVMsQShESYZ0FkvcITu8X8QNwJogcLUmDNNw=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0/go.mod h1:ru6KHrNtNHxM4nD/vd6QrLVWgKhxPYgblq4VAtNawTQ=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 h1:Hf9xI/XLML9ElpiHVDNwvqI0hIFlzV8dgIr35kV1kRU=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0/go.mod h1:NfchwuyNoMcZ5MLHwPrODwUF1HWCXWrL31s8gSAdIKY=
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 h1:EtFWSnwW9hGObjkIdmlnWSydO+Qs8OwzfzXLUPg4xOc=
@@ -1850,14 +1859,14 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.37.0 h1:6VjV6Et+1Hd2iL
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.37.0/go.mod h1:u8hcp8ji5gaM/RfcOo8z9NMnf1pVLfVY7lBY2VOGuUU=
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 h1:s0n95ya5tOG03exJ5JySOdJFtwGo4ZQ+KeY7Zro4CLI=
go.opentelemetry.io/otel/exporters/zipkin v1.36.0/go.mod h1:m9wRxtKA2MZ1HcnNC4BKI+9aYe434qRZTCvI7QGUN7Y=
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc=
go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps=
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
@@ -1917,8 +1926,8 @@ golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1949,8 +1958,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/image v0.33.0 h1:LXRZRnv1+zGd5XBUVRFmYEphyyKJjQjCRiOuAP3sZfQ=
golang.org/x/image v0.33.0/go.mod h1:DD3OsTYT9chzuzTQt+zMcOlBHgfoKQb1gry8p76Y1sc=
golang.org/x/image v0.32.0 h1:6lZQWq75h7L5IWNk0r+SCpUJ6tUVd3v4ZHnbRKLkUDQ=
golang.org/x/image v0.32.0/go.mod h1:/R37rrQmKXtO6tYXAjtDLwQgFLHmhW+V6ayXlxzP2Pc=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1985,8 +1994,8 @@ golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U=
golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -2056,8 +2065,8 @@ golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
golang.org/x/net v0.45.0 h1:RLBg5JKixCy82FtLJpeNlVM0nrSqpCRYzVU1n8kj0tM=
golang.org/x/net v0.45.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -2086,8 +2095,8 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri
golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY=
golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -2110,9 +2119,10 @@ golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -2218,8 +2228,8 @@ golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -2237,8 +2247,8 @@ golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q=
golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2260,8 +2270,8 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -2341,8 +2351,8 @@ golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE=
golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w=
golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -2557,10 +2567,10 @@ google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79 h1:Nt6z9UHqSlIdIGJdz6KhTIs2VRx/iOsA5iE8bmQNcxs=
google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79/go.mod h1:kTmlBHMPqR5uCZPBvwa2B18mvubkjyY3CRLI0c6fj0s=
google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 h1:mepRgnBZa07I4TRuomDE4sTIYieg/osKmzIf4USdWS4=
google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba h1:UKgtfRM7Yh93Sya0Fo8ZzhDP4qBckrrxEr2oF5UIVb8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c h1:AtEkQdl5b6zsybXcbz00j1LwNodDuH6hVifIaNqk7NQ=
google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c/go.mod h1:ea2MjsO70ssTfCjiwHgI0ZFqcw45Ksuk2ckf9G468GA=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c h1:qXWI/sQtv5UKboZ/zUk7h+mrf/lXORyI+n9DKDAusdg=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c/go.mod h1:gw1tLEfykwDz2ET4a12jcXt4couGAm7IwsVaTy0Sflo=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -2601,11 +2611,11 @@ google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsA
google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM=
google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig=
google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI=
google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6 h1:ExN12ndbJ608cboPYflpTny6mXSzPrDLh0iTaVrRrds=
google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6/go.mod h1:6ytKWczdvnpnO+m+JiG9NjEDzR1FJfsnmJdG7B8QVZ8=
google.golang.org/grpc/examples v0.0.0-20230224211313-3775f633ce20 h1:MLBCGN1O7GzIx+cBiwfYPwtmZ41U3Mn/cotLJciaArI=
google.golang.org/grpc/examples v0.0.0-20230224211313-3775f633ce20/go.mod h1:Nr5H8+MlGWr5+xX/STzdoEqJrO+YteqFbMyCsrb6mH0=
google.golang.org/grpc/security/advancedtls v1.0.0 h1:/KQ7VP/1bs53/aopk9QhuPyFAp9Dm9Ejix3lzYkCrDA=
google.golang.org/grpc/security/advancedtls v1.0.0/go.mod h1:o+s4go+e1PJ2AjuQMY5hU82W7lDlefjJA6FqEHRVHWk=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
@@ -2624,8 +2634,8 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw=
google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "4.01"
appVersion: "3.98"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.0.401
version: 4.0.398
@@ -13,12 +13,6 @@ spec:
secretName: {{ template "seaweedfs.name" . }}-ca-cert
commonName: "{{ template "seaweedfs.name" . }}-root-ca"
isCA: true
{{- if .Values.certificates.ca.duration }}
duration: {{ .Values.certificates.ca.duration }}
{{- end }}
{{- if .Values.certificates.ca.renewBefore }}
renewBefore: {{ .Values.certificates.ca.renewBefore }}
{{- end }}
issuerRef:
name: {{ template "seaweedfs.name" . }}-issuer
kind: Issuer
@@ -392,12 +392,10 @@ spec:
nodeSelector:
{{ tpl .Values.filer.nodeSelector . | indent 8 | trim }}
{{- end }}
{{- if and (.Values.filer.enablePVC) (not .Values.filer.data) }}
{{- if and (.Values.filer.enablePVC) (eq .Values.filer.data.type "persistentVolumeClaim") }}
# DEPRECATION: Deprecate in favor of filer.data section below
volumeClaimTemplates:
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
- metadata:
name: data-filer
spec:
accessModes:
@@ -413,9 +411,7 @@ spec:
{{- if $pvc_exists }}
volumeClaimTemplates:
{{- if eq .Values.filer.data.type "persistentVolumeClaim" }}
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
- metadata:
name: data-filer
{{- with .Values.filer.data.annotations }}
annotations:
@@ -429,9 +425,7 @@ spec:
storage: {{ .Values.filer.data.size }}
{{- end }}
{{- if eq .Values.filer.logs.type "persistentVolumeClaim" }}
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
- metadata:
name: seaweedfs-filer-log-volume
{{- with .Values.filer.logs.annotations }}
annotations:
@@ -327,9 +327,7 @@ spec:
{{- if $pvc_exists }}
volumeClaimTemplates:
{{- if eq .Values.master.data.type "persistentVolumeClaim"}}
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
- metadata:
name: data-{{ .Release.Namespace }}
{{- with .Values.master.data.annotations }}
annotations:
@@ -343,9 +341,7 @@ spec:
storage: {{ .Values.master.data.size }}
{{- end }}
{{- if eq .Values.master.logs.type "persistentVolumeClaim"}}
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
- metadata:
name: seaweedfs-master-log-volume
{{- with .Values.master.logs.annotations }}
annotations:
+38 -43
View File
@@ -22,8 +22,6 @@ global:
serviceAccountName: "seaweedfs"
automountServiceAccountToken: true
certificates:
duration: 87600h
renewBefore: 720h
alphacrds: false
monitoring:
enabled: false
@@ -237,27 +235,27 @@ master:
ingress:
enabled: false
className: ""
className: "nginx"
# host: false for "*" hostname
host: "master.seaweedfs.local"
path: "/sw-master/?(.*)"
pathType: ImplementationSpecific
annotations: {}
# nginx.ingress.kubernetes.io/auth-type: "basic"
# nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
# nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Master'
# nginx.ingress.kubernetes.io/service-upstream: "true"
# nginx.ingress.kubernetes.io/rewrite-target: /$1
# nginx.ingress.kubernetes.io/use-regex: "true"
# nginx.ingress.kubernetes.io/enable-rewrite-log: "true"
# nginx.ingress.kubernetes.io/ssl-redirect: "false"
# nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
# nginx.ingress.kubernetes.io/configuration-snippet: |
# sub_filter '<head>' '<head> <base href="/sw-master/">'; #add base url
# sub_filter '="/' '="./'; #make absolute paths to relative
# sub_filter '=/' '=./';
# sub_filter '/seaweedfsstatic' './seaweedfsstatic';
# sub_filter_once off;
annotations:
nginx.ingress.kubernetes.io/auth-type: "basic"
nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Master'
nginx.ingress.kubernetes.io/service-upstream: "true"
nginx.ingress.kubernetes.io/rewrite-target: /$1
nginx.ingress.kubernetes.io/use-regex: "true"
nginx.ingress.kubernetes.io/enable-rewrite-log: "true"
nginx.ingress.kubernetes.io/ssl-redirect: "false"
nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
nginx.ingress.kubernetes.io/configuration-snippet: |
sub_filter '<head>' '<head> <base href="/sw-master/">'; #add base url
sub_filter '="/' '="./'; #make absolute paths to relative
sub_filter '=/' '=./';
sub_filter '/seaweedfsstatic' './seaweedfsstatic';
sub_filter_once off;
tls: []
extraEnvironmentVars:
@@ -771,28 +769,28 @@ filer:
ingress:
enabled: false
className: ""
className: "nginx"
# host: false for "*" hostname
host: "seaweedfs.cluster.local"
path: "/sw-filer/?(.*)"
pathType: ImplementationSpecific
annotations: {}
# nginx.ingress.kubernetes.io/backend-protocol: GRPC
# nginx.ingress.kubernetes.io/auth-type: "basic"
# nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
# nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Filer'
# nginx.ingress.kubernetes.io/service-upstream: "true"
# nginx.ingress.kubernetes.io/rewrite-target: /$1
# nginx.ingress.kubernetes.io/use-regex: "true"
# nginx.ingress.kubernetes.io/enable-rewrite-log: "true"
# nginx.ingress.kubernetes.io/ssl-redirect: "false"
# nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
# nginx.ingress.kubernetes.io/configuration-snippet: |
# sub_filter '<head>' '<head> <base href="/sw-filer/">'; #add base url
# sub_filter '="/' '="./'; #make absolute paths to relative
# sub_filter '=/' '=./';
# sub_filter '/seaweedfsstatic' './seaweedfsstatic';
# sub_filter_once off;
annotations:
nginx.ingress.kubernetes.io/backend-protocol: GRPC
nginx.ingress.kubernetes.io/auth-type: "basic"
nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Filer'
nginx.ingress.kubernetes.io/service-upstream: "true"
nginx.ingress.kubernetes.io/rewrite-target: /$1
nginx.ingress.kubernetes.io/use-regex: "true"
nginx.ingress.kubernetes.io/enable-rewrite-log: "true"
nginx.ingress.kubernetes.io/ssl-redirect: "false"
nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
nginx.ingress.kubernetes.io/configuration-snippet: |
sub_filter '<head>' '<head> <base href="/sw-filer/">'; #add base url
sub_filter '="/' '="./'; #make absolute paths to relative
sub_filter '=/' '=./';
sub_filter '/seaweedfsstatic' './seaweedfsstatic';
sub_filter_once off;
# extraEnvVars is a list of extra environment variables to set with the stateful set.
extraEnvironmentVars:
@@ -875,7 +873,7 @@ filer:
# anonymousRead: false
s3:
enabled: false
enabled: true
imageOverride: null
restartPolicy: null
replicas: 1
@@ -981,7 +979,7 @@ s3:
# Custom command line arguments to add to the s3 command
# Example to fix connection idle seconds:
extraArgs: ["-idleTimeout=30"]
# extraArgs: []
#extraArgs: []
# used to configure livenessProbe on s3 containers
#
@@ -1011,7 +1009,7 @@ s3:
ingress:
enabled: false
className: ""
className: "nginx"
# host: false for "*" hostname
host: "seaweedfs.cluster.local"
path: "/"
@@ -1270,9 +1268,6 @@ certificates:
keySize: 2048
duration: 2160h # 90d
renewBefore: 360h # 15d
ca:
duration: 87600h # 10 years
renewBefore: 720h # 30d
externalCertificates:
# This will avoid the need to use cert-manager and will rely on providing your own external certificates and CA
# you will need to store your provided certificates in the secret read by the different services:
+2 -2
View File
@@ -5,7 +5,7 @@
<groupId>com.seaweedfs</groupId>
<artifactId>seaweedfs-client</artifactId>
<version>4.00</version>
<version>3.80</version>
<name>SeaweedFS Java Client</name>
<description>A java client for SeaweedFS.</description>
@@ -33,7 +33,7 @@
<properties>
<protobuf.version>3.25.5</protobuf.version>
<!-- follow https://github.com/grpc/grpc-java -->
<grpc.version>1.77.0</grpc.version>
<grpc.version>1.75.0</grpc.version>
<guava.version>32.0.0-jre</guava.version>
</properties>
@@ -8,13 +8,10 @@ import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
public class FilerGrpcClient {
@@ -22,10 +19,6 @@ public class FilerGrpcClient {
private static final SslContext sslContext;
private static final String protocol;
// gRPC keepalive settings - must be consistent with server-side settings in grpc_client_server.go
private static final long KEEP_ALIVE_TIME_SECONDS = 60L;
private static final long KEEP_ALIVE_TIMEOUT_SECONDS = 20L;
static {
sslContext = FilerSecurityContext.getGrpcSslContext();
protocol = FilerSecurityContext.isHttpSecurityEnabled() ? "https" : "http";
@@ -36,12 +29,10 @@ public class FilerGrpcClient {
public final int VOLUME_SERVER_ACCESS_FILER_PROXY = 2;
public final Map<String, FilerProto.Locations> vidLocations = new HashMap<>();
protected int randomClientId;
// Connection pool to handle concurrent requests
private static final int CHANNEL_POOL_SIZE = 4;
private final List<ManagedChannel> channelPool;
private final AtomicInteger channelIndex = new AtomicInteger(0);
private final ManagedChannel channel;
private final SeaweedFilerGrpc.SeaweedFilerBlockingStub blockingStub;
private final SeaweedFilerGrpc.SeaweedFilerStub asyncStub;
private final SeaweedFilerGrpc.SeaweedFilerFutureStub futureStub;
private boolean cipher = false;
private String collection = "";
private String replication = "";
@@ -54,18 +45,26 @@ public class FilerGrpcClient {
public FilerGrpcClient(String host, int port, int grpcPort, String cn, SslContext sslContext) {
this(sslContext == null ?
ManagedChannelBuilder.forAddress(host, grpcPort)
.usePlaintext()
.maxInboundMessageSize(1024 * 1024 * 1024) :
cn.isEmpty() ?
NettyChannelBuilder.forAddress(host, grpcPort)
.maxInboundMessageSize(1024 * 1024 * 1024)
.negotiationType(NegotiationType.TLS)
.sslContext(sslContext) :
NettyChannelBuilder.forAddress(host, grpcPort)
.maxInboundMessageSize(1024 * 1024 * 1024)
.negotiationType(NegotiationType.TLS)
.overrideAuthority(cn) //will not check hostname of the filer server
.sslContext(sslContext)
);
filerAddress = SeaweedUtil.joinHostPort(host, port);
// Create a pool of channels for better concurrency handling
channelPool = new ArrayList<>(CHANNEL_POOL_SIZE);
for (int i = 0; i < CHANNEL_POOL_SIZE; i++) {
channelPool.add(createChannelBuilder(host, grpcPort, sslContext, cn).build());
}
// Get filer configuration using first channel
FilerProto.GetFilerConfigurationResponse filerConfigurationResponse = SeaweedFilerGrpc
.newBlockingStub(channelPool.get(0)).getFilerConfiguration(
FilerProto.GetFilerConfigurationResponse filerConfigurationResponse =
this.getBlockingStub().getFilerConfiguration(
FilerProto.GetFilerConfigurationRequest.newBuilder().build());
cipher = filerConfigurationResponse.getCipher();
collection = filerConfigurationResponse.getCollection();
@@ -74,39 +73,11 @@ public class FilerGrpcClient {
}
/**
* Creates a NettyChannelBuilder with common gRPC configuration.
* Supports plaintext and TLS modes with optional authority override.
*/
private NettyChannelBuilder createChannelBuilder(String host, int grpcPort, SslContext sslContext, String cn) {
NettyChannelBuilder builder = NettyChannelBuilder.forAddress(host, grpcPort)
.maxInboundMessageSize(1024 * 1024 * 1024)
.maxInboundMetadataSize(1024 * 1024)
.flowControlWindow(16 * 1024 * 1024)
.initialFlowControlWindow(16 * 1024 * 1024)
.maxHeaderListSize(16 * 1024 * 1024)
.keepAliveTime(KEEP_ALIVE_TIME_SECONDS, TimeUnit.SECONDS)
.keepAliveTimeout(KEEP_ALIVE_TIMEOUT_SECONDS, TimeUnit.SECONDS)
.keepAliveWithoutCalls(true)
.withOption(io.grpc.netty.shaded.io.netty.channel.ChannelOption.SO_RCVBUF, 16 * 1024 * 1024)
.withOption(io.grpc.netty.shaded.io.netty.channel.ChannelOption.SO_SNDBUF, 16 * 1024 * 1024);
if (sslContext == null) {
builder.usePlaintext();
} else {
builder.negotiationType(NegotiationType.TLS).sslContext(sslContext);
if (!cn.isEmpty()) {
builder.overrideAuthority(cn);
}
}
return builder;
}
// Get a channel from the pool using round-robin
private ManagedChannel getChannel() {
int raw = channelIndex.getAndIncrement();
int index = Math.floorMod(raw, CHANNEL_POOL_SIZE);
return channelPool.get(index);
private FilerGrpcClient(ManagedChannelBuilder<?> channelBuilder) {
channel = channelBuilder.build();
blockingStub = SeaweedFilerGrpc.newBlockingStub(channel);
asyncStub = SeaweedFilerGrpc.newStub(channel);
futureStub = SeaweedFilerGrpc.newFutureStub(channel);
}
public boolean isCipher() {
@@ -122,25 +93,19 @@ public class FilerGrpcClient {
}
public void shutdown() throws InterruptedException {
for (ManagedChannel channel : channelPool) {
channel.shutdown();
}
for (ManagedChannel channel : channelPool) {
channel.awaitTermination(5, TimeUnit.SECONDS);
}
channel.shutdown().awaitTermination(5, TimeUnit.SECONDS);
}
public SeaweedFilerGrpc.SeaweedFilerBlockingStub getBlockingStub() {
// Return a new stub using a channel from the pool (round-robin)
return SeaweedFilerGrpc.newBlockingStub(getChannel());
return blockingStub;
}
public SeaweedFilerGrpc.SeaweedFilerStub getAsyncStub() {
return SeaweedFilerGrpc.newStub(getChannel());
return asyncStub;
}
public SeaweedFilerGrpc.SeaweedFilerFutureStub getFutureStub() {
return SeaweedFilerGrpc.newFutureStub(getChannel());
return futureStub;
}
public void setAccessVolumeServerDirectly() {
@@ -23,7 +23,7 @@ public class SeaweedInputStream extends InputStream {
private final long contentLength;
private FilerProto.Entry entry;
private long position = 0; // cursor of the file
private long position = 0; // cursor of the file
private boolean closed = false;
@@ -44,6 +44,7 @@ public class SeaweedInputStream extends InputStream {
}
this.contentLength = SeaweedRead.fileSize(entry);
this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerClient, entry.getChunksList());
LOG.debug("new path:{} entry:{} visibleIntervalList:{}", path, entry, visibleIntervalList);
@@ -63,6 +64,7 @@ public class SeaweedInputStream extends InputStream {
}
this.contentLength = SeaweedRead.fileSize(entry);
this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerClient, entry.getChunksList());
LOG.debug("new path:{} entry:{} visibleIntervalList:{}", path, entry, visibleIntervalList);
@@ -97,8 +99,7 @@ public class SeaweedInputStream extends InputStream {
throw new IllegalArgumentException("requested read length is less than zero");
}
if (len > (b.length - off)) {
throw new IllegalArgumentException(
"requested read length is more than will fit after requested offset in buffer");
throw new IllegalArgumentException("requested read length is more than will fit after requested offset in buffer");
}
ByteBuffer buf = ByteBuffer.wrap(b, off, len);
@@ -113,30 +114,21 @@ public class SeaweedInputStream extends InputStream {
throw new IllegalArgumentException("attempting to read from negative offset");
}
if (position >= contentLength) {
return -1; // Hadoop prefers -1 to EOFException
return -1; // Hadoop prefers -1 to EOFException
}
long bytesRead = 0;
int len = buf.remaining();
if (this.position < Integer.MAX_VALUE && (this.position + len) <= entry.getContent().size()) {
entry.getContent().substring((int) this.position, (int) (this.position + len)).copyTo(buf);
bytesRead = len; // FIX: Update bytesRead after inline copy
if (this.position< Integer.MAX_VALUE && (this.position + len )<= entry.getContent().size()) {
entry.getContent().substring((int)this.position, (int)(this.position + len)).copyTo(buf);
} else {
// Use the known contentLength instead of recomputing from the entry to avoid
// races
bytesRead = SeaweedRead.read(this.filerClient, this.visibleIntervalList, this.position, buf,
this.contentLength);
bytesRead = SeaweedRead.read(this.filerClient, this.visibleIntervalList, this.position, buf, SeaweedRead.fileSize(entry));
}
if (bytesRead > Integer.MAX_VALUE) {
throw new IOException("Unexpected Content-Length");
}
// Clamp premature EOFs: do not return -1 unless position >= contentLength
if (bytesRead < 0 && position < contentLength) {
bytesRead = 0;
}
if (bytesRead > 0) {
this.position += bytesRead;
}
@@ -196,15 +188,12 @@ public class SeaweedInputStream extends InputStream {
}
final long remaining = this.contentLength - this.position;
return remaining <= Integer.MAX_VALUE
? (int) remaining
: Integer.MAX_VALUE;
? (int) remaining : Integer.MAX_VALUE;
}
/**
* Returns the length of the file that this stream refers to. Note that the
* length returned is the length
* as of the time the Stream was opened. Specifically, if there have been
* subsequent appends to the file,
* Returns the length of the file that this stream refers to. Note that the length returned is the length
* as of the time the Stream was opened. Specifically, if there have been subsequent appends to the file,
* they wont be reflected in the returned length.
*
* @return length of the file.
@@ -23,13 +23,15 @@ public class SeaweedOutputStream extends OutputStream {
private final ThreadPoolExecutor threadExecutor;
private final ExecutorCompletionService<Void> completionService;
private final ConcurrentLinkedDeque<WriteOperation> writeOperations;
private final boolean shouldSaveMetadata = false;
private FilerProto.Entry.Builder entry;
private long position; // Flushed bytes (committed to service)
private long position;
private boolean closed;
private volatile IOException lastError;
private long lastFlushOffset;
private long lastTotalAppendOffset = 0;
private ByteBuffer buffer;
private long outputIndex;
private String replication = "";
private String collection = "";
@@ -42,8 +44,7 @@ public class SeaweedOutputStream extends OutputStream {
}
public SeaweedOutputStream(FilerClient filerClient, final String path, FilerProto.Entry.Builder entry,
final long position, final int bufferSize, final String replication) {
final long position, final int bufferSize, final String replication) {
this.filerClient = filerClient;
this.replication = replication;
this.path = path;
@@ -57,7 +58,8 @@ public class SeaweedOutputStream extends OutputStream {
this.maxConcurrentRequestCount = Runtime.getRuntime().availableProcessors();
this.threadExecutor = new ThreadPoolExecutor(maxConcurrentRequestCount,
this.threadExecutor
= new ThreadPoolExecutor(maxConcurrentRequestCount,
maxConcurrentRequestCount,
120L,
TimeUnit.SECONDS,
@@ -75,7 +77,8 @@ public class SeaweedOutputStream extends OutputStream {
.setFileMode(0755)
.setCrtime(now)
.setMtime(now)
.clearGroupName());
.clearGroupName()
);
}
}
@@ -83,35 +86,14 @@ public class SeaweedOutputStream extends OutputStream {
public void setReplication(String replication) {
this.replication = replication;
}
public void setCollection(String collection) {
this.collection = collection;
}
/**
* Get the current position in the output stream.
* This returns the total position including both flushed and buffered data.
*
* @return current position (flushed + buffered bytes)
*/
public synchronized long getPos() throws IOException {
// Guard against NPE if called after close()
if (buffer == null) {
return position;
}
// Return current position (flushed + buffered)
return position + buffer.position();
}
public static String getParentDirectory(String path) {
int protoIndex = path.indexOf("://");
if (protoIndex >= 0) {
int pathStart = path.indexOf("/", protoIndex + 3);
if (pathStart < 0) {
// No path segment; treat as root (e.g., "seaweedfs://host")
return "/";
}
int pathStart = path.indexOf("/", protoIndex+3);
path = path.substring(pathStart);
}
if (path.equals("/")) {
@@ -134,13 +116,6 @@ public class SeaweedOutputStream extends OutputStream {
private synchronized void flushWrittenBytesToServiceInternal(final long offset) throws IOException {
try {
// Set the file size in attributes based on our position
// This ensures Parquet footer metadata matches what we actually wrote
FilerProto.FuseAttributes.Builder attrBuilder = entry.getAttributes().toBuilder();
attrBuilder.setFileSize(offset);
entry.setAttributes(attrBuilder);
SeaweedWrite.writeMeta(filerClient, getParentDirectory(path), entry);
} catch (Exception ex) {
throw new IOException(ex);
@@ -150,7 +125,7 @@ public class SeaweedOutputStream extends OutputStream {
@Override
public void write(final int byteVal) throws IOException {
write(new byte[] { (byte) (byteVal & 0xFF) });
write(new byte[]{(byte) (byteVal & 0xFF)});
}
@Override
@@ -166,6 +141,8 @@ public class SeaweedOutputStream extends OutputStream {
throw new IndexOutOfBoundsException();
}
// System.out.println(path + " write [" + (outputIndex + off) + "," + ((outputIndex + off) + length) + ")");
int currentOffset = off;
int writableBytes = bufferSize - buffer.position();
int numberOfBytesToWrite = length;
@@ -177,11 +154,9 @@ public class SeaweedOutputStream extends OutputStream {
break;
}
// System.out.println(path + " [" + (outputIndex + currentOffset) + "," +
// ((outputIndex + currentOffset) + writableBytes) + ") " + buffer.capacity());
// System.out.println(path + " [" + (outputIndex + currentOffset) + "," + ((outputIndex + currentOffset) + writableBytes) + ") " + buffer.capacity());
buffer.put(data, currentOffset, writableBytes);
currentOffset += writableBytes;
writeCurrentBufferToService();
numberOfBytesToWrite = numberOfBytesToWrite - writableBytes;
writableBytes = bufferSize - buffer.position();
@@ -216,6 +191,7 @@ public class SeaweedOutputStream extends OutputStream {
return;
}
LOG.debug("close path: {}", path);
try {
flushInternal();
threadExecutor.shutdown();
@@ -233,35 +209,28 @@ public class SeaweedOutputStream extends OutputStream {
}
private synchronized void writeCurrentBufferToService() throws IOException {
int bufferPos = buffer.position();
if (bufferPos == 0) {
if (buffer.position() == 0) {
return;
}
int written = submitWriteBufferToService(buffer, position);
position += written;
position += submitWriteBufferToService(buffer, position);
buffer = ByteBufferPool.request(bufferSize);
}
private synchronized int submitWriteBufferToService(final ByteBuffer bufferToWrite, final long writePosition)
throws IOException {
private synchronized int submitWriteBufferToService(final ByteBuffer bufferToWrite, final long writePosition) throws IOException {
((Buffer) bufferToWrite).flip();
((Buffer)bufferToWrite).flip();
int bytesLength = bufferToWrite.limit() - bufferToWrite.position();
if (threadExecutor.getQueue().size() >= maxConcurrentRequestCount) {
waitForTaskToComplete();
}
final Future<Void> job = completionService.submit(() -> {
// System.out.println(path + " is going to save [" + (writePosition) + "," +
// ((writePosition) + bytesLength) + ")");
SeaweedWrite.writeData(entry, replication, collection, filerClient, writePosition, bufferToWrite.array(),
bufferToWrite.position(), bufferToWrite.limit(), path);
// System.out.println(path + " saved [" + (writePosition) + "," +
// ((writePosition) + bytesLength) + ")");
// System.out.println(path + " is going to save [" + (writePosition) + "," + ((writePosition) + bytesLength) + ")");
SeaweedWrite.writeData(entry, replication, collection, filerClient, writePosition, bufferToWrite.array(), bufferToWrite.position(), bufferToWrite.limit(), path);
// System.out.println(path + " saved [" + (writePosition) + "," + ((writePosition) + bytesLength) + ")");
ByteBufferPool.release(bufferToWrite);
return null;
});
@@ -349,10 +318,12 @@ public class SeaweedOutputStream extends OutputStream {
private static class WriteOperation {
private final Future<Void> task;
private final long startOffset;
private final long length;
WriteOperation(final Future<Void> task, final long startOffset, final long length) {
this.task = task;
this.startOffset = startOffset;
this.length = length;
}
}
@@ -24,10 +24,9 @@ public class SeaweedRead {
// returns bytesRead
public static long read(FilerClient filerClient, List<VisibleInterval> visibleIntervals,
final long position, final ByteBuffer buf, final long fileSize) throws IOException {
final long position, final ByteBuffer buf, final long fileSize) throws IOException {
int originalRemaining = buf.remaining();
List<ChunkView> chunkViews = viewFromVisibles(visibleIntervals, position, originalRemaining);
List<ChunkView> chunkViews = viewFromVisibles(visibleIntervals, position, buf.remaining());
Map<String, FilerProto.Locations> knownLocations = new HashMap<>();
@@ -52,7 +51,7 @@ public class SeaweedRead {
}
}
// TODO parallel this
//TODO parallel this
long readCount = 0;
long startOffset = position;
for (ChunkView chunkView : chunkViews) {
@@ -60,7 +59,7 @@ public class SeaweedRead {
if (startOffset < chunkView.logicOffset) {
long gap = chunkView.logicOffset - startOffset;
LOG.debug("zero [{},{})", startOffset, startOffset + gap);
buf.position(buf.position() + (int) gap);
buf.position(buf.position()+ (int)gap);
readCount += gap;
startOffset += gap;
}
@@ -82,17 +81,12 @@ public class SeaweedRead {
}
// Fix: Calculate the correct limit based on the read position and requested
// size,
// not the buffer's absolute limit. This fixes the 78-byte EOF error when
// seeking
// near the end of the file.
long limit = Math.min(position + originalRemaining, fileSize);
long limit = Math.min(buf.limit(), fileSize);
if (startOffset < limit) {
long gap = limit - startOffset;
LOG.debug("zero2 [{},{})", startOffset, startOffset + gap);
buf.position(buf.position() + (int) gap);
buf.position(buf.position()+ (int)gap);
readCount += gap;
startOffset += gap;
}
@@ -100,8 +94,7 @@ public class SeaweedRead {
return readCount;
}
private static int readChunkView(FilerClient filerClient, long startOffset, ByteBuffer buf, ChunkView chunkView,
FilerProto.Locations locations) throws IOException {
private static int readChunkView(FilerClient filerClient, long startOffset, ByteBuffer buf, ChunkView chunkView, FilerProto.Locations locations) throws IOException {
byte[] chunkData = chunkCache.getChunk(chunkView.fileId);
@@ -112,15 +105,13 @@ public class SeaweedRead {
int len = (int) chunkView.size - (int) (startOffset - chunkView.logicOffset);
LOG.debug("readChunkView fid:{} chunkData.length:{} chunkView.offset:{} chunkView[{};{}) startOffset:{}",
chunkView.fileId, chunkData.length, chunkView.offset, chunkView.logicOffset,
chunkView.logicOffset + chunkView.size, startOffset);
chunkView.fileId, chunkData.length, chunkView.offset, chunkView.logicOffset, chunkView.logicOffset + chunkView.size, startOffset);
buf.put(chunkData, (int) (startOffset - chunkView.logicOffset + chunkView.offset), len);
return len;
}
public static byte[] doFetchFullChunkData(FilerClient filerClient, ChunkView chunkView,
FilerProto.Locations locations) throws IOException {
public static byte[] doFetchFullChunkData(FilerClient filerClient, ChunkView chunkView, FilerProto.Locations locations) throws IOException {
byte[] data = null;
IOException lastException = null;
@@ -223,7 +214,8 @@ public class SeaweedRead {
chunkStart,
isFullChunk,
chunk.cipherKey,
chunk.isCompressed));
chunk.isCompressed
));
}
}
return views;
@@ -247,10 +239,7 @@ public class SeaweedRead {
}
public static long fileSize(FilerProto.Entry entry) {
long chunksSize = totalSize(entry.getChunksList());
long attrSize = entry.getAttributes().getFileSize();
long finalSize = Math.max(chunksSize, attrSize);
return finalSize;
return Math.max(totalSize(entry.getChunksList()), entry.getAttributes().getFileSize());
}
public static long totalSize(List<FilerProto.FileChunk> chunksList) {
@@ -274,8 +263,7 @@ public class SeaweedRead {
public final byte[] cipherKey;
public final boolean isCompressed;
public VisibleInterval(long start, long stop, String fileId, long modifiedTime, long chunkOffset,
boolean isFullChunk, byte[] cipherKey, boolean isCompressed) {
public VisibleInterval(long start, long stop, String fileId, long modifiedTime, long chunkOffset, boolean isFullChunk, byte[] cipherKey, boolean isCompressed) {
this.start = start;
this.stop = stop;
this.modifiedTime = modifiedTime;
@@ -309,8 +297,7 @@ public class SeaweedRead {
public final byte[] cipherKey;
public final boolean isCompressed;
public ChunkView(String fileId, long offset, long size, long logicOffset, boolean isFullChunk, byte[] cipherKey,
boolean isCompressed) {
public ChunkView(String fileId, long offset, long size, long logicOffset, boolean isFullChunk, byte[] cipherKey, boolean isCompressed) {
this.fileId = fileId;
this.offset = offset;
this.size = size;
@@ -2,7 +2,6 @@ package seaweedfs.client;
import com.google.common.base.Strings;
import com.google.protobuf.ByteString;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.mime.HttpMultipartMode;
@@ -26,13 +25,13 @@ public class SeaweedWrite {
private static final SecureRandom random = new SecureRandom();
public static void writeData(FilerProto.Entry.Builder entry,
final String replication,
String collection,
final FilerClient filerClient,
final long offset,
final byte[] bytes,
final long bytesOffset, final long bytesLength,
final String path) throws IOException {
final String replication,
String collection,
final FilerClient filerClient,
final long offset,
final byte[] bytes,
final long bytesOffset, final long bytesLength,
final String path) throws IOException {
IOException lastException = null;
for (long waitTime = 1000L; waitTime < 10 * 1000; waitTime += waitTime / 2) {
@@ -61,50 +60,21 @@ public class SeaweedWrite {
}
public static FilerProto.FileChunk.Builder writeChunk(final String replication,
final String collection,
final FilerClient filerClient,
final long offset,
final byte[] bytes,
final long bytesOffset,
final long bytesLength,
final String path) throws IOException {
// Retry assignVolume call for transient network/server errors
FilerProto.AssignVolumeResponse response = null;
IOException lastException = null;
int maxRetries = 3;
for (int attempt = 0; attempt < maxRetries; attempt++) {
try {
response = filerClient.getBlockingStub().assignVolume(
FilerProto.AssignVolumeRequest.newBuilder()
.setCollection(
Strings.isNullOrEmpty(collection) ? filerClient.getCollection() : collection)
.setReplication(
Strings.isNullOrEmpty(replication) ? filerClient.getReplication() : replication)
.setDataCenter("")
.setTtlSec(0)
.setPath(path)
.build());
break; // Success, exit retry loop
} catch (io.grpc.StatusRuntimeException e) {
lastException = new IOException(
"assignVolume failed (attempt " + (attempt + 1) + "/" + maxRetries + "): " + e.getMessage(), e);
if (attempt < maxRetries - 1) {
try {
Thread.sleep(100 * (attempt + 1)); // Exponential backoff: 100ms, 200ms, 300ms
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
throw new IOException("Interrupted during retry", ie);
}
}
}
}
if (response == null) {
throw lastException != null ? lastException
: new IOException("assignVolume failed after " + maxRetries + " attempts");
}
final String collection,
final FilerClient filerClient,
final long offset,
final byte[] bytes,
final long bytesOffset,
final long bytesLength,
final String path) throws IOException {
FilerProto.AssignVolumeResponse response = filerClient.getBlockingStub().assignVolume(
FilerProto.AssignVolumeRequest.newBuilder()
.setCollection(Strings.isNullOrEmpty(collection) ? filerClient.getCollection() : collection)
.setReplication(Strings.isNullOrEmpty(replication) ? filerClient.getReplication() : replication)
.setDataCenter("")
.setTtlSec(0)
.setPath(path)
.build());
if (!Strings.isNullOrEmpty(response.getError())) {
throw new IOException(response.getError());
@@ -113,8 +83,7 @@ public class SeaweedWrite {
String fileId = response.getFileId();
String auth = response.getAuth();
String targetUrl = filerClient.getChunkUrl(fileId, response.getLocation().getUrl(),
response.getLocation().getPublicUrl());
String targetUrl = filerClient.getChunkUrl(fileId, response.getLocation().getUrl(), response.getLocation().getPublicUrl());
ByteString cipherKeyString = com.google.protobuf.ByteString.EMPTY;
byte[] cipherKey = null;
@@ -125,6 +94,8 @@ public class SeaweedWrite {
String etag = multipartUpload(targetUrl, auth, bytes, bytesOffset, bytesLength, cipherKey);
LOG.debug("write file chunk {} size {}", targetUrl, bytesLength);
return FilerProto.FileChunk.newBuilder()
.setFileId(fileId)
.setOffset(offset)
@@ -135,28 +106,27 @@ public class SeaweedWrite {
}
public static void writeMeta(final FilerClient filerClient,
final String parentDirectory,
final FilerProto.Entry.Builder entry) throws IOException {
final String parentDirectory,
final FilerProto.Entry.Builder entry) throws IOException {
synchronized (entry) {
List<FilerProto.FileChunk> chunks = FileChunkManifest.maybeManifestize(filerClient, entry.getChunksList(),
parentDirectory);
List<FilerProto.FileChunk> chunks = FileChunkManifest.maybeManifestize(filerClient, entry.getChunksList(), parentDirectory);
entry.clearChunks();
entry.addAllChunks(chunks);
filerClient.getBlockingStub().createEntry(
FilerProto.CreateEntryRequest.newBuilder()
.setDirectory(parentDirectory)
.setEntry(entry)
.build());
.build()
);
}
}
private static String multipartUpload(String targetUrl,
String auth,
final byte[] bytes,
final long bytesOffset, final long bytesLength,
byte[] cipherKey) throws IOException {
String auth,
final byte[] bytes,
final long bytesOffset, final long bytesLength,
byte[] cipherKey) throws IOException {
MessageDigest md = null;
try {
md = MessageDigest.getInstance("MD5");
@@ -192,10 +162,8 @@ public class SeaweedWrite {
try {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
HttpEntity entity = response.getEntity();
if (entity != null && entity.getContentType() != null
&& entity.getContentType().getValue().equals("application/json")) {
throw new IOException(EntityUtils.toString(entity, "UTF-8"));
if (response.getEntity().getContentType() != null && response.getEntity().getContentType().getValue().equals("application/json")) {
throw new IOException(EntityUtils.toString(response.getEntity(), "UTF-8"));
} else {
throw new IOException(response.getStatusLine().getReasonPhrase());
}
@@ -1,323 +0,0 @@
package seaweedfs.client;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import static org.junit.Assert.*;
/**
* Integration tests for FilerClient.
*
* These tests verify FilerClient operations against a running SeaweedFS filer
* instance.
*
* Prerequisites:
* - SeaweedFS master, volume server, and filer must be running
* - Default ports: filer HTTP 8888, filer gRPC 18888
*
* To run tests:
* export SEAWEEDFS_TEST_ENABLED=true
* mvn test -Dtest=FilerClientIntegrationTest
*/
public class FilerClientIntegrationTest {
private FilerClient filerClient;
private static final String TEST_ROOT = "/test-client-integration";
private static final boolean TESTS_ENABLED = "true".equalsIgnoreCase(System.getenv("SEAWEEDFS_TEST_ENABLED"));
@Before
public void setUp() throws Exception {
if (!TESTS_ENABLED) {
return;
}
filerClient = new FilerClient("localhost", 18888);
// Clean up any existing test directory
if (filerClient.exists(TEST_ROOT)) {
filerClient.rm(TEST_ROOT, true, true);
}
// Create test root directory
filerClient.mkdirs(TEST_ROOT, 0755);
}
@After
public void tearDown() throws Exception {
if (!TESTS_ENABLED || filerClient == null) {
return;
}
try {
// Clean up test directory
if (filerClient.exists(TEST_ROOT)) {
filerClient.rm(TEST_ROOT, true, true);
}
} finally {
filerClient.shutdown();
}
}
@Test
public void testMkdirs() {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String testDir = TEST_ROOT + "/testdir";
boolean success = filerClient.mkdirs(testDir, 0755);
assertTrue("Directory creation should succeed", success);
assertTrue("Directory should exist", filerClient.exists(testDir));
}
@Test
public void testTouch() {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String testFile = TEST_ROOT + "/testfile.txt";
boolean success = filerClient.touch(testFile, 0644);
assertTrue("Touch should succeed", success);
assertTrue("File should exist", filerClient.exists(testFile));
}
@Test
public void testExists() {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
assertTrue("Root should exist", filerClient.exists("/"));
assertTrue("Test root should exist", filerClient.exists(TEST_ROOT));
assertFalse("Non-existent path should not exist",
filerClient.exists(TEST_ROOT + "/nonexistent"));
}
@Test
public void testListEntries() {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
// Create some test files and directories
filerClient.touch(TEST_ROOT + "/file1.txt", 0644);
filerClient.touch(TEST_ROOT + "/file2.txt", 0644);
filerClient.mkdirs(TEST_ROOT + "/subdir", 0755);
List<FilerProto.Entry> entries = filerClient.listEntries(TEST_ROOT);
assertNotNull("Entries should not be null", entries);
assertEquals("Should have 3 entries", 3, entries.size());
}
@Test
public void testListEntriesWithPrefix() {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
// Create test files
filerClient.touch(TEST_ROOT + "/test1.txt", 0644);
filerClient.touch(TEST_ROOT + "/test2.txt", 0644);
filerClient.touch(TEST_ROOT + "/other.txt", 0644);
List<FilerProto.Entry> entries = filerClient.listEntries(TEST_ROOT, "test", "", 100, false);
assertNotNull("Entries should not be null", entries);
assertEquals("Should have 2 entries starting with 'test'", 2, entries.size());
}
@Test
public void testDeleteFile() {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String testFile = TEST_ROOT + "/deleteme.txt";
filerClient.touch(testFile, 0644);
assertTrue("File should exist before delete", filerClient.exists(testFile));
boolean success = filerClient.rm(testFile, false, true);
assertTrue("Delete should succeed", success);
assertFalse("File should not exist after delete", filerClient.exists(testFile));
}
@Test
public void testDeleteDirectoryRecursive() {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String testDir = TEST_ROOT + "/deletedir";
filerClient.mkdirs(testDir, 0755);
filerClient.touch(testDir + "/file.txt", 0644);
assertTrue("Directory should exist", filerClient.exists(testDir));
assertTrue("File should exist", filerClient.exists(testDir + "/file.txt"));
boolean success = filerClient.rm(testDir, true, true);
assertTrue("Delete should succeed", success);
assertFalse("Directory should not exist after delete", filerClient.exists(testDir));
}
@Test
public void testRename() {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String srcFile = TEST_ROOT + "/source.txt";
String dstFile = TEST_ROOT + "/destination.txt";
filerClient.touch(srcFile, 0644);
assertTrue("Source file should exist", filerClient.exists(srcFile));
boolean success = filerClient.mv(srcFile, dstFile);
assertTrue("Rename should succeed", success);
assertFalse("Source file should not exist after rename", filerClient.exists(srcFile));
assertTrue("Destination file should exist after rename", filerClient.exists(dstFile));
}
@Test
public void testGetEntry() {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String testFile = TEST_ROOT + "/getentry.txt";
filerClient.touch(testFile, 0644);
FilerProto.Entry entry = filerClient.lookupEntry(TEST_ROOT, "getentry.txt");
assertNotNull("Entry should not be null", entry);
assertEquals("Entry name should match", "getentry.txt", entry.getName());
assertFalse("Entry should not be a directory", entry.getIsDirectory());
}
@Test
public void testGetEntryForDirectory() {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String testDir = TEST_ROOT + "/testsubdir";
filerClient.mkdirs(testDir, 0755);
FilerProto.Entry entry = filerClient.lookupEntry(TEST_ROOT, "testsubdir");
assertNotNull("Entry should not be null", entry);
assertEquals("Entry name should match", "testsubdir", entry.getName());
assertTrue("Entry should be a directory", entry.getIsDirectory());
}
@Test
public void testCreateAndListNestedDirectories() {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String nestedPath = TEST_ROOT + "/level1/level2/level3";
boolean success = filerClient.mkdirs(nestedPath, 0755);
assertTrue("Nested directory creation should succeed", success);
assertTrue("Nested directory should exist", filerClient.exists(nestedPath));
// Verify each level exists
assertTrue("Level 1 should exist", filerClient.exists(TEST_ROOT + "/level1"));
assertTrue("Level 2 should exist", filerClient.exists(TEST_ROOT + "/level1/level2"));
assertTrue("Level 3 should exist", filerClient.exists(nestedPath));
}
@Test
public void testMultipleFilesInDirectory() {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String testDir = TEST_ROOT + "/multifiles";
filerClient.mkdirs(testDir, 0755);
// Create 10 files
for (int i = 0; i < 10; i++) {
filerClient.touch(testDir + "/file" + i + ".txt", 0644);
}
List<FilerProto.Entry> entries = filerClient.listEntries(testDir);
assertNotNull("Entries should not be null", entries);
assertEquals("Should have 10 files", 10, entries.size());
}
@Test
public void testRenameDirectory() {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String srcDir = TEST_ROOT + "/sourcedir";
String dstDir = TEST_ROOT + "/destdir";
filerClient.mkdirs(srcDir, 0755);
filerClient.touch(srcDir + "/file.txt", 0644);
boolean success = filerClient.mv(srcDir, dstDir);
assertTrue("Directory rename should succeed", success);
assertFalse("Source directory should not exist", filerClient.exists(srcDir));
assertTrue("Destination directory should exist", filerClient.exists(dstDir));
assertTrue("File should exist in destination", filerClient.exists(dstDir + "/file.txt"));
}
@Test
public void testLookupNonExistentEntry() {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
FilerProto.Entry entry = filerClient.lookupEntry(TEST_ROOT, "nonexistent.txt");
assertNull("Entry for non-existent file should be null", entry);
}
@Test
public void testEmptyDirectory() {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String emptyDir = TEST_ROOT + "/emptydir";
filerClient.mkdirs(emptyDir, 0755);
List<FilerProto.Entry> entries = filerClient.listEntries(emptyDir);
assertNotNull("Entries should not be null", entries);
assertTrue("Empty directory should have no entries", entries.isEmpty());
}
}
@@ -1,303 +0,0 @@
package seaweedfs.client;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import static org.junit.Assert.*;
/**
* Unit test to reproduce the Parquet EOF issue.
*
* The issue: When Parquet writes column chunks, it calls getPos() to record
* offsets.
* If getPos() returns a position that doesn't include buffered (unflushed)
* data,
* the footer metadata will have incorrect offsets.
*
* This test simulates Parquet's behavior:
* 1. Write some data (column chunk 1)
* 2. Call getPos() - Parquet records this as the END of chunk 1
* 3. Write more data (column chunk 2)
* 4. Call getPos() - Parquet records this as the END of chunk 2
* 5. Close the file
* 6. Verify that the recorded positions match the actual file content
*
* Prerequisites:
* - SeaweedFS master, volume server, and filer must be running
* - Default ports: filer HTTP 8888, filer gRPC 18888
*
* To run:
* export SEAWEEDFS_TEST_ENABLED=true
* cd other/java/client
* mvn test -Dtest=GetPosBufferTest
*/
public class GetPosBufferTest {
private FilerClient filerClient;
private static final String TEST_ROOT = "/test-getpos-buffer";
private static final boolean TESTS_ENABLED = "true".equalsIgnoreCase(System.getenv("SEAWEEDFS_TEST_ENABLED"));
@Before
public void setUp() throws Exception {
if (!TESTS_ENABLED) {
return;
}
String filerHost = System.getenv().getOrDefault("SEAWEEDFS_FILER_HOST", "localhost");
String filerGrpcPort = System.getenv().getOrDefault("SEAWEEDFS_FILER_GRPC_PORT", "18888");
filerClient = new FilerClient(filerHost, Integer.parseInt(filerGrpcPort));
// Clean up any existing test directory
if (filerClient.exists(TEST_ROOT)) {
filerClient.rm(TEST_ROOT, true, true);
}
// Create test root directory
filerClient.mkdirs(TEST_ROOT, 0755);
}
@After
public void tearDown() throws Exception {
if (!TESTS_ENABLED) {
return;
}
if (filerClient != null) {
filerClient.rm(TEST_ROOT, true, true);
filerClient.shutdown();
}
}
@Test
public void testGetPosWithBufferedData() throws IOException {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
System.out.println("\n=== Testing getPos() with buffered data ===");
String testPath = TEST_ROOT + "/getpos-test.bin";
// Simulate what Parquet does when writing column chunks
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
// Write "column chunk 1" - 100 bytes
byte[] chunk1 = new byte[100];
for (int i = 0; i < 100; i++) {
chunk1[i] = (byte) i;
}
outputStream.write(chunk1);
// Parquet calls getPos() here to record end of chunk 1
long posAfterChunk1 = outputStream.getPos();
System.out.println("Position after chunk 1 (100 bytes): " + posAfterChunk1);
assertEquals("getPos() should return 100 after writing 100 bytes", 100, posAfterChunk1);
// Write "column chunk 2" - 200 bytes
byte[] chunk2 = new byte[200];
for (int i = 0; i < 200; i++) {
chunk2[i] = (byte) (i + 100);
}
outputStream.write(chunk2);
// Parquet calls getPos() here to record end of chunk 2
long posAfterChunk2 = outputStream.getPos();
System.out.println("Position after chunk 2 (200 more bytes): " + posAfterChunk2);
assertEquals("getPos() should return 300 after writing 300 bytes total", 300, posAfterChunk2);
// Write "column chunk 3" - small chunk of 78 bytes (the problematic size!)
byte[] chunk3 = new byte[78];
for (int i = 0; i < 78; i++) {
chunk3[i] = (byte) (i + 50);
}
outputStream.write(chunk3);
// Parquet calls getPos() here to record end of chunk 3
long posAfterChunk3 = outputStream.getPos();
System.out.println("Position after chunk 3 (78 more bytes): " + posAfterChunk3);
assertEquals("getPos() should return 378 after writing 378 bytes total", 378, posAfterChunk3);
// Close to flush everything
outputStream.close();
System.out.println("File closed successfully");
// Now read the file and verify its actual size matches what getPos() reported
FilerProto.Entry entry = filerClient.lookupEntry(
SeaweedOutputStream.getParentDirectory(testPath),
SeaweedOutputStream.getFileName(testPath));
long actualFileSize = SeaweedRead.fileSize(entry);
System.out.println("Actual file size on disk: " + actualFileSize);
assertEquals("File size should match the last getPos() value", 378, actualFileSize);
// Now read the file and verify we can read all the data
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
byte[] readBuffer = new byte[500]; // Larger buffer to read everything
int totalRead = 0;
int bytesRead;
while ((bytesRead = inputStream.read(readBuffer, totalRead, readBuffer.length - totalRead)) > 0) {
totalRead += bytesRead;
}
inputStream.close();
System.out.println("Total bytes read: " + totalRead);
assertEquals("Should read exactly 378 bytes", 378, totalRead);
// Verify the data is correct
for (int i = 0; i < 100; i++) {
assertEquals("Chunk 1 data mismatch at byte " + i, (byte) i, readBuffer[i]);
}
for (int i = 0; i < 200; i++) {
assertEquals("Chunk 2 data mismatch at byte " + (100 + i), (byte) (i + 100), readBuffer[100 + i]);
}
for (int i = 0; i < 78; i++) {
assertEquals("Chunk 3 data mismatch at byte " + (300 + i), (byte) (i + 50), readBuffer[300 + i]);
}
System.out.println("SUCCESS: All data verified correctly!\n");
}
@Test
public void testGetPosWithSmallWrites() throws IOException {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
System.out.println("\n=== Testing getPos() with many small writes (Parquet pattern) ===");
String testPath = TEST_ROOT + "/small-writes-test.bin";
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
// Parquet writes column data in small chunks and frequently calls getPos()
String[] columnData = { "Alice", "Bob", "Charlie", "David" };
long[] recordedPositions = new long[columnData.length];
for (int i = 0; i < columnData.length; i++) {
byte[] data = columnData[i].getBytes(StandardCharsets.UTF_8);
outputStream.write(data);
// Parquet calls getPos() after each value to track offsets
recordedPositions[i] = outputStream.getPos();
System.out.println("After writing '" + columnData[i] + "': pos=" + recordedPositions[i]);
}
long finalPos = outputStream.getPos();
System.out.println("Final position before close: " + finalPos);
outputStream.close();
// Verify file size
FilerProto.Entry entry = filerClient.lookupEntry(
SeaweedOutputStream.getParentDirectory(testPath),
SeaweedOutputStream.getFileName(testPath));
long actualFileSize = SeaweedRead.fileSize(entry);
System.out.println("Actual file size: " + actualFileSize);
assertEquals("File size should match final getPos()", finalPos, actualFileSize);
// Verify we can read using the recorded positions
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
long currentPos = 0;
for (int i = 0; i < columnData.length; i++) {
long nextPos = recordedPositions[i];
int length = (int) (nextPos - currentPos);
byte[] buffer = new byte[length];
int bytesRead = inputStream.read(buffer, 0, length);
assertEquals("Should read " + length + " bytes for '" + columnData[i] + "'", length, bytesRead);
String readData = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8);
System.out.println("Read at offset " + currentPos + ": '" + readData + "'");
assertEquals("Data mismatch", columnData[i], readData);
currentPos = nextPos;
}
inputStream.close();
System.out.println("SUCCESS: Small writes with getPos() tracking work correctly!\n");
}
@Test
public void testGetPosWithExactly78BytesBuffered() throws IOException {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
System.out.println("\n=== Testing getPos() with EXACTLY 78 bytes buffered (the bug size!) ===");
String testPath = TEST_ROOT + "/78-bytes-test.bin";
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
// Write some initial data
byte[] initial = new byte[1000];
for (int i = 0; i < 1000; i++) {
initial[i] = (byte) i;
}
outputStream.write(initial);
outputStream.flush(); // Ensure this is flushed
long posAfterFlush = outputStream.getPos();
System.out.println("Position after 1000 bytes + flush: " + posAfterFlush);
assertEquals("Should be at position 1000 after flush", 1000, posAfterFlush);
// Now write EXACTLY 78 bytes (the problematic buffer size in our bug)
byte[] problematicChunk = new byte[78];
for (int i = 0; i < 78; i++) {
problematicChunk[i] = (byte) (i + 50);
}
outputStream.write(problematicChunk);
// DO NOT FLUSH - this is the bug scenario!
// Parquet calls getPos() here while the 78 bytes are still buffered
long posWithBufferedData = outputStream.getPos();
System.out.println("Position with 78 bytes BUFFERED (not flushed): " + posWithBufferedData);
// This MUST return 1078, not 1000!
assertEquals("getPos() MUST include buffered data", 1078, posWithBufferedData);
// Now close (which will flush)
outputStream.close();
// Verify actual file size
FilerProto.Entry entry = filerClient.lookupEntry(
SeaweedOutputStream.getParentDirectory(testPath),
SeaweedOutputStream.getFileName(testPath));
long actualFileSize = SeaweedRead.fileSize(entry);
System.out.println("Actual file size: " + actualFileSize);
assertEquals("File size must be 1078", 1078, actualFileSize);
// Try to read at position 1000 for 78 bytes (what Parquet would try)
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
inputStream.seek(1000);
byte[] readBuffer = new byte[78];
int bytesRead = inputStream.read(readBuffer, 0, 78);
System.out.println("Bytes read at position 1000: " + bytesRead);
assertEquals("Should successfully read 78 bytes at position 1000", 78, bytesRead);
// Verify the data matches
for (int i = 0; i < 78; i++) {
assertEquals("Data mismatch at byte " + i, problematicChunk[i], readBuffer[i]);
}
inputStream.close();
System.out.println("SUCCESS: getPos() correctly includes buffered data!\n");
}
}
@@ -1,507 +0,0 @@
package seaweedfs.client;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Random;
import static org.junit.Assert.*;
/**
* Integration tests for SeaweedInputStream and SeaweedOutputStream.
*
* These tests verify stream operations against a running SeaweedFS instance.
*
* Prerequisites:
* - SeaweedFS master, volume server, and filer must be running
* - Default ports: filer HTTP 8888, filer gRPC 18888
*
* To run tests:
* export SEAWEEDFS_TEST_ENABLED=true
* mvn test -Dtest=SeaweedStreamIntegrationTest
*/
public class SeaweedStreamIntegrationTest {
private FilerClient filerClient;
private static final String TEST_ROOT = "/test-stream-integration";
private static final boolean TESTS_ENABLED = "true".equalsIgnoreCase(System.getenv("SEAWEEDFS_TEST_ENABLED"));
@Before
public void setUp() throws Exception {
if (!TESTS_ENABLED) {
return;
}
filerClient = new FilerClient("localhost", 18888);
// Clean up any existing test directory
if (filerClient.exists(TEST_ROOT)) {
filerClient.rm(TEST_ROOT, true, true);
}
// Create test root directory
filerClient.mkdirs(TEST_ROOT, 0755);
}
@After
public void tearDown() throws Exception {
if (!TESTS_ENABLED || filerClient == null) {
return;
}
try {
// Clean up test directory
if (filerClient.exists(TEST_ROOT)) {
filerClient.rm(TEST_ROOT, true, true);
}
} finally {
filerClient.shutdown();
}
}
@Test
public void testWriteAndReadSmallFile() throws IOException {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String testPath = TEST_ROOT + "/small.txt";
String testContent = "Hello, SeaweedFS!";
// Write file
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8));
outputStream.close();
// Verify file exists
assertTrue("File should exist", filerClient.exists(testPath));
// Read file
FilerProto.Entry entry = filerClient.lookupEntry(
SeaweedOutputStream.getParentDirectory(testPath),
SeaweedOutputStream.getFileName(testPath));
assertNotNull("Entry should not be null", entry);
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
byte[] buffer = new byte[testContent.length()];
int bytesRead = inputStream.read(buffer);
inputStream.close();
assertEquals("Should read all bytes", testContent.length(), bytesRead);
assertEquals("Content should match", testContent, new String(buffer, StandardCharsets.UTF_8));
}
@Test
public void testWriteAndReadLargeFile() throws IOException {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String testPath = TEST_ROOT + "/large.bin";
int fileSize = 10 * 1024 * 1024; // 10 MB
// Generate random data
byte[] originalData = new byte[fileSize];
new Random(42).nextBytes(originalData); // Use seed for reproducibility
// Write file
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
outputStream.write(originalData);
outputStream.close();
// Verify file exists
assertTrue("File should exist", filerClient.exists(testPath));
// Read file
FilerProto.Entry entry = filerClient.lookupEntry(
SeaweedOutputStream.getParentDirectory(testPath),
SeaweedOutputStream.getFileName(testPath));
assertNotNull("Entry should not be null", entry);
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
// Read file in chunks to handle large files properly
byte[] readData = new byte[fileSize];
int totalRead = 0;
int bytesRead;
byte[] buffer = new byte[8192]; // Read in 8KB chunks
while ((bytesRead = inputStream.read(buffer)) > 0) {
System.arraycopy(buffer, 0, readData, totalRead, bytesRead);
totalRead += bytesRead;
}
inputStream.close();
assertEquals("Should read all bytes", fileSize, totalRead);
assertArrayEquals("Content should match", originalData, readData);
}
@Test
public void testWriteInChunks() throws IOException {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String testPath = TEST_ROOT + "/chunked.txt";
String[] chunks = { "First chunk. ", "Second chunk. ", "Third chunk." };
// Write file in chunks
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
for (String chunk : chunks) {
outputStream.write(chunk.getBytes(StandardCharsets.UTF_8));
}
outputStream.close();
// Read and verify
FilerProto.Entry entry = filerClient.lookupEntry(
SeaweedOutputStream.getParentDirectory(testPath),
SeaweedOutputStream.getFileName(testPath));
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
byte[] buffer = new byte[1024];
int bytesRead = inputStream.read(buffer);
inputStream.close();
String expected = String.join("", chunks);
String actual = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8);
assertEquals("Content should match", expected, actual);
}
@Test
public void testReadWithOffset() throws IOException {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String testPath = TEST_ROOT + "/offset.txt";
String testContent = "0123456789ABCDEFGHIJ";
// Write file
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8));
outputStream.close();
// Read with offset
FilerProto.Entry entry = filerClient.lookupEntry(
SeaweedOutputStream.getParentDirectory(testPath),
SeaweedOutputStream.getFileName(testPath));
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
inputStream.seek(10); // Skip first 10 bytes
byte[] buffer = new byte[10];
int bytesRead = inputStream.read(buffer);
inputStream.close();
assertEquals("Should read 10 bytes", 10, bytesRead);
assertEquals("Should read from offset", "ABCDEFGHIJ",
new String(buffer, StandardCharsets.UTF_8));
}
@Test
public void testReadPartial() throws IOException {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String testPath = TEST_ROOT + "/partial.txt";
String testContent = "The quick brown fox jumps over the lazy dog";
// Write file
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8));
outputStream.close();
// Read partial
FilerProto.Entry entry = filerClient.lookupEntry(
SeaweedOutputStream.getParentDirectory(testPath),
SeaweedOutputStream.getFileName(testPath));
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
// Read only "quick brown"
inputStream.seek(4);
byte[] buffer = new byte[11];
int bytesRead = inputStream.read(buffer);
inputStream.close();
assertEquals("Should read 11 bytes", 11, bytesRead);
assertEquals("Should read partial content", "quick brown",
new String(buffer, StandardCharsets.UTF_8));
}
@Test
public void testEmptyFile() throws IOException {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String testPath = TEST_ROOT + "/empty.txt";
// Write empty file
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
outputStream.close();
// Verify file exists
assertTrue("File should exist", filerClient.exists(testPath));
// Read empty file
FilerProto.Entry entry = filerClient.lookupEntry(
SeaweedOutputStream.getParentDirectory(testPath),
SeaweedOutputStream.getFileName(testPath));
assertNotNull("Entry should not be null", entry);
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
byte[] buffer = new byte[100];
int bytesRead = inputStream.read(buffer);
inputStream.close();
assertEquals("Should read 0 bytes from empty file", -1, bytesRead);
}
@Test
public void testOverwriteFile() throws IOException {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String testPath = TEST_ROOT + "/overwrite.txt";
String originalContent = "Original content";
String newContent = "New content that overwrites the original";
// Write original file
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
outputStream.write(originalContent.getBytes(StandardCharsets.UTF_8));
outputStream.close();
// Overwrite file
outputStream = new SeaweedOutputStream(filerClient, testPath);
outputStream.write(newContent.getBytes(StandardCharsets.UTF_8));
outputStream.close();
// Read and verify
FilerProto.Entry entry = filerClient.lookupEntry(
SeaweedOutputStream.getParentDirectory(testPath),
SeaweedOutputStream.getFileName(testPath));
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
byte[] buffer = new byte[1024];
int bytesRead = inputStream.read(buffer);
inputStream.close();
String actual = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8);
assertEquals("Should have new content", newContent, actual);
}
@Test
public void testMultipleReads() throws IOException {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String testPath = TEST_ROOT + "/multireads.txt";
String testContent = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
// Write file
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8));
outputStream.close();
// Read in multiple small chunks
FilerProto.Entry entry = filerClient.lookupEntry(
SeaweedOutputStream.getParentDirectory(testPath),
SeaweedOutputStream.getFileName(testPath));
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
StringBuilder result = new StringBuilder();
byte[] buffer = new byte[5];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) > 0) {
result.append(new String(buffer, 0, bytesRead, StandardCharsets.UTF_8));
}
inputStream.close();
assertEquals("Should read entire content", testContent, result.toString());
}
@Test
public void testBinaryData() throws IOException {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String testPath = TEST_ROOT + "/binary.bin";
byte[] binaryData = new byte[256];
for (int i = 0; i < 256; i++) {
binaryData[i] = (byte) i;
}
// Write binary file
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
outputStream.write(binaryData);
outputStream.close();
// Read and verify
FilerProto.Entry entry = filerClient.lookupEntry(
SeaweedOutputStream.getParentDirectory(testPath),
SeaweedOutputStream.getFileName(testPath));
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
byte[] readData = new byte[256];
int bytesRead = inputStream.read(readData);
inputStream.close();
assertEquals("Should read all bytes", 256, bytesRead);
assertArrayEquals("Binary data should match", binaryData, readData);
}
@Test
public void testFlush() throws IOException {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String testPath = TEST_ROOT + "/flush.txt";
String testContent = "Content to flush";
// Write file with flush
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8));
outputStream.flush(); // Explicitly flush
outputStream.close();
// Verify file was written
assertTrue("File should exist after flush", filerClient.exists(testPath));
// Read and verify
FilerProto.Entry entry = filerClient.lookupEntry(
SeaweedOutputStream.getParentDirectory(testPath),
SeaweedOutputStream.getFileName(testPath));
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
byte[] buffer = new byte[testContent.length()];
int bytesRead = inputStream.read(buffer);
inputStream.close();
assertEquals("Content should match", testContent,
new String(buffer, 0, bytesRead, StandardCharsets.UTF_8));
}
/**
* Tests range reads similar to how Parquet reads column chunks.
* This simulates:
* 1. Seeking to specific offsets
* 2. Reading specific byte ranges
* 3. Verifying each read() call returns the correct number of bytes
*
* This test specifically addresses the bug where read() was returning 0
* for inline content or -1 prematurely for chunked reads.
*/
@Test
public void testRangeReads() throws IOException {
if (!TESTS_ENABLED) {
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set");
return;
}
String testPath = TEST_ROOT + "/rangereads.dat";
// Create a 1275-byte file (similar to the Parquet file size that was failing)
byte[] testData = new byte[1275];
Random random = new Random(42); // Fixed seed for reproducibility
random.nextBytes(testData);
// Write file
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath);
outputStream.write(testData);
outputStream.close();
// Read file entry
FilerProto.Entry entry = filerClient.lookupEntry(
SeaweedOutputStream.getParentDirectory(testPath),
SeaweedOutputStream.getFileName(testPath));
// Test 1: Read last 8 bytes (like reading Parquet footer length)
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry);
inputStream.seek(1267);
byte[] buffer = new byte[8];
int bytesRead = inputStream.read(buffer, 0, 8);
assertEquals("Should read 8 bytes at offset 1267", 8, bytesRead);
assertArrayEquals("Content at offset 1267 should match",
Arrays.copyOfRange(testData, 1267, 1275), buffer);
inputStream.close();
// Test 2: Read large chunk in middle (like reading column data)
inputStream = new SeaweedInputStream(filerClient, testPath, entry);
inputStream.seek(383);
buffer = new byte[884]; // Read bytes 383-1267
bytesRead = inputStream.read(buffer, 0, 884);
assertEquals("Should read 884 bytes at offset 383", 884, bytesRead);
assertArrayEquals("Content at offset 383 should match",
Arrays.copyOfRange(testData, 383, 1267), buffer);
inputStream.close();
// Test 3: Read from beginning (like reading Parquet magic bytes)
inputStream = new SeaweedInputStream(filerClient, testPath, entry);
buffer = new byte[4];
bytesRead = inputStream.read(buffer, 0, 4);
assertEquals("Should read 4 bytes at offset 0", 4, bytesRead);
assertArrayEquals("Content at offset 0 should match",
Arrays.copyOfRange(testData, 0, 4), buffer);
inputStream.close();
// Test 4: Multiple sequential reads without seeking (like
// H2SeekableInputStream.readFully)
// This is the critical test case that was failing!
inputStream = new SeaweedInputStream(filerClient, testPath, entry);
inputStream.seek(1197); // Position where EOF was being returned prematurely
byte[] fullBuffer = new byte[78]; // Try to read the "missing" 78 bytes
int totalRead = 0;
int offset = 0;
int remaining = 78;
// Simulate Parquet's H2SeekableInputStream.readFully() loop
while (remaining > 0) {
int read = inputStream.read(fullBuffer, offset, remaining);
if (read == -1) {
fail(String.format(
"Got EOF after reading %d bytes, but expected to read %d more bytes (total requested: 78)",
totalRead, remaining));
}
assertTrue("Each read() should return positive bytes", read > 0);
totalRead += read;
offset += read;
remaining -= read;
}
assertEquals("Should read all 78 bytes in readFully loop", 78, totalRead);
assertArrayEquals("Content at offset 1197 should match",
Arrays.copyOfRange(testData, 1197, 1275), fullBuffer);
inputStream.close();
// Test 5: Read entire file in one go
inputStream = new SeaweedInputStream(filerClient, testPath, entry);
byte[] allData = new byte[1275];
bytesRead = inputStream.read(allData, 0, 1275);
assertEquals("Should read entire 1275 bytes", 1275, bytesRead);
assertArrayEquals("Entire content should match", testData, allData);
inputStream.close();
}
}
+2 -2
View File
@@ -16,14 +16,14 @@
</dependency>
<dependency>
<groupId>com.seaweedfs</groupId>
<artifactId>seaweedfs-hadoop3-client</artifactId>
<artifactId>seaweedfs-hadoop2-client</artifactId>
<version>3.80</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.4.0</version>
<version>2.10.2</version>
<scope>compile</scope>
</dependency>
</dependencies>
+120
View File
@@ -0,0 +1,120 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>hdfs-over-ftp</groupId>
<artifactId>hdfs-over-ftp</artifactId>
<version>1.0</version>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.4.3</version>
</parent>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger2</artifactId>
<version>2.9.2</version>
</dependency>
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger-ui</artifactId>
<version>2.10.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.2.4</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.ftpserver</groupId>
<artifactId>ftpserver-core</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>com.seaweedfs</groupId>
<artifactId>seaweedfs-hadoop3-client</artifactId>
<version>1.6.2</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
<compilerArguments>
<verbose />
<bootclasspath>${java.home}/lib/rt.jar</bootclasspath>
</compilerArguments>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.6</version>
<configuration>
<archive>
<manifest>
<mainClass>org.apache.hadoop.seaweed.ftp.ApplicationServer</mainClass>
<addClasspath>true</addClasspath>
<classpathPrefix>lib/</classpathPrefix>
</manifest>
<manifestEntries>
<Class-Path>./</Class-Path>
</manifestEntries>
</archive>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<descriptors>
<descriptor>src/main/resources/assembly.xml</descriptor>
</descriptors>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
@@ -0,0 +1,14 @@
package org.apache.hadoop.seaweed.ftp;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class ApplicationServer {
public static void main(String[] args) {
SpringApplication.run(ApplicationServer.class, args);
}
}
@@ -0,0 +1,27 @@
package org.apache.hadoop.seaweed.ftp.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import springfox.documentation.builders.ApiInfoBuilder;
import springfox.documentation.builders.PathSelectors;
import springfox.documentation.builders.RequestHandlerSelectors;
import springfox.documentation.spi.DocumentationType;
import springfox.documentation.spring.web.plugins.Docket;
import springfox.documentation.swagger2.annotations.EnableSwagger2;
@Configuration
@EnableSwagger2
public class SwaggerConfig {
@Bean
public Docket createRestApi() {
return new Docket(DocumentationType.SWAGGER_2)
.pathMapping("/")
.select()
.apis(RequestHandlerSelectors.basePackage("org.apache.hadoop.seaweed.ftp"))
.paths(PathSelectors.any())
.build().apiInfo(new ApiInfoBuilder()
.title("FTP API Doc")
.version("1.0")
.build());
}
}
@@ -0,0 +1,71 @@
package org.apache.hadoop.seaweed.ftp.controller;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.apache.hadoop.seaweed.ftp.service.HFtpService;
import org.apache.hadoop.seaweed.ftp.controller.vo.Result;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.HashMap;
import java.util.Map;
@RestController
@RequestMapping("/manager")
@Api(tags = "FTP操作管理")
public class FtpManagerController {
private static Logger log = Logger.getLogger(FtpManagerController.class);
@Autowired
private HFtpService hdfsOverFtpServer;
@GetMapping("/status")
@ApiOperation("查看FTP服务状态")
public Result status() {
Map map = new HashMap<>();
try {
boolean status = hdfsOverFtpServer.statusServer();
map.put("is_running", status);
return new Result(true, map, "FTP 服务状态获取成功");
}catch (Exception e) {
log.error(e);
map.put("is_running", false);
return new Result(true, map, "FTP 服务状态获取成功");
}
}
@PutMapping("/start")
@ApiOperation("启动FTP服务")
public Result start() {
try {
boolean status = hdfsOverFtpServer.statusServer();
if(!status) {
hdfsOverFtpServer.startServer();
}
return new Result(true, "FTP 服务启动成功");
}catch (Exception e) {
log.error(e);
return new Result(false, "FTP 服务启动失败");
}
}
@PutMapping("/stop")
@ApiOperation("停止FTP服务")
public Result stop() {
try {
boolean status = hdfsOverFtpServer.statusServer();
if(status) {
hdfsOverFtpServer.stopServer();
}
return new Result(true, "FTP 服务停止成功");
}catch (Exception e) {
log.error(e);
return new Result(false, "FTP 服务停止失败");
}
}
}
@@ -0,0 +1,98 @@
package org.apache.hadoop.seaweed.ftp.controller;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.apache.ftpserver.ftplet.User;
import org.apache.ftpserver.usermanager.Md5PasswordEncryptor;
import org.apache.ftpserver.usermanager.UserFactory;
import org.apache.hadoop.seaweed.ftp.controller.vo.FtpUser;
import org.apache.hadoop.seaweed.ftp.controller.vo.Result;
import org.apache.hadoop.seaweed.ftp.users.HdfsUserManager;
import org.apache.log4j.Logger;
import org.springframework.web.bind.annotation.*;
import java.io.File;
@RestController
@RequestMapping("/user")
@Api(tags = "FTP用户管理")
public class UserController {
private static Logger log = Logger.getLogger(UserController.class);
/***
* {
* "name": "test",
* "password": "test",
* "homeDirectory": "/buckets/test/"
* }
* @param ftpUser
* @return
*/
@PostMapping("/add")
@ApiOperation("新增/编辑用户")
public Result add(@RequestBody FtpUser ftpUser) {
try {
HdfsUserManager userManagerFactory = new HdfsUserManager();
userManagerFactory.setFile(new File(System.getProperty("user.dir") + File.separator + "users.properties"));
userManagerFactory.setPasswordEncryptor(new Md5PasswordEncryptor());
UserFactory userFactory = new UserFactory();
userFactory.setHomeDirectory(ftpUser.getHomeDirectory());
userFactory.setName(ftpUser.getName());
userFactory.setPassword(ftpUser.getPassword());
userFactory.setEnabled(ftpUser.isEnabled());
userFactory.setMaxIdleTime(ftpUser.getMaxIdleTime());
User user = userFactory.createUser();
userManagerFactory.save(user, ftpUser.isRenamePush());
return new Result(true, "新建用户成功");
}catch (Exception e) {
log.error(e);
return new Result(false, "新建用户失败");
}
}
@DeleteMapping("/delete/{user}")
@ApiOperation("删除用户")
public Result delete(@PathVariable(value = "user") String user) {
try {
HdfsUserManager userManagerFactory = new HdfsUserManager();
userManagerFactory.setFile(new File(System.getProperty("user.dir") + File.separator + "users.properties"));
userManagerFactory.delete(user);
return new Result(true, "删除用户成功");
}catch (Exception e) {
log.error(e);
return new Result(false, "删除用户失败");
}
}
@GetMapping("/show/{userName}")
@ApiOperation("查看用户")
public Result show(@PathVariable(value = "userName") String userName) {
try {
HdfsUserManager userManagerFactory = new HdfsUserManager();
userManagerFactory.setFile(new File(System.getProperty("user.dir") + File.separator + "users.properties"));
User user = userManagerFactory.getUserByName(userName);
FtpUser ftpUser = new FtpUser(user.getHomeDirectory(), user.getPassword(), user.getEnabled(), user.getName(), user.getMaxIdleTime(), HdfsUserManager.getUserRenamePush(userName));
return new Result(true, ftpUser, "获取用户信息成功");
}catch (Exception e) {
log.error(e);
return new Result(false, "获取用户信息失败");
}
}
@GetMapping("/list")
@ApiOperation("列举用户")
public Result list() {
try {
HdfsUserManager userManagerFactory = new HdfsUserManager();
userManagerFactory.setFile(new File(System.getProperty("user.dir") + File.separator + "users.properties"));
String[] allUserNames = userManagerFactory.getAllUserNames();
return new Result(true, allUserNames, "列举用户成功");
}catch (Exception e) {
log.error(e);
return new Result(false, "列举用户失败");
}
}
}
@@ -0,0 +1,71 @@
package org.apache.hadoop.seaweed.ftp.controller.vo;
public class FtpUser {
private String homeDirectory;
private String password;
private boolean enabled;
private String name;
private int maxIdleTime;
private boolean renamePush;
public FtpUser() {
}
public FtpUser(String homeDirectory, String password, boolean enabled, String name, int maxIdleTime, boolean renamePush) {
this.homeDirectory = homeDirectory;
this.password = password;
this.enabled = enabled;
this.name = name;
this.maxIdleTime = maxIdleTime;
this.renamePush = renamePush;
}
public String getHomeDirectory() {
return homeDirectory;
}
public void setHomeDirectory(String homeDirectory) {
this.homeDirectory = homeDirectory;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public boolean isEnabled() {
return enabled;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public int getMaxIdleTime() {
return maxIdleTime;
}
public void setMaxIdleTime(int maxIdleTime) {
this.maxIdleTime = maxIdleTime;
}
public boolean isRenamePush() {
return renamePush;
}
public void setRenamePush(boolean renamePush) {
this.renamePush = renamePush;
}
}
@@ -0,0 +1,43 @@
package org.apache.hadoop.seaweed.ftp.controller.vo;
public class Result {
private boolean status;
private Object data;
private String message;
public Result(boolean status, String message) {
this.status = status;
this.message = message;
}
public Result(boolean status, Object data, String message) {
this.status = status;
this.message = message;
this.data = data;
}
public boolean isStatus() {
return status;
}
public void setStatus(boolean status) {
this.status = status;
}
public String getMessage() {
return message;
}
public void setMessage(String message) {
this.message = message;
}
public Object getData() {
return data;
}
public void setData(Object data) {
this.data = data;
}
}
@@ -0,0 +1,102 @@
package org.apache.hadoop.seaweed.ftp.service;
import org.apache.ftpserver.DataConnectionConfiguration;
import org.apache.ftpserver.DataConnectionConfigurationFactory;
import org.apache.ftpserver.FtpServer;
import org.apache.ftpserver.FtpServerFactory;
import org.apache.ftpserver.command.CommandFactoryFactory;
import org.apache.ftpserver.listener.ListenerFactory;
import org.apache.hadoop.seaweed.ftp.service.filesystem.HdfsFileSystemManager;
import org.apache.hadoop.seaweed.ftp.service.filesystem.HdfsOverFtpSystem;
import org.apache.hadoop.seaweed.ftp.users.HdfsUserManager;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.io.File;
/**
* reference: https://github.com/AShiou/hof
*/
@Component
public class HFtpService {
private static Logger log = Logger.getLogger(HFtpService.class);
@Value("${ftp.port}")
private int port = 0;
@Value("${ftp.passive-address}")
private String passiveAddress;
@Value("${ftp.passive-ports}")
private String passivePorts;
@Value("${hdfs.uri}")
private String hdfsUri;
@Value("${seaweedFs.enable}")
private boolean seaweedFsEnable;
@Value("${seaweedFs.access}")
private String seaweedFsAccess;
@Value("${seaweedFs.replication}")
private String seaweedFsReplication;
private FtpServer ftpServer = null;
public void startServer() throws Exception {
log.info("Starting HDFS-Over-Ftp server. port: " + port + " passive-address: " + passiveAddress + " passive-ports: " + passivePorts + " hdfs-uri: " + hdfsUri);
HdfsOverFtpSystem.setHdfsUri(hdfsUri);
HdfsOverFtpSystem.setSeaweedFsEnable(seaweedFsEnable);
HdfsOverFtpSystem.setSeaweedFsAccess(seaweedFsAccess);
HdfsOverFtpSystem.setSeaweedFsReplication(seaweedFsReplication);
FtpServerFactory server = new FtpServerFactory();
server.setFileSystem(new HdfsFileSystemManager());
ListenerFactory factory = new ListenerFactory();
factory.setPort(port);
DataConnectionConfigurationFactory dccFactory = new DataConnectionConfigurationFactory();
dccFactory.setPassiveAddress("0.0.0.0");
dccFactory.setPassivePorts(passivePorts);
dccFactory.setPassiveExternalAddress(passiveAddress);
DataConnectionConfiguration dcc = dccFactory.createDataConnectionConfiguration();
factory.setDataConnectionConfiguration(dcc);
server.addListener("default", factory.createListener());
HdfsUserManager userManager = new HdfsUserManager();
final File file = loadResource("/users.properties");
userManager.setFile(file);
server.setUserManager(userManager);
CommandFactoryFactory cmFact = new CommandFactoryFactory();
cmFact.setUseDefaultCommands(true);
server.setCommandFactory(cmFact.createCommandFactory());
// start the server
ftpServer = server.createServer();
ftpServer.start();
}
public void stopServer() {
log.info("Stopping Hdfs-Over-Ftp server. port: " + port + " passive-address: " + passiveAddress + " passive-ports: " + passivePorts + " hdfs-uri: " + hdfsUri);
ftpServer.stop();
}
public boolean statusServer() {
try {
return !ftpServer.isStopped();
}catch (Exception e) {
return false;
}
}
private static File loadResource(String resourceName) {
return new File(System.getProperty("user.dir") + resourceName);
}
}
@@ -0,0 +1,333 @@
package org.apache.hadoop.seaweed.ftp.service.filesystem;
import org.apache.ftpserver.ftplet.FtpFile;
import org.apache.ftpserver.ftplet.User;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.seaweed.ftp.users.HdfsUser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
/**
* This class implements all actions to HDFS
*/
public class HdfsFileObject implements FtpFile {
private final Logger log = LoggerFactory.getLogger(HdfsFileObject.class);
private Path homePath;
private Path path;
private Path fullPath;
private HdfsUser user;
/**
* Constructs HdfsFileObject from path
*
* @param path path to represent object
* @param user accessor of the object
*/
public HdfsFileObject(String homePath, String path, User user) {
this.homePath = new Path(homePath);
this.path = new Path(path);
this.fullPath = new Path(homePath + path);
this.user = (HdfsUser) user;
}
public String getAbsolutePath() {
// strip the last '/' if necessary
String fullName = path.toString();
int filelen = fullName.length();
if ((filelen != 1) && (fullName.charAt(filelen - 1) == '/')) {
fullName = fullName.substring(0, filelen - 1);
}
return fullName;
}
public String getName() {
return path.getName();
}
/**
* HDFS has no hidden objects
*
* @return always false
*/
public boolean isHidden() {
return false;
}
/**
* Checks if the object is a directory
*
* @return true if the object is a directory
*/
public boolean isDirectory() {
try {
log.debug("is directory? : " + fullPath);
FileSystem dfs = HdfsOverFtpSystem.getDfs();
FileStatus fs = dfs.getFileStatus(fullPath);
return fs.isDir();
} catch (IOException e) {
log.debug(fullPath + " is not dir", e);
return false;
}
}
/**
* Checks if the object is a file
*
* @return true if the object is a file
*/
public boolean isFile() {
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
return dfs.isFile(fullPath);
} catch (IOException e) {
log.debug(fullPath + " is not file", e);
return false;
}
}
/**
* Checks if the object does exist
*
* @return true if the object does exist
*/
public boolean doesExist() {
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
dfs.getFileStatus(fullPath);
return true;
} catch (IOException e) {
// log.debug(path + " does not exist", e);
return false;
}
}
public boolean isReadable() {
return true;
}
public boolean isWritable() {
return true;
}
public boolean isRemovable() {
return true;
}
/**
* Get owner of the object
*
* @return owner of the object
*/
public String getOwnerName() {
return "root";
/*
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
FileStatus fs = dfs.getFileStatus(fullPath);
String owner = fs.getOwner();
if(owner.length() == 0) {
return "root";
}
return owner;
} catch (IOException e) {
e.printStackTrace();
return null;
}
*/
}
/**
* Get group of the object
*
* @return group of the object
*/
public String getGroupName() {
return "root";
/*
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
FileStatus fs = dfs.getFileStatus(fullPath);
String group = fs.getGroup();
if(group.length() == 0) {
return "root";
}
return group;
} catch (IOException e) {
e.printStackTrace();
return null;
}
*/
}
/**
* Get link count
*
* @return 3 is for a directory and 1 is for a file
*/
public int getLinkCount() {
return isDirectory() ? 3 : 1;
}
/**
* Get last modification date
*
* @return last modification date as a long
*/
public long getLastModified() {
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
FileStatus fs = dfs.getFileStatus(fullPath);
return fs.getModificationTime();
} catch (IOException e) {
e.printStackTrace();
return 0;
}
}
public boolean setLastModified(long l) {
return false;
}
/**
* Get a size of the object
*
* @return size of the object in bytes
*/
public long getSize() {
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
FileStatus fs = dfs.getFileStatus(fullPath);
log.debug("getSize(): " + fullPath + " : " + fs.getLen());
return fs.getLen();
} catch (IOException e) {
e.printStackTrace();
return 0;
}
}
public Object getPhysicalFile() {
return null;
}
/**
* Create a new dir from the object
*
* @return true if dir is created
*/
public boolean mkdir() {
try {
FileSystem fs = HdfsOverFtpSystem.getDfs();
fs.mkdirs(fullPath);
// fs.setOwner(path, user.getName(), user.getMainGroup());
return true;
} catch (IOException e) {
e.printStackTrace();
return false;
}
}
/**
* Delete object from the HDFS filesystem
*
* @return true if the object is deleted
*/
public boolean delete() {
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
dfs.delete(fullPath, true);
return true;
} catch (IOException e) {
e.printStackTrace();
return false;
}
}
public boolean move(FtpFile ftpFile) {
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
dfs.rename(fullPath, new Path(fullPath.getParent() + File.separator + ftpFile.getName()));
return true;
} catch (IOException e) {
e.printStackTrace();
return false;
}
}
/**
* List files of the directory
*
* @return List of files in the directory
*/
public List<FtpFile> listFiles() {
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
FileStatus fileStats[] = dfs.listStatus(fullPath);
// get the virtual name of the base directory
String virtualFileStr = getAbsolutePath();
if (virtualFileStr.charAt(virtualFileStr.length() - 1) != '/') {
virtualFileStr += '/';
}
FtpFile[] virtualFiles = new FtpFile[fileStats.length];
for (int i = 0; i < fileStats.length; i++) {
File fileObj = new File(fileStats[i].getPath().toString());
String fileName = virtualFileStr + fileObj.getName();
virtualFiles[i] = new HdfsFileObject(homePath.toString(), fileName, user);
}
return Collections.unmodifiableList(Arrays.asList(virtualFiles));
} catch (IOException e) {
log.debug("", e);
return null;
}
}
/**
* Creates output stream to write to the object
*
* @param l is not used here
* @return OutputStream
* @throws IOException
*/
public OutputStream createOutputStream(long l) {
try {
FileSystem fs = HdfsOverFtpSystem.getDfs();
FSDataOutputStream out = fs.create(fullPath);
// fs.setOwner(fullPath, user.getName(), user.getMainGroup());
return out;
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
/**
* Creates input stream to read from the object
*
* @param l is not used here
* @return OutputStream
* @throws IOException
*/
public InputStream createInputStream(long l) {
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
FSDataInputStream in = dfs.open(fullPath);
return in;
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
}
@@ -0,0 +1,14 @@
package org.apache.hadoop.seaweed.ftp.service.filesystem;
import org.apache.ftpserver.ftplet.FileSystemFactory;
import org.apache.ftpserver.ftplet.FileSystemView;
import org.apache.ftpserver.ftplet.User;
/**
* Impelented FileSystemManager to use HdfsFileSystemView
*/
public class HdfsFileSystemManager implements FileSystemFactory {
public FileSystemView createFileSystemView(User user) {
return new HdfsFileSystemView(user);
}
}
@@ -0,0 +1,104 @@
package org.apache.hadoop.seaweed.ftp.service.filesystem;
import org.apache.ftpserver.ftplet.FileSystemView;
import org.apache.ftpserver.ftplet.FtpFile;
import org.apache.ftpserver.ftplet.User;
import org.apache.hadoop.fs.Path;
import java.io.File;
/**
* Implemented FileSystemView to use HdfsFileObject
*/
public class HdfsFileSystemView implements FileSystemView {
private String homePath;
private String currPath = File.separator;
private User user;
/**
* Constructor - set the user object.
*/
protected HdfsFileSystemView(User user) {
if (user == null) {
throw new IllegalArgumentException("user can not be null");
}
if (user.getHomeDirectory() == null) {
throw new IllegalArgumentException(
"User home directory can not be null");
}
this.homePath = user.getHomeDirectory();
this.user = user;
}
public FtpFile getHomeDirectory() {
return new HdfsFileObject(homePath, File.separator, user);
}
public FtpFile getWorkingDirectory() {
FtpFile fileObj;
if (currPath.equals(File.separator)) {
fileObj = new HdfsFileObject(homePath, File.separator, user);
} else {
fileObj = new HdfsFileObject(homePath, currPath, user);
}
return fileObj;
}
public boolean changeWorkingDirectory(String dir) {
Path path;
if (dir.startsWith(File.separator) || new Path(currPath).equals(new Path(dir))) {
path = new Path(dir);
} else if (currPath.length() > 1) {
path = new Path(currPath + File.separator + dir);
} else {
if(dir.startsWith("/")) {
path = new Path(dir);
}else {
path = new Path(File.separator + dir);
}
}
// 防止退回根目录
if (path.getName().equals("..")) {
path = new Path(File.separator);
}
HdfsFileObject file = new HdfsFileObject(homePath, path.toString(), user);
if (file.isDirectory()) {
currPath = path.toString();
return true;
} else {
return false;
}
}
public FtpFile getFile(String file) {
String path;
if (file.startsWith(File.separator)) {
path = file;
} else if (currPath.length() > 1) {
path = currPath + File.separator + file;
} else {
path = File.separator + file;
}
return new HdfsFileObject(homePath, path, user);
}
/**
* Is the file content random accessible?
*/
public boolean isRandomAccessible() {
return true;
}
/**
* Dispose file system view - does nothing.
*/
public void dispose() {
}
}
@@ -0,0 +1,72 @@
package org.apache.hadoop.seaweed.ftp.service.filesystem;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
/**
* Class to store DFS connection
*/
public class HdfsOverFtpSystem {
private static FileSystem fs = null;
private static String hdfsUri;
private static boolean seaweedFsEnable;
private static String seaweedFsAccess;
private static String seaweedFsReplication;
private final static Logger log = LoggerFactory.getLogger(HdfsOverFtpSystem.class);
private static void hdfsInit() throws IOException {
Configuration configuration = new Configuration();
configuration.set("fs.defaultFS", hdfsUri);
if(seaweedFsEnable) {
configuration.set("fs.seaweedfs.impl", "seaweed.hdfs.SeaweedFileSystem");
configuration.set("fs.seaweed.volume.server.access", seaweedFsAccess);
configuration.set("fs.seaweed.replication", seaweedFsReplication);
}
fs = FileSystem.get(configuration);
log.info("HDFS load success");
}
/**
* Get dfs
*
* @return dfs
* @throws IOException
*/
public static FileSystem getDfs() throws IOException {
if (fs == null) {
hdfsInit();
}
return fs;
}
public static void setHdfsUri(String hdfsUri) {
HdfsOverFtpSystem.hdfsUri = hdfsUri;
}
public static String getHdfsUri() {
return hdfsUri;
}
public static void setSeaweedFsEnable(boolean seaweedFsEnable) {
HdfsOverFtpSystem.seaweedFsEnable = seaweedFsEnable;
}
public static void setSeaweedFsAccess(String seaweedFsAccess) {
HdfsOverFtpSystem.seaweedFsAccess = seaweedFsAccess;
}
public static void setSeaweedFsReplication(String seaweedFsReplication) {
HdfsOverFtpSystem.seaweedFsReplication = seaweedFsReplication;
}
}
@@ -0,0 +1,239 @@
package org.apache.hadoop.seaweed.ftp.users;
import org.apache.ftpserver.ftplet.Authority;
import org.apache.ftpserver.ftplet.AuthorizationRequest;
import org.apache.ftpserver.ftplet.User;
import org.apache.log4j.Logger;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class HdfsUser implements User, Serializable {
private static final long serialVersionUID = -47371353779731294L;
private String name = null;
private String password = null;
private int maxIdleTimeSec = 0; // no limit
private String homeDir = null;
private boolean isEnabled = true;
private List<? extends Authority> authorities = new ArrayList<Authority>();
private ArrayList<String> groups = new ArrayList<String>();
private Logger log = Logger.getLogger(HdfsUser.class);
/**
* Default constructor.
*/
public HdfsUser() {
}
/**
* Copy constructor.
*/
public HdfsUser(User user) {
name = user.getName();
password = user.getPassword();
authorities = user.getAuthorities();
maxIdleTimeSec = user.getMaxIdleTime();
homeDir = user.getHomeDirectory();
isEnabled = user.getEnabled();
}
public ArrayList<String> getGroups() {
return groups;
}
/**
* Get the main group of the user
*
* @return main group of the user
*/
public String getMainGroup() {
if (groups.size() > 0) {
return groups.get(0);
} else {
log.error("User " + name + " is not a memer of any group");
return "error";
}
}
/**
* Checks if user is a member of the group
*
* @param group to check
* @return true if the user id a member of the group
*/
public boolean isGroupMember(String group) {
for (String userGroup : groups) {
if (userGroup.equals(group)) {
return true;
}
}
return false;
}
/**
* Set users' groups
*
* @param groups to set
*/
public void setGroups(ArrayList<String> groups) {
if (groups.size() < 1) {
log.error("User " + name + " is not a memer of any group");
}
this.groups = groups;
}
/**
* Get the user name.
*/
public String getName() {
return name;
}
/**
* Set user name.
*/
public void setName(String name) {
this.name = name;
}
/**
* Get the user password.
*/
public String getPassword() {
return password;
}
/**
* Set user password.
*/
public void setPassword(String pass) {
password = pass;
}
public List<Authority> getAuthorities() {
if (authorities != null) {
return Collections.unmodifiableList(authorities);
} else {
return null;
}
}
public void setAuthorities(List<Authority> authorities) {
if (authorities != null) {
this.authorities = Collections.unmodifiableList(authorities);
} else {
this.authorities = null;
}
}
/**
* Get the maximum idle time in second.
*/
public int getMaxIdleTime() {
return maxIdleTimeSec;
}
/**
* Set the maximum idle time in second.
*/
public void setMaxIdleTime(int idleSec) {
maxIdleTimeSec = idleSec;
if (maxIdleTimeSec < 0) {
maxIdleTimeSec = 0;
}
}
/**
* Get the user enable status.
*/
public boolean getEnabled() {
return isEnabled;
}
/**
* Set the user enable status.
*/
public void setEnabled(boolean enb) {
isEnabled = enb;
}
/**
* Get the user home directory.
*/
public String getHomeDirectory() {
return homeDir;
}
/**
* Set the user home directory.
*/
public void setHomeDirectory(String home) {
homeDir = home;
}
/**
* String representation.
*/
public String toString() {
return name;
}
/**
* {@inheritDoc}
*/
public AuthorizationRequest authorize(AuthorizationRequest request) {
List<Authority> authorities = getAuthorities();
// check for no authorities at all
if (authorities == null) {
return null;
}
boolean someoneCouldAuthorize = false;
for (Authority authority : authorities) {
if (authority.canAuthorize(request)) {
someoneCouldAuthorize = true;
request = authority.authorize(request);
// authorization failed, return null
if (request == null) {
return null;
}
}
}
if (someoneCouldAuthorize) {
return request;
} else {
return null;
}
}
/**
* {@inheritDoc}
*/
public List<Authority> getAuthorities(Class<? extends Authority> clazz) {
List<Authority> selected = new ArrayList<Authority>();
for (Authority authority : authorities) {
if (authority.getClass().equals(clazz)) {
selected.add(authority);
}
}
return selected;
}
}
@@ -0,0 +1,453 @@
package org.apache.hadoop.seaweed.ftp.users;
import org.apache.ftpserver.FtpServerConfigurationException;
import org.apache.ftpserver.ftplet.*;
import org.apache.ftpserver.usermanager.*;
import org.apache.ftpserver.usermanager.impl.*;
import org.apache.ftpserver.util.BaseProperties;
import org.apache.ftpserver.util.IoUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.*;
public class HdfsUserManager extends AbstractUserManager {
private final Logger LOG = LoggerFactory
.getLogger(HdfsUserManager.class);
private final static String DEPRECATED_PREFIX = "FtpServer.user.";
private final static String PREFIX = "ftpserver.user.";
private static BaseProperties userDataProp;
private File userDataFile = new File("users.conf");
private boolean isConfigured = false;
private PasswordEncryptor passwordEncryptor = new Md5PasswordEncryptor();
/**
* Retrieve the file used to load and store users
*
* @return The file
*/
public File getFile() {
return userDataFile;
}
/**
* Set the file used to store and read users. Must be set before
* {@link #configure()} is called.
*
* @param propFile A file containing users
*/
public void setFile(File propFile) {
if (isConfigured) {
throw new IllegalStateException("Must be called before configure()");
}
this.userDataFile = propFile;
}
/**
* Retrieve the password encryptor used for this user manager
*
* @return The password encryptor. Default to {@link Md5PasswordEncryptor}
* if no other has been provided
*/
public PasswordEncryptor getPasswordEncryptor() {
return passwordEncryptor;
}
/**
* Set the password encryptor to use for this user manager
*
* @param passwordEncryptor The password encryptor
*/
public void setPasswordEncryptor(PasswordEncryptor passwordEncryptor) {
this.passwordEncryptor = passwordEncryptor;
}
/**
* Lazy init the user manager
*/
private void lazyInit() {
if (!isConfigured) {
configure();
}
}
/**
* Configure user manager.
*/
public void configure() {
isConfigured = true;
try {
userDataProp = new BaseProperties();
if (userDataFile != null && userDataFile.exists()) {
FileInputStream fis = null;
try {
fis = new FileInputStream(userDataFile);
userDataProp.load(fis);
} finally {
IoUtils.close(fis);
}
}
} catch (IOException e) {
throw new FtpServerConfigurationException(
"Error loading user data file : "
+ userDataFile.getAbsolutePath(), e);
}
convertDeprecatedPropertyNames();
}
private void convertDeprecatedPropertyNames() {
Enumeration<?> keys = userDataProp.propertyNames();
boolean doSave = false;
while (keys.hasMoreElements()) {
String key = (String) keys.nextElement();
if (key.startsWith(DEPRECATED_PREFIX)) {
String newKey = PREFIX
+ key.substring(DEPRECATED_PREFIX.length());
userDataProp.setProperty(newKey, userDataProp.getProperty(key));
userDataProp.remove(key);
doSave = true;
}
}
if (doSave) {
try {
saveUserData();
} catch (FtpException e) {
throw new FtpServerConfigurationException(
"Failed to save updated user data", e);
}
}
}
public synchronized void save(User usr, boolean renamePush) throws FtpException {
lazyInit();
userDataProp.setProperty(PREFIX + usr.getName() + ".rename.push", renamePush);
save(usr);
}
/**
* Save user data. Store the properties.
*/
public synchronized void save(User usr) throws FtpException {
lazyInit();
// null value check
if (usr.getName() == null) {
throw new NullPointerException("User name is null.");
}
String thisPrefix = PREFIX + usr.getName() + '.';
// set other properties
userDataProp.setProperty(thisPrefix + ATTR_PASSWORD, getPassword(usr));
String home = usr.getHomeDirectory();
if (home == null) {
home = "/";
}
userDataProp.setProperty(thisPrefix + ATTR_HOME, home);
userDataProp.setProperty(thisPrefix + ATTR_ENABLE, usr.getEnabled());
userDataProp.setProperty(thisPrefix + ATTR_WRITE_PERM, usr
.authorize(new WriteRequest()) != null);
userDataProp.setProperty(thisPrefix + ATTR_MAX_IDLE_TIME, usr
.getMaxIdleTime());
TransferRateRequest transferRateRequest = new TransferRateRequest();
transferRateRequest = (TransferRateRequest) usr
.authorize(transferRateRequest);
if (transferRateRequest != null) {
userDataProp.setProperty(thisPrefix + ATTR_MAX_UPLOAD_RATE,
transferRateRequest.getMaxUploadRate());
userDataProp.setProperty(thisPrefix + ATTR_MAX_DOWNLOAD_RATE,
transferRateRequest.getMaxDownloadRate());
} else {
userDataProp.remove(thisPrefix + ATTR_MAX_UPLOAD_RATE);
userDataProp.remove(thisPrefix + ATTR_MAX_DOWNLOAD_RATE);
}
// request that always will succeed
ConcurrentLoginRequest concurrentLoginRequest = new ConcurrentLoginRequest(
0, 0);
concurrentLoginRequest = (ConcurrentLoginRequest) usr
.authorize(concurrentLoginRequest);
if (concurrentLoginRequest != null) {
userDataProp.setProperty(thisPrefix + ATTR_MAX_LOGIN_NUMBER,
concurrentLoginRequest.getMaxConcurrentLogins());
userDataProp.setProperty(thisPrefix + ATTR_MAX_LOGIN_PER_IP,
concurrentLoginRequest.getMaxConcurrentLoginsPerIP());
} else {
userDataProp.remove(thisPrefix + ATTR_MAX_LOGIN_NUMBER);
userDataProp.remove(thisPrefix + ATTR_MAX_LOGIN_PER_IP);
}
saveUserData();
}
/**
* @throws FtpException
*/
private void saveUserData() throws FtpException {
File dir = userDataFile.getAbsoluteFile().getParentFile();
if (dir != null && !dir.exists() && !dir.mkdirs()) {
String dirName = dir.getAbsolutePath();
throw new FtpServerConfigurationException(
"Cannot create directory for user data file : " + dirName);
}
// save user data
FileOutputStream fos = null;
try {
fos = new FileOutputStream(userDataFile);
userDataProp.store(fos, "Generated file - don't edit (please)");
} catch (IOException ex) {
LOG.error("Failed saving user data", ex);
throw new FtpException("Failed saving user data", ex);
} finally {
IoUtils.close(fos);
}
}
public synchronized void list() throws FtpException {
lazyInit();
Map dataMap = new HashMap();
Enumeration<String> propNames = (Enumeration<String>) userDataProp.propertyNames();
ArrayList<String> a = Collections.list(propNames);
a.remove("i18nMap");//去除i18nMap
for(String attrName : a){
// dataMap.put(attrName, propNames.);
}
}
/**
* Delete an user. Removes all this user entries from the properties. After
* removing the corresponding from the properties, save the data.
*/
public synchronized void delete(String usrName) throws FtpException {
lazyInit();
// remove entries from properties
String thisPrefix = PREFIX + usrName + '.';
Enumeration<?> propNames = userDataProp.propertyNames();
ArrayList<String> remKeys = new ArrayList<String>();
while (propNames.hasMoreElements()) {
String thisKey = propNames.nextElement().toString();
if (thisKey.startsWith(thisPrefix)) {
remKeys.add(thisKey);
}
}
Iterator<String> remKeysIt = remKeys.iterator();
while (remKeysIt.hasNext()) {
userDataProp.remove(remKeysIt.next());
}
saveUserData();
}
/**
* Get user password. Returns the encrypted value.
* <p/>
* <pre>
* If the password value is not null
* password = new password
* else
* if user does exist
* password = old password
* else
* password = &quot;&quot;
* </pre>
*/
private String getPassword(User usr) {
String name = usr.getName();
String password = usr.getPassword();
if (password != null) {
password = passwordEncryptor.encrypt(password);
} else {
String blankPassword = passwordEncryptor.encrypt("");
if (doesExist(name)) {
String key = PREFIX + name + '.' + ATTR_PASSWORD;
password = userDataProp.getProperty(key, blankPassword);
} else {
password = blankPassword;
}
}
return password;
}
/**
* Get all user names.
*/
public synchronized String[] getAllUserNames() {
lazyInit();
// get all user names
String suffix = '.' + ATTR_HOME;
ArrayList<String> ulst = new ArrayList<String>();
Enumeration<?> allKeys = userDataProp.propertyNames();
int prefixlen = PREFIX.length();
int suffixlen = suffix.length();
while (allKeys.hasMoreElements()) {
String key = (String) allKeys.nextElement();
if (key.endsWith(suffix)) {
String name = key.substring(prefixlen);
int endIndex = name.length() - suffixlen;
name = name.substring(0, endIndex);
ulst.add(name);
}
}
Collections.sort(ulst);
return ulst.toArray(new String[0]);
}
private ArrayList<String> parseGroups(String groupsLine) {
String groupsArray[] = groupsLine.split(",");
return new ArrayList(Arrays.asList(groupsArray));
}
public static synchronized boolean getUserRenamePush(String userName) {
return userDataProp.getBoolean(PREFIX + userName + ".rename.push", false);
}
/**
* Load user data.
*/
public synchronized User getUserByName(String userName) {
lazyInit();
if (!doesExist(userName)) {
return null;
}
String baseKey = PREFIX + userName + '.';
HdfsUser user = new HdfsUser();
user.setName(userName);
user.setEnabled(userDataProp.getBoolean(baseKey + ATTR_ENABLE, true));
user.setHomeDirectory(userDataProp
.getProperty(baseKey + ATTR_HOME, "/"));
// user.setGroups(parseGroups(userDataProp
// .getProperty(baseKey + "groups")));
List<Authority> authorities = new ArrayList<Authority>();
if (userDataProp.getBoolean(baseKey + ATTR_WRITE_PERM, false)) {
authorities.add(new WritePermission());
}
int maxLogin = userDataProp.getInteger(baseKey + ATTR_MAX_LOGIN_NUMBER,
0);
int maxLoginPerIP = userDataProp.getInteger(baseKey
+ ATTR_MAX_LOGIN_PER_IP, 0);
authorities.add(new ConcurrentLoginPermission(maxLogin, maxLoginPerIP));
int uploadRate = userDataProp.getInteger(
baseKey + ATTR_MAX_UPLOAD_RATE, 0);
int downloadRate = userDataProp.getInteger(baseKey
+ ATTR_MAX_DOWNLOAD_RATE, 0);
authorities.add(new TransferRatePermission(downloadRate, uploadRate));
user.setAuthorities(authorities);
user.setMaxIdleTime(userDataProp.getInteger(baseKey
+ ATTR_MAX_IDLE_TIME, 0));
return user;
}
/**
* User existance check
*/
public synchronized boolean doesExist(String name) {
lazyInit();
String key = PREFIX + name + '.' + ATTR_HOME;
return userDataProp.containsKey(key);
}
/**
* User authenticate method
*/
public synchronized User authenticate(Authentication authentication)
throws AuthenticationFailedException {
lazyInit();
if (authentication instanceof UsernamePasswordAuthentication) {
UsernamePasswordAuthentication upauth = (UsernamePasswordAuthentication) authentication;
String user = upauth.getUsername();
String password = upauth.getPassword();
if (user == null) {
throw new AuthenticationFailedException("Authentication failed");
}
if (password == null) {
password = "";
}
String storedPassword = userDataProp.getProperty(PREFIX + user + '.'
+ ATTR_PASSWORD);
if (storedPassword == null) {
// user does not exist
throw new AuthenticationFailedException("Authentication failed");
}
if (passwordEncryptor.matches(password, storedPassword)) {
return getUserByName(user);
} else {
throw new AuthenticationFailedException("Authentication failed");
}
} else if (authentication instanceof AnonymousAuthentication) {
if (doesExist("anonymous")) {
return getUserByName("anonymous");
} else {
throw new AuthenticationFailedException("Authentication failed");
}
} else {
throw new IllegalArgumentException(
"Authentication not supported by this user manager");
}
}
/**
* Close the user manager - remove existing entries.
*/
public synchronized void dispose() {
if (userDataProp != null) {
userDataProp.clear();
userDataProp = null;
}
}
}
@@ -0,0 +1,15 @@
server:
port: 8080
ftp:
port: 2222
passive-address: localhost
passive-ports: 30000-30999
hdfs:
uri: seaweedfs://localhost:8888
seaweedFs:
enable: true
access: direct # direct/filerProxy/publicUrl
replication: "000"
@@ -0,0 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
<id>package</id>
<formats>
<!-- 指定打包格式,支持的打包格式有zip、tar、tar.gz (or tgz)、tar.bz2 (or tbz2)、jar、dir、war,可以同时指定多个打包格式 -->
<format>tar.gz</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
<directory>src/main/resources</directory>
<outputDirectory>/</outputDirectory>
<includes>
<include>application.yml</include>
<include>logback-spring.xml</include>
<include>users.properties</include>
<include>kafka-producer.properties</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.build.directory}</directory>
<outputDirectory>/</outputDirectory>
<includes>
<include>*.jar</include>
</includes>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<useProjectArtifact>false</useProjectArtifact>
<outputDirectory>lib</outputDirectory>
<scope>runtime</scope>
<unpack>false</unpack>
</dependencySet>
</dependencySets>
</assembly>
@@ -0,0 +1,40 @@
<?xml version="1.0" encoding="UTF-8" ?>
<configuration>
<!--定义日志文件的存储地址 勿在 LogBack 的配置中使用相对路径-->
<property name="LOG_HOME" value="${user.dir}/logs/" />
<!-- 控制台输出 -->
<appender name="Stdout" class="ch.qos.logback.core.ConsoleAppender">
<!-- 日志输出编码 -->
<layout class="ch.qos.logback.classic.PatternLayout">
<!--格式化输出:%d表示日期,%thread表示线程名,%-5level:级别从左显示5个字符宽度%msg:日志消息,%n是换行符-->
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n
</pattern>
</layout>
</appender>
<!-- 按照每天生成日志文件 -->
<appender name="RollingFile"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<File>${LOG_HOME}/fileLog.log</File>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${LOG_HOME}/fileLog.log.%d.%i</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>100 MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
</rollingPolicy>
<encoder>
<pattern>
%d %p (%file:%line\)- %m%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<!-- 日志输出级别 -->
<root level="info">
<appender-ref ref="Stdout" />
<appender-ref ref="RollingFile" />
</root>
</configuration>
+12
View File
@@ -0,0 +1,12 @@
#Generated file - don't edit (please)
#Thu Mar 11 19:11:12 CST 2021
ftpserver.user.test.idletime=0
ftpserver.user.test.maxloginperip=0
ftpserver.user.test.userpassword=44664D4D827C740293D2AA244FB60445
ftpserver.user.test.enableflag=true
ftpserver.user.test.maxloginnumber=0
ftpserver.user.test.rename.push=true
ftpserver.user.test.homedirectory=/buckets/test/
ftpserver.user.test.downloadrate=0
ftpserver.user.test.writepermission=true
ftpserver.user.test.uploadrate=0
+333
View File
@@ -0,0 +1,333 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.seaweedfs</groupId>
<artifactId>seaweedfs-hadoop2-client</artifactId>
<name>SeaweedFS HDFS2 Client</name>
<version>${seaweedfs.client.version}</version>
<description>A java client for SeaweedFS.</description>
<url>https://github.com/seaweedfs/seaweedfs</url>
<developers>
<developer>
<name>Chris Lu</name>
<email>chris.lu@gmail.com</email>
<organization>SeaweedFS</organization>
<organizationUrl>https://seaweedfs.com</organizationUrl>
</developer>
</developers>
<licenses>
<license>
<name>The Apache License, Version 2.0</name>
<url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
</license>
</licenses>
<scm>
<connection>scm:git:git://github.com/seaweedfs/seaweedfs.git</connection>
<developerConnection>scm:git:ssh://github.com:seaweedfs/seaweedfs.git</developerConnection>
<url>https://github.com/seaweedfs/seaweedfs/tree/master</url>
</scm>
<build>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>8</source>
<target>8</target>
<release>8</release>
</configuration>
</plugin>
<plugin>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>org/slf4j/**</exclude>
<exclude>META-INF/maven/org.slf4j/**</exclude>
</excludes>
</filter>
</filters>
<transformers>
<transformer />
</transformers>
<relocations>
<relocation>
<pattern>com.google</pattern>
<shadedPattern>shaded.com.google</shadedPattern>
</relocation>
<relocation>
<pattern>io.grpc.internal</pattern>
<shadedPattern>shaded.io.grpc.internal</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.commons</pattern>
<shadedPattern>shaded.org.apache.commons</shadedPattern>
<excludes>
<exclude>org.apache.hadoop</exclude>
<exclude>org.apache.log4j</exclude>
</excludes>
</relocation>
<relocation>
<pattern>org.apache.http</pattern>
<shadedPattern>shaded.org.apache.http</shadedPattern>
</relocation>
</relocations>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-gpg-plugin</artifactId>
<version>1.5</version>
<executions>
<execution>
<id>sign-artifacts</id>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.sonatype.central</groupId>
<artifactId>central-publishing-maven-plugin</artifactId>
<version>0.5.0</version>
<extensions>true</extensions>
<configuration>
<publishingServerId>central</publishingServerId>
<autoPublish>true</autoPublish>
</configuration>
</plugin>
<plugin>
<artifactId>maven-source-plugin</artifactId>
<version>2.2.1</version>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.0.1</version>
<executions>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.2.4</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<artifactId>hadoop-hdfs-client</artifactId>
<groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-yarn-api</artifactId>
<groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-yarn-client</artifactId>
<groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
<groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-annotations</artifactId>
<groupId>org.apache.hadoop</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.2.4</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<artifactId>commons-cli</artifactId>
<groupId>commons-cli</groupId>
</exclusion>
<exclusion>
<artifactId>commons-math3</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>commons-io</artifactId>
<groupId>commons-io</groupId>
</exclusion>
<exclusion>
<artifactId>commons-net</artifactId>
<groupId>commons-net</groupId>
</exclusion>
<exclusion>
<artifactId>commons-collections</artifactId>
<groupId>commons-collections</groupId>
</exclusion>
<exclusion>
<artifactId>javax.servlet-api</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
<exclusion>
<artifactId>javax.activation-api</artifactId>
<groupId>javax.activation</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-server</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-util</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-servlet</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-webapp</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>jsp-api</artifactId>
<groupId>javax.servlet.jsp</groupId>
</exclusion>
<exclusion>
<artifactId>jersey-core</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>jersey-servlet</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>jersey-json</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>jersey-server</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>reload4j</artifactId>
<groupId>ch.qos.reload4j</groupId>
</exclusion>
<exclusion>
<artifactId>commons-beanutils</artifactId>
<groupId>commons-beanutils</groupId>
</exclusion>
<exclusion>
<artifactId>commons-configuration2</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>commons-lang3</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>commons-text</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-reload4j</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>avro</artifactId>
<groupId>org.apache.avro</groupId>
</exclusion>
<exclusion>
<artifactId>re2j</artifactId>
<groupId>com.google.re2j</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-auth</artifactId>
<groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
<artifactId>jsch</artifactId>
<groupId>com.jcraft</groupId>
</exclusion>
<exclusion>
<artifactId>curator-client</artifactId>
<groupId>org.apache.curator</groupId>
</exclusion>
<exclusion>
<artifactId>curator-recipes</artifactId>
<groupId>org.apache.curator</groupId>
</exclusion>
<exclusion>
<artifactId>htrace-core4</artifactId>
<groupId>org.apache.htrace</groupId>
</exclusion>
<exclusion>
<artifactId>zookeeper</artifactId>
<groupId>org.apache.zookeeper</groupId>
</exclusion>
<exclusion>
<artifactId>commons-compress</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>kerb-simplekdc</artifactId>
<groupId>org.apache.kerby</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-databind</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
<exclusion>
<artifactId>stax2-api</artifactId>
<groupId>org.codehaus.woodstox</groupId>
</exclusion>
<exclusion>
<artifactId>woodstox-core</artifactId>
<groupId>com.fasterxml.woodstox</groupId>
</exclusion>
<exclusion>
<artifactId>dnsjava</artifactId>
<groupId>dnsjava</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-annotations</artifactId>
<groupId>org.apache.hadoop</groupId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<properties>
<seaweedfs.client.version>3.80</seaweedfs.client.version>
<hadoop.version>3.2.4</hadoop.version>
</properties>
</project>
+176
View File
@@ -0,0 +1,176 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<properties>
<seaweedfs.client.version>3.80</seaweedfs.client.version>
<hadoop.version>3.2.4</hadoop.version>
</properties>
<groupId>com.seaweedfs</groupId>
<artifactId>seaweedfs-hadoop2-client</artifactId>
<version>${seaweedfs.client.version}</version>
<name>SeaweedFS HDFS2 Client</name>
<description>A java client for SeaweedFS.</description>
<url>https://github.com/seaweedfs/seaweedfs</url>
<licenses>
<license>
<name>The Apache License, Version 2.0</name>
<url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
</license>
</licenses>
<developers>
<developer>
<name>Chris Lu</name>
<email>chris.lu@gmail.com</email>
<organization>SeaweedFS</organization>
<organizationUrl>https://seaweedfs.com</organizationUrl>
</developer>
</developers>
<scm>
<connection>scm:git:git://github.com/seaweedfs/seaweedfs.git</connection>
<developerConnection>scm:git:ssh://github.com:seaweedfs/seaweedfs.git</developerConnection>
<url>https://github.com/seaweedfs/seaweedfs/tree/master</url>
</scm>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>8</source>
<target>8</target>
<release>8</release>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>org/slf4j/**</exclude>
<exclude>META-INF/maven/org.slf4j/**</exclude>
</excludes>
</filter>
</filters>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
</transformers>
<relocations>
<relocation>
<pattern>com.google</pattern>
<shadedPattern>shaded.com.google</shadedPattern>
</relocation>
<relocation>
<pattern>io.grpc.internal</pattern>
<shadedPattern>shaded.io.grpc.internal</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.commons</pattern>
<shadedPattern>shaded.org.apache.commons</shadedPattern>
<excludes>
<exclude>org.apache.hadoop</exclude>
<exclude>org.apache.log4j</exclude>
</excludes>
</relocation>
<relocation>
<pattern>org.apache.http</pattern>
<shadedPattern>shaded.org.apache.http</shadedPattern>
</relocation>
</relocations>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-gpg-plugin</artifactId>
<version>1.5</version>
<executions>
<execution>
<id>sign-artifacts</id>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.sonatype.central</groupId>
<artifactId>central-publishing-maven-plugin</artifactId>
<version>0.5.0</version>
<extensions>true</extensions>
<configuration>
<publishingServerId>central</publishingServerId>
<autoPublish>true</autoPublish>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>2.2.1</version>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.0.1</version>
<executions>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.seaweedfs</groupId>
<artifactId>seaweedfs-client</artifactId>
<version>${seaweedfs.client.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
</project>
@@ -0,0 +1,25 @@
package seaweed.hdfs;
import org.apache.hadoop.fs.*;
import java.io.IOException;
import java.nio.ByteBuffer;
public class BufferedByteBufferReadableInputStream extends BufferedFSInputStream implements ByteBufferReadable {
public BufferedByteBufferReadableInputStream(FSInputStream in, int size) {
super(in, size);
if (!(in instanceof Seekable) || !(in instanceof PositionedReadable)) {
throw new IllegalArgumentException("In is not an instance of Seekable or PositionedReadable");
}
}
@Override
public int read(ByteBuffer buf) throws IOException {
if (this.in instanceof ByteBufferReadable) {
return ((ByteBufferReadable)this.in).read(buf);
} else {
throw new UnsupportedOperationException("Byte-buffer read unsupported by input stream");
}
}
}
@@ -0,0 +1,35 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package seaweed.hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.DelegateToFileSystem;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
public class SeaweedAbstractFileSystem extends DelegateToFileSystem {
SeaweedAbstractFileSystem(final URI uri, final Configuration conf)
throws IOException, URISyntaxException {
super(uri, new SeaweedFileSystem(), conf, "seaweedfs", false);
}
}
@@ -0,0 +1,634 @@
package seaweed.hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Progressable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import seaweedfs.client.FilerProto;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
public class SeaweedFileSystem extends FileSystem {
public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host";
public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
public static final String FS_SEAWEED_FILER_PORT_GRPC = "fs.seaweed.filer.port.grpc";
public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size";
public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication";
public static final String FS_SEAWEED_VOLUME_SERVER_ACCESS = "fs.seaweed.volume.server.access";
public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024;
public static final String FS_SEAWEED_FILER_CN = "fs.seaweed.filer.cn";
private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
private URI uri;
private Path workingDirectory = new Path("/");
private SeaweedFileSystemStore seaweedFileSystemStore;
public URI getUri() {
return uri;
}
public String getScheme() {
return "seaweedfs";
}
@Override
public void initialize(URI uri, Configuration conf) throws IOException { // get
super.initialize(uri, conf);
// get host information from uri (overrides info in conf)
String host = uri.getHost();
host = (host == null) ? conf.get(FS_SEAWEED_FILER_HOST, "localhost") : host;
conf.set(FS_SEAWEED_FILER_HOST, host);
// get port information from uri, (overrides info in conf)
int port = uri.getPort();
port = (port == -1) ? FS_SEAWEED_DEFAULT_PORT : port;
conf.setInt(FS_SEAWEED_FILER_PORT, port);
int grpcPort = conf.getInt(FS_SEAWEED_FILER_PORT_GRPC, port+10000);
setConf(conf);
this.uri = uri;
String cn = conf.get(FS_SEAWEED_FILER_CN, "");
seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, cn, conf);
}
@Override
public void close() throws IOException {
super.close();
this.seaweedFileSystemStore.close();
}
@Override
public FSDataInputStream open(Path path, int bufferSize) throws IOException {
LOG.debug("open path: {} bufferSize:{}", path, bufferSize);
path = qualify(path);
try {
int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
FSInputStream inputStream = seaweedFileSystemStore.openFileForRead(path, statistics);
return new FSDataInputStream(new BufferedByteBufferReadableInputStream(inputStream, 4 * seaweedBufferSize));
} catch (Exception ex) {
LOG.warn("open path: {} bufferSize:{}", path, bufferSize, ex);
return null;
}
}
@Override
public FSDataOutputStream create(Path path, FsPermission permission, final boolean overwrite, final int bufferSize,
final short replication, final long blockSize, final Progressable progress) throws IOException {
LOG.debug("create path: {} bufferSize:{} blockSize:{}", path, bufferSize, blockSize);
path = qualify(path);
try {
String replicaPlacement = this.getConf().get(FS_SEAWEED_REPLICATION, String.format("%03d", replication - 1));
int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, seaweedBufferSize, replicaPlacement);
return new FSDataOutputStream(outputStream, statistics);
} catch (Exception ex) {
LOG.warn("create path: {} bufferSize:{} blockSize:{}", path, bufferSize, blockSize, ex);
return null;
}
}
/**
* {@inheritDoc}
*
* @throws FileNotFoundException if the parent directory is not present -or
* is not a directory.
*/
@Override
public FSDataOutputStream createNonRecursive(Path path,
FsPermission permission,
EnumSet<CreateFlag> flags,
int bufferSize,
short replication,
long blockSize,
Progressable progress) throws IOException {
Path parent = path.getParent();
if (parent != null) {
// expect this to raise an exception if there is no parent
if (!getFileStatus(parent).isDirectory()) {
throw new FileAlreadyExistsException("Not a directory: " + parent);
}
}
int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
return create(path, permission,
flags.contains(CreateFlag.OVERWRITE), bufferSize,
replication, seaweedBufferSize, progress);
}
@Override
public FSDataOutputStream append(Path path, int bufferSize, Progressable progressable) throws IOException {
LOG.debug("append path: {} bufferSize:{}", path, bufferSize);
path = qualify(path);
try {
int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
OutputStream outputStream = seaweedFileSystemStore.createFile(path, false, null, seaweedBufferSize, "");
return new FSDataOutputStream(outputStream, statistics);
} catch (Exception ex) {
LOG.warn("append path: {} bufferSize:{}", path, bufferSize, ex);
return null;
}
}
@Override
public boolean rename(Path src, Path dst) throws IOException {
LOG.debug("rename path: {} => {}", src, dst);
if (src.isRoot()) {
return false;
}
if (src.equals(dst)) {
return true;
}
FilerProto.Entry entry = seaweedFileSystemStore.lookupEntry(dst);
Path adjustedDst = dst;
if (entry != null) {
FileStatus dstFileStatus = getFileStatus(dst);
String sourceFileName = src.getName();
if (!dstFileStatus.isDirectory()) {
return false;
}
adjustedDst = new Path(dst, sourceFileName);
}
Path qualifiedSrcPath = qualify(src);
Path qualifiedDstPath = qualify(adjustedDst);
seaweedFileSystemStore.rename(qualifiedSrcPath, qualifiedDstPath);
return true;
}
@Override
public boolean delete(Path path, boolean recursive) throws IOException {
LOG.debug("delete path: {} recursive:{}", path, recursive);
path = qualify(path);
FilerProto.Entry entry = seaweedFileSystemStore.lookupEntry(path);
if (entry == null) {
return true;
}
FileStatus fileStatus = getFileStatus(path);
return seaweedFileSystemStore.deleteEntries(path, fileStatus.isDirectory(), recursive);
}
@Override
public FileStatus[] listStatus(Path path) throws IOException {
LOG.debug("listStatus path: {}", path);
path = qualify(path);
return seaweedFileSystemStore.listEntries(path);
}
@Override
public Path getWorkingDirectory() {
return workingDirectory;
}
@Override
public void setWorkingDirectory(Path path) {
if (path.isAbsolute()) {
workingDirectory = path;
} else {
workingDirectory = new Path(workingDirectory, path);
}
}
@Override
public boolean mkdirs(Path path, FsPermission fsPermission) throws IOException {
LOG.debug("mkdirs path: {}", path);
path = qualify(path);
FilerProto.Entry entry = seaweedFileSystemStore.lookupEntry(path);
if (entry == null) {
UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
return seaweedFileSystemStore.createDirectory(path, currentUser,
fsPermission == null ? FsPermission.getDirDefault() : fsPermission,
FsPermission.getUMask(getConf()));
}
FileStatus fileStatus = getFileStatus(path);
if (fileStatus.isDirectory()) {
return true;
} else {
throw new FileAlreadyExistsException("Path is a file: " + path);
}
}
@Override
public FileStatus getFileStatus(Path path) throws IOException {
LOG.debug("getFileStatus path: {}", path);
path = qualify(path);
return seaweedFileSystemStore.getFileStatus(path);
}
/**
* Set owner of a path (i.e. a file or a directory).
* The parameters owner and group cannot both be null.
*
* @param path The path
* @param owner If it is null, the original username remains unchanged.
* @param group If it is null, the original groupname remains unchanged.
*/
@Override
public void setOwner(Path path, final String owner, final String group)
throws IOException {
LOG.debug("setOwner path: {}", path);
path = qualify(path);
seaweedFileSystemStore.setOwner(path, owner, group);
}
/**
* Set permission of a path.
*
* @param path The path
* @param permission Access permission
*/
@Override
public void setPermission(Path path, final FsPermission permission) throws IOException {
LOG.debug("setPermission path: {}", path);
if (permission == null) {
throw new IllegalArgumentException("The permission can't be null");
}
path = qualify(path);
seaweedFileSystemStore.setPermission(path, permission);
}
Path qualify(Path path) {
return path.makeQualified(uri, workingDirectory);
}
/**
* Concat existing files together.
*
* @param trg the path to the target destination.
* @param psrcs the paths to the sources to use for the concatenation.
* @throws IOException IO failure
* @throws UnsupportedOperationException if the operation is unsupported
* (default).
*/
@Override
public void concat(final Path trg, final Path[] psrcs) throws IOException {
throw new UnsupportedOperationException("Not implemented by the " +
getClass().getSimpleName() + " FileSystem implementation");
}
/**
* Truncate the file in the indicated path to the indicated size.
* <ul>
* <li>Fails if path is a directory.</li>
* <li>Fails if path does not exist.</li>
* <li>Fails if path is not closed.</li>
* <li>Fails if new size is greater than current size.</li>
* </ul>
*
* @param f The path to the file to be truncated
* @param newLength The size the file is to be truncated to
* @return <code>true</code> if the file has been truncated to the desired
* <code>newLength</code> and is immediately available to be reused for
* write operations such as <code>append</code>, or
* <code>false</code> if a background process of adjusting the length of
* the last block has been started, and clients should wait for it to
* complete before proceeding with further file updates.
* @throws IOException IO failure
* @throws UnsupportedOperationException if the operation is unsupported
* (default).
*/
@Override
public boolean truncate(Path f, long newLength) throws IOException {
throw new UnsupportedOperationException("Not implemented by the " +
getClass().getSimpleName() + " FileSystem implementation");
}
@Override
public void createSymlink(final Path target, final Path link,
final boolean createParent) throws
IOException {
// Supporting filesystems should override this method
throw new UnsupportedOperationException(
"Filesystem does not support symlinks!");
}
public boolean supportsSymlinks() {
return false;
}
/**
* Create a snapshot.
*
* @param path The directory where snapshots will be taken.
* @param snapshotName The name of the snapshot
* @return the snapshot path.
* @throws IOException IO failure
* @throws UnsupportedOperationException if the operation is unsupported
*/
@Override
public Path createSnapshot(Path path, String snapshotName)
throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
+ " doesn't support createSnapshot");
}
/**
* Rename a snapshot.
*
* @param path The directory path where the snapshot was taken
* @param snapshotOldName Old name of the snapshot
* @param snapshotNewName New name of the snapshot
* @throws IOException IO failure
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
*/
@Override
public void renameSnapshot(Path path, String snapshotOldName,
String snapshotNewName) throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
+ " doesn't support renameSnapshot");
}
/**
* Delete a snapshot of a directory.
*
* @param path The directory that the to-be-deleted snapshot belongs to
* @param snapshotName The name of the snapshot
* @throws IOException IO failure
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
*/
@Override
public void deleteSnapshot(Path path, String snapshotName)
throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
+ " doesn't support deleteSnapshot");
}
/**
* Modifies ACL entries of files and directories. This method can add new ACL
* entries or modify the permissions on existing ACL entries. All existing
* ACL entries that are not specified in this call are retained without
* changes. (Modifications are merged into the current ACL.)
*
* @param path Path to modify
* @param aclSpec List&lt;AclEntry&gt; describing modifications
* @throws IOException if an ACL could not be modified
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
*/
@Override
public void modifyAclEntries(Path path, List<AclEntry> aclSpec)
throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
+ " doesn't support modifyAclEntries");
}
/**
* Removes ACL entries from files and directories. Other ACL entries are
* retained.
*
* @param path Path to modify
* @param aclSpec List describing entries to remove
* @throws IOException if an ACL could not be modified
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
*/
@Override
public void removeAclEntries(Path path, List<AclEntry> aclSpec)
throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
+ " doesn't support removeAclEntries");
}
/**
* Removes all default ACL entries from files and directories.
*
* @param path Path to modify
* @throws IOException if an ACL could not be modified
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
*/
@Override
public void removeDefaultAcl(Path path)
throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
+ " doesn't support removeDefaultAcl");
}
/**
* Removes all but the base ACL entries of files and directories. The entries
* for user, group, and others are retained for compatibility with permission
* bits.
*
* @param path Path to modify
* @throws IOException if an ACL could not be removed
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
*/
@Override
public void removeAcl(Path path)
throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
+ " doesn't support removeAcl");
}
/**
* Fully replaces ACL of files and directories, discarding all existing
* entries.
*
* @param path Path to modify
* @param aclSpec List describing modifications, which must include entries
* for user, group, and others for compatibility with permission bits.
* @throws IOException if an ACL could not be modified
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
*/
@Override
public void setAcl(Path path, List<AclEntry> aclSpec) throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
+ " doesn't support setAcl");
}
/**
* Gets the ACL of a file or directory.
*
* @param path Path to get
* @return AclStatus describing the ACL of the file or directory
* @throws IOException if an ACL could not be read
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
*/
@Override
public AclStatus getAclStatus(Path path) throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
+ " doesn't support getAclStatus");
}
/**
* Set an xattr of a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
* <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
* @param name xattr name.
* @param value xattr value.
* @param flag xattr set flag
* @throws IOException IO failure
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
*/
@Override
public void setXAttr(Path path, String name, byte[] value,
EnumSet<XAttrSetFlag> flag) throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
+ " doesn't support setXAttr");
}
/**
* Get an xattr name and value for a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
* <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attribute
* @param name xattr name.
* @return byte[] xattr value.
* @throws IOException IO failure
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
*/
@Override
public byte[] getXAttr(Path path, String name) throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
+ " doesn't support getXAttr");
}
/**
* Get all of the xattr name/value pairs for a file or directory.
* Only those xattrs which the logged-in user has permissions to view
* are returned.
* <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @return Map describing the XAttrs of the file or directory
* @throws IOException IO failure
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
*/
@Override
public Map<String, byte[]> getXAttrs(Path path) throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
+ " doesn't support getXAttrs");
}
/**
* Get all of the xattrs name/value pairs for a file or directory.
* Only those xattrs which the logged-in user has permissions to view
* are returned.
* <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @param names XAttr names.
* @return Map describing the XAttrs of the file or directory
* @throws IOException IO failure
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
*/
@Override
public Map<String, byte[]> getXAttrs(Path path, List<String> names)
throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
+ " doesn't support getXAttrs");
}
/**
* Get all of the xattr names for a file or directory.
* Only those xattr names which the logged-in user has permissions to view
* are returned.
* <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @return List{@literal <String>} of the XAttr names of the file or directory
* @throws IOException IO failure
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
*/
@Override
public List<String> listXAttrs(Path path) throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
+ " doesn't support listXAttrs");
}
/**
* Remove an xattr of a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
* <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to remove extended attribute
* @param name xattr name
* @throws IOException IO failure
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
*/
@Override
public void removeXAttr(Path path, String name) throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
+ " doesn't support removeXAttr");
}
}
@@ -0,0 +1,291 @@
package seaweed.hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import seaweedfs.client.*;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import static seaweed.hdfs.SeaweedFileSystem.*;
public class SeaweedFileSystemStore {
private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystemStore.class);
private FilerClient filerClient;
private Configuration conf;
public SeaweedFileSystemStore(String host, int port, int grpcPort, String cn, Configuration conf) {
filerClient = new FilerClient(host, port, grpcPort, cn);
this.conf = conf;
String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct");
if (volumeServerAccessMode.equals("publicUrl")) {
filerClient.setAccessVolumeServerByPublicUrl();
} else if (volumeServerAccessMode.equals("filerProxy")) {
filerClient.setAccessVolumeServerByFilerProxy();
}
}
public void close() {
try {
this.filerClient.shutdown();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public static String getParentDirectory(Path path) {
return path.isRoot() ? "/" : path.getParent().toUri().getPath();
}
static int permissionToMode(FsPermission permission, boolean isDirectory) {
int p = permission.toShort();
if (isDirectory) {
p = p | 1 << 31;
}
return p;
}
public boolean createDirectory(final Path path, UserGroupInformation currentUser,
final FsPermission permission, final FsPermission umask) {
LOG.debug("createDirectory path: {} permission: {} umask: {}",
path,
permission,
umask);
return filerClient.mkdirs(
path.toUri().getPath(),
permissionToMode(permission, true),
currentUser.getUserName(),
currentUser.getGroupNames()
);
}
public FileStatus[] listEntries(final Path path) throws IOException {
LOG.debug("listEntries path: {}", path);
FileStatus pathStatus = getFileStatus(path);
if (pathStatus == null) {
return new FileStatus[0];
}
if (!pathStatus.isDirectory()) {
return new FileStatus[]{pathStatus};
}
List<FileStatus> fileStatuses = new ArrayList<FileStatus>();
List<FilerProto.Entry> entries = filerClient.listEntries(path.toUri().getPath());
for (FilerProto.Entry entry : entries) {
FileStatus fileStatus = doGetFileStatus(new Path(path, entry.getName()), entry);
fileStatuses.add(fileStatus);
}
LOG.debug("listEntries path: {} size {}", fileStatuses, fileStatuses.size());
return fileStatuses.toArray(new FileStatus[0]);
}
public FileStatus getFileStatus(final Path path) throws IOException {
FilerProto.Entry entry = lookupEntry(path);
if (entry == null) {
throw new FileNotFoundException("File does not exist: " + path);
}
LOG.debug("doGetFileStatus path:{} entry:{}", path, entry);
FileStatus fileStatus = doGetFileStatus(path, entry);
return fileStatus;
}
public boolean deleteEntries(final Path path, boolean isDirectory, boolean recursive) {
LOG.debug("deleteEntries path: {} isDirectory {} recursive: {}",
path,
String.valueOf(isDirectory),
String.valueOf(recursive));
if (path.isRoot()) {
return true;
}
if (recursive && isDirectory) {
List<FilerProto.Entry> entries = filerClient.listEntries(path.toUri().getPath());
for (FilerProto.Entry entry : entries) {
deleteEntries(new Path(path, entry.getName()), entry.getIsDirectory(), true);
}
}
return filerClient.deleteEntry(getParentDirectory(path), path.getName(), true, recursive, true);
}
private FileStatus doGetFileStatus(Path path, FilerProto.Entry entry) {
FilerProto.FuseAttributes attributes = entry.getAttributes();
long length = SeaweedRead.fileSize(entry);
boolean isDir = entry.getIsDirectory();
int block_replication = 1;
int blocksize = this.conf.getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
long modification_time = attributes.getMtime() * 1000; // milliseconds
long access_time = 0;
FsPermission permission = FsPermission.createImmutable((short) attributes.getFileMode());
String owner = attributes.getUserName();
String group = attributes.getGroupNameCount() > 0 ? attributes.getGroupName(0) : "";
return new FileStatus(length, isDir, block_replication, blocksize,
modification_time, access_time, permission, owner, group, null, path);
}
public FilerProto.Entry lookupEntry(Path path) {
return filerClient.lookupEntry(getParentDirectory(path), path.getName());
}
public void rename(Path source, Path destination) {
LOG.debug("rename source: {} destination:{}", source, destination);
if (source.isRoot()) {
return;
}
LOG.info("rename source: {} destination:{}", source, destination);
FilerProto.Entry entry = lookupEntry(source);
if (entry == null) {
LOG.warn("rename non-existing source: {}", source);
return;
}
filerClient.mv(source.toUri().getPath(), destination.toUri().getPath());
}
public OutputStream createFile(final Path path,
final boolean overwrite,
FsPermission permission,
int bufferSize,
String replication) throws IOException {
permission = permission == null ? FsPermission.getFileDefault() : permission;
LOG.debug("createFile path: {} overwrite: {} permission: {}",
path,
overwrite,
permission.toString());
UserGroupInformation userGroupInformation = UserGroupInformation.getCurrentUser();
long now = System.currentTimeMillis() / 1000L;
FilerProto.Entry.Builder entry = null;
long writePosition = 0;
if (!overwrite) {
FilerProto.Entry existingEntry = lookupEntry(path);
LOG.debug("createFile merged entry path:{} existingEntry:{}", path, existingEntry);
if (existingEntry != null) {
entry = FilerProto.Entry.newBuilder();
entry.mergeFrom(existingEntry);
entry.clearContent();
entry.getAttributesBuilder().setMtime(now);
LOG.debug("createFile merged entry path:{} entry:{} from:{}", path, entry, existingEntry);
writePosition = SeaweedRead.fileSize(existingEntry);
}
}
if (entry == null) {
entry = FilerProto.Entry.newBuilder()
.setName(path.getName())
.setIsDirectory(false)
.setAttributes(FilerProto.FuseAttributes.newBuilder()
.setFileMode(permissionToMode(permission, false))
.setCrtime(now)
.setMtime(now)
.setUserName(userGroupInformation.getUserName())
.clearGroupName()
.addAllGroupName(Arrays.asList(userGroupInformation.getGroupNames()))
);
SeaweedWrite.writeMeta(filerClient, getParentDirectory(path), entry);
}
return new SeaweedHadoopOutputStream(filerClient, path.toString(), entry, writePosition, bufferSize, replication);
}
public FSInputStream openFileForRead(final Path path, FileSystem.Statistics statistics) throws IOException {
LOG.debug("openFileForRead path:{}", path);
FilerProto.Entry entry = lookupEntry(path);
if (entry == null) {
throw new FileNotFoundException("read non-exist file " + path);
}
return new SeaweedHadoopInputStream(filerClient,
statistics,
path.toUri().getPath(),
entry);
}
public void setOwner(Path path, String owner, String group) {
LOG.debug("setOwner path:{} owner:{} group:{}", path, owner, group);
FilerProto.Entry entry = lookupEntry(path);
if (entry == null) {
LOG.debug("setOwner path:{} entry:{}", path, entry);
return;
}
FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
FilerProto.FuseAttributes.Builder attributesBuilder = entry.getAttributes().toBuilder();
if (owner != null) {
attributesBuilder.setUserName(owner);
}
if (group != null) {
attributesBuilder.clearGroupName();
attributesBuilder.addGroupName(group);
}
entryBuilder.setAttributes(attributesBuilder);
LOG.debug("setOwner path:{} entry:{}", path, entryBuilder);
filerClient.updateEntry(getParentDirectory(path), entryBuilder.build());
}
public void setPermission(Path path, FsPermission permission) {
LOG.debug("setPermission path:{} permission:{}", path, permission);
FilerProto.Entry entry = lookupEntry(path);
if (entry == null) {
LOG.debug("setPermission path:{} entry:{}", path, entry);
return;
}
FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
FilerProto.FuseAttributes.Builder attributesBuilder = entry.getAttributes().toBuilder();
attributesBuilder.setFileMode(permissionToMode(permission, entry.getIsDirectory()));
entryBuilder.setAttributes(attributesBuilder);
LOG.debug("setPermission path:{} entry:{}", path, entryBuilder);
filerClient.updateEntry(getParentDirectory(path), entryBuilder.build());
}
}
@@ -0,0 +1,150 @@
package seaweed.hdfs;
// based on org.apache.hadoop.fs.azurebfs.services.AbfsInputStream
import org.apache.hadoop.fs.ByteBufferReadable;
import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileSystem.Statistics;
import seaweedfs.client.FilerClient;
import seaweedfs.client.FilerProto;
import seaweedfs.client.SeaweedInputStream;
import java.io.EOFException;
import java.io.IOException;
import java.nio.ByteBuffer;
public class SeaweedHadoopInputStream extends FSInputStream implements ByteBufferReadable {
private final SeaweedInputStream seaweedInputStream;
private final Statistics statistics;
public SeaweedHadoopInputStream(
final FilerClient filerClient,
final Statistics statistics,
final String path,
final FilerProto.Entry entry) throws IOException {
this.seaweedInputStream = new SeaweedInputStream(filerClient, path, entry);
this.statistics = statistics;
}
@Override
public int read() throws IOException {
return seaweedInputStream.read();
}
@Override
public int read(final byte[] b, final int off, final int len) throws IOException {
return seaweedInputStream.read(b, off, len);
}
// implement ByteBufferReadable
@Override
public synchronized int read(ByteBuffer buf) throws IOException {
int bytesRead = seaweedInputStream.read(buf);
if (bytesRead > 0) {
if (statistics != null) {
statistics.incrementBytesRead(bytesRead);
}
}
return bytesRead;
}
/**
* Seek to given position in stream.
*
* @param n position to seek to
* @throws IOException if there is an error
* @throws EOFException if attempting to seek past end of file
*/
@Override
public synchronized void seek(long n) throws IOException {
seaweedInputStream.seek(n);
}
@Override
public synchronized long skip(long n) throws IOException {
return seaweedInputStream.skip(n);
}
/**
* Return the size of the remaining available bytes
* if the size is less than or equal to {@link Integer#MAX_VALUE},
* otherwise, return {@link Integer#MAX_VALUE}.
* <p>
* This is to match the behavior of DFSInputStream.available(),
* which some clients may rely on (HBase write-ahead log reading in
* particular).
*/
@Override
public synchronized int available() throws IOException {
return seaweedInputStream.available();
}
/**
* Returns the length of the file that this stream refers to. Note that the length returned is the length
* as of the time the Stream was opened. Specifically, if there have been subsequent appends to the file,
* they wont be reflected in the returned length.
*
* @return length of the file.
* @throws IOException if the stream is closed
*/
public long length() throws IOException {
return seaweedInputStream.length();
}
/**
* Return the current offset from the start of the file
*
* @throws IOException throws {@link IOException} if there is an error
*/
@Override
public synchronized long getPos() throws IOException {
return seaweedInputStream.getPos();
}
/**
* Seeks a different copy of the data. Returns true if
* found a new source, false otherwise.
*
* @throws IOException throws {@link IOException} if there is an error
*/
@Override
public boolean seekToNewSource(long l) throws IOException {
return false;
}
@Override
public synchronized void close() throws IOException {
seaweedInputStream.close();
}
/**
* Not supported by this stream. Throws {@link UnsupportedOperationException}
*
* @param readlimit ignored
*/
@Override
public synchronized void mark(int readlimit) {
throw new UnsupportedOperationException("mark()/reset() not supported on this stream");
}
/**
* Not supported by this stream. Throws {@link UnsupportedOperationException}
*/
@Override
public synchronized void reset() throws IOException {
throw new UnsupportedOperationException("mark()/reset() not supported on this stream");
}
/**
* gets whether mark and reset are supported by {@code ADLFileInputStream}. Always returns false.
*
* @return always {@code false}
*/
@Override
public boolean markSupported() {
return false;
}
}
@@ -0,0 +1,16 @@
package seaweed.hdfs;
// adapted from org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream
import seaweedfs.client.FilerClient;
import seaweedfs.client.FilerProto;
import seaweedfs.client.SeaweedOutputStream;
public class SeaweedHadoopOutputStream extends SeaweedOutputStream {
public SeaweedHadoopOutputStream(FilerClient filerClient, final String path, FilerProto.Entry.Builder entry,
final long position, final int bufferSize, final String replication) {
super(filerClient, path, entry, position, bufferSize, replication);
}
}
-199
View File
@@ -1,199 +0,0 @@
# SeaweedFS Hadoop3 Client
Hadoop FileSystem implementation for SeaweedFS, compatible with Hadoop 3.x.
## Building
```bash
mvn clean install
```
## Testing
This project includes two types of tests:
### 1. Configuration Tests (No SeaweedFS Required)
These tests verify configuration handling and initialization logic without requiring a running SeaweedFS instance:
```bash
mvn test -Dtest=SeaweedFileSystemConfigTest
```
### 2. Integration Tests (Requires SeaweedFS)
These tests verify actual FileSystem operations against a running SeaweedFS instance.
#### Prerequisites
1. Start SeaweedFS with default ports:
```bash
# Terminal 1: Start master
weed master
# Terminal 2: Start volume server
weed volume -mserver=localhost:9333
# Terminal 3: Start filer
weed filer -master=localhost:9333
```
2. Verify services are running:
- Master: http://localhost:9333
- Filer HTTP: http://localhost:8888
- Filer gRPC: localhost:18888
#### Running Integration Tests
```bash
# Enable integration tests
export SEAWEEDFS_TEST_ENABLED=true
# Run all tests
mvn test
# Run specific test
mvn test -Dtest=SeaweedFileSystemTest
```
### Test Configuration
Integration tests can be configured via environment variables or system properties:
- `SEAWEEDFS_TEST_ENABLED`: Set to `true` to enable integration tests (default: false)
- Tests use these default connection settings:
- Filer Host: localhost
- Filer HTTP Port: 8888
- Filer gRPC Port: 18888
### Running Tests with Custom Configuration
To test against a different SeaweedFS instance, modify the test code or use Hadoop configuration:
```java
conf.set("fs.seaweed.filer.host", "your-host");
conf.setInt("fs.seaweed.filer.port", 8888);
conf.setInt("fs.seaweed.filer.port.grpc", 18888);
```
## Test Coverage
The test suite covers:
- **Configuration & Initialization**
- URI parsing and configuration
- Default values
- Configuration overrides
- Working directory management
- **File Operations**
- Create files
- Read files
- Write files
- Append to files
- Delete files
- **Directory Operations**
- Create directories
- List directory contents
- Delete directories (recursive and non-recursive)
- **Metadata Operations**
- Get file status
- Set permissions
- Set owner/group
- Rename files and directories
## Usage in Hadoop
1. Copy the built JAR to your Hadoop classpath:
```bash
cp target/seaweedfs-hadoop3-client-*.jar $HADOOP_HOME/share/hadoop/common/lib/
```
2. Configure `core-site.xml`:
```xml
<configuration>
<property>
<name>fs.seaweedfs.impl</name>
<value>seaweed.hdfs.SeaweedFileSystem</value>
</property>
<property>
<name>fs.seaweed.filer.host</name>
<value>localhost</value>
</property>
<property>
<name>fs.seaweed.filer.port</name>
<value>8888</value>
</property>
<property>
<name>fs.seaweed.filer.port.grpc</name>
<value>18888</value>
</property>
<!-- Optional: Replication configuration with three priority levels:
1) If set to non-empty value (e.g. "001") - uses that value
2) If set to empty string "" - uses SeaweedFS filer's default replication
3) If not configured (property not present) - uses HDFS replication parameter
-->
<!-- <property>
<name>fs.seaweed.replication</name>
<value>001</value>
</property> -->
</configuration>
```
3. Use SeaweedFS with Hadoop commands:
```bash
hadoop fs -ls seaweedfs://localhost:8888/
hadoop fs -mkdir seaweedfs://localhost:8888/test
hadoop fs -put local.txt seaweedfs://localhost:8888/test/
```
## Continuous Integration
For CI environments, tests can be run in two modes:
1. **Configuration Tests Only** (default, no SeaweedFS required):
```bash
mvn test -Dtest=SeaweedFileSystemConfigTest
```
2. **Full Integration Tests** (requires SeaweedFS):
```bash
# Start SeaweedFS in CI environment
# Then run:
export SEAWEEDFS_TEST_ENABLED=true
mvn test
```
## Troubleshooting
### Tests are skipped
If you see "Skipping test - SEAWEEDFS_TEST_ENABLED not set":
```bash
export SEAWEEDFS_TEST_ENABLED=true
```
### Connection refused errors
Ensure SeaweedFS is running and accessible:
```bash
curl http://localhost:8888/
```
### gRPC errors
Verify the gRPC port is accessible:
```bash
# Should show the port is listening
netstat -an | grep 18888
```
## Contributing
When adding new features, please include:
1. Configuration tests (no SeaweedFS required)
2. Integration tests (with SEAWEEDFS_TEST_ENABLED guard)
3. Documentation updates
+12 -257
View File
@@ -140,7 +140,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.4.0</version>
<version>3.2.4</version>
<scope>provided</scope>
<exclusions>
<exclusion>
@@ -172,17 +172,9 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.4.0</version>
<version>3.2.4</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<artifactId>hadoop-shaded-protobuf_3_21</artifactId>
<groupId>org.apache.hadoop.thirdparty</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-shaded-guava</artifactId>
<groupId>org.apache.hadoop.thirdparty</groupId>
</exclusion>
<exclusion>
<artifactId>commons-cli</artifactId>
<groupId>commons-cli</groupId>
@@ -208,8 +200,8 @@
<groupId>javax.servlet</groupId>
</exclusion>
<exclusion>
<artifactId>jakarta.activation-api</artifactId>
<groupId>jakarta.activation</groupId>
<artifactId>javax.activation-api</artifactId>
<groupId>javax.activation</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-server</artifactId>
@@ -241,11 +233,7 @@
</exclusion>
<exclusion>
<artifactId>jersey-json</artifactId>
<groupId>com.github.pjfanning</groupId>
</exclusion>
<exclusion>
<artifactId>jettison</artifactId>
<groupId>org.codehaus.jettison</groupId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>jersey-server</artifactId>
@@ -299,32 +287,20 @@
<artifactId>curator-recipes</artifactId>
<groupId>org.apache.curator</groupId>
</exclusion>
<exclusion>
<artifactId>htrace-core4</artifactId>
<groupId>org.apache.htrace</groupId>
</exclusion>
<exclusion>
<artifactId>zookeeper</artifactId>
<groupId>org.apache.zookeeper</groupId>
</exclusion>
<exclusion>
<artifactId>netty-handler</artifactId>
<groupId>io.netty</groupId>
</exclusion>
<exclusion>
<artifactId>netty-transport-native-epoll</artifactId>
<groupId>io.netty</groupId>
</exclusion>
<exclusion>
<artifactId>metrics-core</artifactId>
<groupId>io.dropwizard.metrics</groupId>
</exclusion>
<exclusion>
<artifactId>commons-compress</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>bcprov-jdk15on</artifactId>
<groupId>org.bouncycastle</groupId>
</exclusion>
<exclusion>
<artifactId>kerb-core</artifactId>
<artifactId>kerb-simplekdc</artifactId>
<groupId>org.apache.kerby</groupId>
</exclusion>
<exclusion>
@@ -343,227 +319,6 @@
<artifactId>dnsjava</artifactId>
<groupId>dnsjava</groupId>
</exclusion>
<exclusion>
<artifactId>snappy-java</artifactId>
<groupId>org.xerial.snappy</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-annotations</artifactId>
<groupId>org.apache.hadoop</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.1</version>
<scope>test</scope>
<exclusions>
<exclusion>
<artifactId>hamcrest-core</artifactId>
<groupId>org.hamcrest</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>3.12.4</version>
<scope>test</scope>
<exclusions>
<exclusion>
<artifactId>byte-buddy</artifactId>
<groupId>net.bytebuddy</groupId>
</exclusion>
<exclusion>
<artifactId>byte-buddy-agent</artifactId>
<groupId>net.bytebuddy</groupId>
</exclusion>
<exclusion>
<artifactId>objenesis</artifactId>
<groupId>org.objenesis</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.4.0</version>
<type>test-jar</type>
<scope>test</scope>
<exclusions>
<exclusion>
<artifactId>hadoop-shaded-protobuf_3_21</artifactId>
<groupId>org.apache.hadoop.thirdparty</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-shaded-guava</artifactId>
<groupId>org.apache.hadoop.thirdparty</groupId>
</exclusion>
<exclusion>
<artifactId>commons-cli</artifactId>
<groupId>commons-cli</groupId>
</exclusion>
<exclusion>
<artifactId>commons-math3</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>commons-io</artifactId>
<groupId>commons-io</groupId>
</exclusion>
<exclusion>
<artifactId>commons-net</artifactId>
<groupId>commons-net</groupId>
</exclusion>
<exclusion>
<artifactId>commons-collections</artifactId>
<groupId>commons-collections</groupId>
</exclusion>
<exclusion>
<artifactId>javax.servlet-api</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
<exclusion>
<artifactId>jakarta.activation-api</artifactId>
<groupId>jakarta.activation</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-server</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-util</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-servlet</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-webapp</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>jsp-api</artifactId>
<groupId>javax.servlet.jsp</groupId>
</exclusion>
<exclusion>
<artifactId>jersey-core</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>jersey-servlet</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>jersey-json</artifactId>
<groupId>com.github.pjfanning</groupId>
</exclusion>
<exclusion>
<artifactId>jettison</artifactId>
<groupId>org.codehaus.jettison</groupId>
</exclusion>
<exclusion>
<artifactId>jersey-server</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>reload4j</artifactId>
<groupId>ch.qos.reload4j</groupId>
</exclusion>
<exclusion>
<artifactId>commons-beanutils</artifactId>
<groupId>commons-beanutils</groupId>
</exclusion>
<exclusion>
<artifactId>commons-configuration2</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>commons-lang3</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>commons-text</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-reload4j</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>avro</artifactId>
<groupId>org.apache.avro</groupId>
</exclusion>
<exclusion>
<artifactId>re2j</artifactId>
<groupId>com.google.re2j</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-auth</artifactId>
<groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
<artifactId>jsch</artifactId>
<groupId>com.jcraft</groupId>
</exclusion>
<exclusion>
<artifactId>curator-client</artifactId>
<groupId>org.apache.curator</groupId>
</exclusion>
<exclusion>
<artifactId>curator-recipes</artifactId>
<groupId>org.apache.curator</groupId>
</exclusion>
<exclusion>
<artifactId>zookeeper</artifactId>
<groupId>org.apache.zookeeper</groupId>
</exclusion>
<exclusion>
<artifactId>netty-handler</artifactId>
<groupId>io.netty</groupId>
</exclusion>
<exclusion>
<artifactId>netty-transport-native-epoll</artifactId>
<groupId>io.netty</groupId>
</exclusion>
<exclusion>
<artifactId>metrics-core</artifactId>
<groupId>io.dropwizard.metrics</groupId>
</exclusion>
<exclusion>
<artifactId>commons-compress</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>bcprov-jdk15on</artifactId>
<groupId>org.bouncycastle</groupId>
</exclusion>
<exclusion>
<artifactId>kerb-core</artifactId>
<groupId>org.apache.kerby</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-databind</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
<exclusion>
<artifactId>stax2-api</artifactId>
<groupId>org.codehaus.woodstox</groupId>
</exclusion>
<exclusion>
<artifactId>woodstox-core</artifactId>
<groupId>com.fasterxml.woodstox</groupId>
</exclusion>
<exclusion>
<artifactId>dnsjava</artifactId>
<groupId>dnsjava</groupId>
</exclusion>
<exclusion>
<artifactId>snappy-java</artifactId>
<groupId>org.xerial.snappy</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-annotations</artifactId>
<groupId>org.apache.hadoop</groupId>
@@ -572,7 +327,7 @@
</dependency>
</dependencies>
<properties>
<seaweedfs.client.version>4.00</seaweedfs.client.version>
<hadoop.version>3.4.0</hadoop.version>
<seaweedfs.client.version>3.80</seaweedfs.client.version>
<hadoop.version>3.2.4</hadoop.version>
</properties>
</project>

Some files were not shown because too many files have changed in this diff Show More