Compare commits

..
Author SHA1 Message Date
Chris Lu 263b7bbf53 admin: require CSRF for S3 tables bucket writes 2026-02-10 17:58:45 -08:00
Chris LuandGitHub 5c365e7090 s3api: return 400 for invalid namespace query in REST table routes (#8296)
* s3api: reject invalid namespace query in REST table routes

* s3api: expand namespace validation REST tests
2026-02-10 17:57:08 -08:00
Chris LuandGitHub 822dbed552 s3api: fix ListObjectsV2 NextContinuationToken duplication for nested prefix (#8294)
* s3api: fix duplicate ListObjectsV2 continuation token for nested prefix

* s3api: include prefix in common-prefix continuation token
2026-02-10 14:17:41 -08:00
Chris LuandGitHub 2d97685390 ci: fix container_release_unified manual dispatch and workflow parsing (#8293)
ci: fix unified release workflow dispatch matrix filtering
2026-02-10 13:15:52 -08:00
Chris LuandGitHub 1b2f719d7c admin: fix file browser items-per-page selector (#8291)
* admin: fix file browser page size selector

Fix file browser pagination page-size selectors to use explicit select IDs instead of this.value in templ-generated handlers, which could resolve to undefined and produce limit=undefined in requests.

Add a focused template render regression test to prevent this from recurring.

Fixes #8284

* revert file browser template regression test
2026-02-10 12:56:34 -08:00
Chris LuandGitHub b73bd08470 ci: move manual container builds to unified release workflow (#8290)
* ci: move manual dev container build into unified release workflow

* ci: make unified manual container build release-tag based
2026-02-10 12:47:39 -08:00
Chris LuandGitHub 17f85361e9 Remove unsupported iceberg rest signing-region from tests and docs (#8289) 2026-02-10 12:23:31 -08:00
Chris LuandGitHub b261c89675 Fix RocksDB container build compatibility and add manual rocksdb dispatch (#8288)
* Fix RocksDB build compatibility and add manual rocksdb trigger

* Upgrade RocksDB defaults and keep grocksdb v1.10.7

* Add manual latest-image trigger inputs for ref and variant

* Allow manual latest build to set image tag and source ref

* Fix manual variant selection using setup job matrix output
2026-02-10 11:48:42 -08:00
Chris LuGitHubgemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
0385acba02 s3tables: fix shared table-location bucket mapping collisions (#8286)
* s3tables: prevent shared table-location bucket mapping overwrite

* Update weed/s3api/bucket_paths.go

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
2026-02-10 11:28:29 -08:00
Chris LuandGitHub d6825ffce2 Iceberg: implement stage-create finalize flow (phase 1) (#8279)
* iceberg: implement stage-create and create-on-commit finalize

* iceberg: add create validation error typing and stage-create integration test

* tests: merge stage-create integration check into catalog suite

* tests: cover stage-create finalize lifecycle in catalog integration

* iceberg: persist and cleanup stage-create markers

* iceberg: add stage-create rollout flag and marker pruning

* docs: add stage-create support design and rollout plan

* docs: drop stage-create design draft from PR

* iceberg: use conservative 72h stage-marker retention

* iceberg: address review comments on create-on-commit and tests

* iceberg: keep stage-create metadata out of table location

* refactor(iceberg): split iceberg.go into focused files
2026-02-10 09:46:09 -08:00
Chris LuandGitHub d88f6ed0af Iceberg commit reliability: preserve statistics updates and return 409 conflicts (#8277)
* iceberg: harden table commit updates and conflict handling

* iceberg: refine commit retry and statistics patching

* iceberg: cleanup metadata on non-conflict commit errors
2026-02-09 23:00:03 -08:00
Chris LuandGitHub 5ae3be44d1 iceberg: persist namespace properties for create/get (#8276)
* iceberg: persist namespace properties via s3tables metadata

* iceberg: simplify namespace properties normalization

* s3tables: broaden namespace properties round-trip test

* adjust logs

* adjust logs
2026-02-09 22:20:45 -08:00
Chris LuGitHubgemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
1c62808c0e iceberg: wire pagination for list namespaces/tables REST APIs (#8275)
* s3api/iceberg: wire list pagination tokens and page size

* fmt

* Update weed/s3api/iceberg/iceberg.go

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
2026-02-09 21:46:55 -08:00
Chris Lu db76eb26e7 compile 2026-02-09 21:06:07 -08:00
Chris Lu 4ccc7668ce admin: resolve merge conflicts 2026-02-09 20:56:20 -08:00
Chris LuandGitHub aef2de3109 s3tables: support multi-level namespaces in parser/admin paths (#8273)
* s3tables: support multi-level namespace normalization

* admin: handle namespace parsing errors centrally

* admin: clean namespace validation duplication
2026-02-09 20:20:05 -08:00
Chris Lu be26ce74ce s3tables: support multi-level namespace normalization 2026-02-09 19:42:31 -08:00
51 changed files with 3759 additions and 2024 deletions
-1
View File
@@ -3,7 +3,6 @@ name: "docker: build dev containers"
on:
push:
branches: [ master ]
workflow_dispatch: {}
permissions:
contents: read
+59 -20
View File
@@ -4,22 +4,58 @@ on:
push:
tags:
- '*'
workflow_dispatch: {}
workflow_dispatch:
inputs:
source_ref:
description: 'Git ref to build (branch, tag, or commit SHA)'
required: true
default: 'master'
image_tag:
description: 'Docker tag to publish (without variant suffix)'
required: true
default: 'latest'
variant:
description: 'Variant to build manually'
required: true
type: choice
default: all
options:
- all
- standard
- large_disk
permissions:
contents: read
jobs:
setup:
runs-on: ubuntu-latest
outputs:
variants: ${{ steps.set-variants.outputs.variants }}
steps:
- name: Select variants for this run
id: set-variants
run: |
if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ github.event.inputs.variant }}" != "all" ]; then
variants="[\"${{ github.event.inputs.variant }}\"]"
else
variants='["standard","large_disk"]'
fi
echo "variants=$variants" >> "$GITHUB_OUTPUT"
build:
needs: [setup]
runs-on: ubuntu-latest
strategy:
matrix:
platform: [amd64, arm64, arm, 386]
variant: [standard, large_disk]
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
- name: Free Disk Space
run: |
echo "Available disk space before cleanup:"
@@ -55,7 +91,7 @@ jobs:
images: |
chrislusf/seaweedfs
ghcr.io/chrislusf/seaweedfs
tags: type=raw,value=latest,suffix=${{ steps.config.outputs.tag_suffix }}
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
@@ -97,13 +133,13 @@ jobs:
file: ./docker/Dockerfile.go_build
platforms: linux/${{ matrix.platform }}
# Push to GHCR only during build to avoid Docker Hub rate limits
tags: ghcr.io/chrislusf/seaweedfs:latest${{ steps.config.outputs.tag_suffix }}-${{ matrix.platform }}
tags: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-${{ matrix.platform }}
labels: ${{ steps.docker_meta.outputs.labels }}
cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.platform }}
cache-to: type=gha,mode=max,scope=${{ matrix.variant }}-${{ matrix.platform }}
build-args: |
BUILDKIT_INLINE_CACHE=1
BRANCH=${{ github.sha }}
BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
${{ steps.config.outputs.build_args }}
- name: Clean up build artifacts
if: always()
@@ -115,14 +151,16 @@ jobs:
create-manifest:
runs-on: ubuntu-latest
needs: [build]
needs: [setup, build]
if: github.event_name != 'pull_request'
strategy:
matrix:
variant: [standard, large_disk]
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
- name: Configure variant
id: config
@@ -140,7 +178,7 @@ jobs:
images: |
chrislusf/seaweedfs
ghcr.io/chrislusf/seaweedfs
tags: type=raw,value=latest,suffix=${{ steps.config.outputs.tag_suffix }}
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
@@ -162,14 +200,15 @@ jobs:
- name: Create and push manifest
run: |
SUFFIX="${{ steps.config.outputs.tag_suffix }}"
BASE_TAG="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}"
# Create manifest on GHCR first (no rate limits)
echo "Creating GHCR manifest (no rate limits)..."
docker buildx imagetools create -t ghcr.io/chrislusf/seaweedfs:latest${SUFFIX} \
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-amd64 \
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-arm64 \
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-arm \
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-386
docker buildx imagetools create -t ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
# Copy the complete multi-arch image from GHCR to Docker Hub
# This only requires one pull from GHCR (no rate limit) and one push to Docker Hub
@@ -205,16 +244,16 @@ jobs:
# Use crane or skopeo to copy, fallback to docker if not available
if command -v crane &> /dev/null; then
echo "Using crane to copy..."
retry_with_backoff crane copy ghcr.io/chrislusf/seaweedfs:latest${SUFFIX} chrislusf/seaweedfs:latest${SUFFIX}
retry_with_backoff crane copy ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
elif command -v skopeo &> /dev/null; then
echo "Using skopeo to copy..."
retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs:latest${SUFFIX} docker://chrislusf/seaweedfs:latest${SUFFIX}
retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} docker://chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
else
echo "Using docker buildx imagetools (pulling 4 images from Docker Hub)..."
# Fallback: create manifest directly on Docker Hub (pulls from Docker Hub - rate limited)
retry_with_backoff docker buildx imagetools create -t chrislusf/seaweedfs:latest${SUFFIX} \
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-amd64 \
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-arm64 \
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-arm \
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-386
retry_with_backoff docker buildx imagetools create -t chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
fi
-152
View File
@@ -1,152 +0,0 @@
name: "docker: build release containers for normal volume"
# DISABLED: Merged into container_release_unified.yml
on:
workflow_dispatch:
inputs:
force_run:
description: 'This workflow is disabled. Use container_release_unified.yml instead'
required: true
default: 'disabled'
permissions:
contents: read
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
platform: [amd64, arm64, arm, 386]
include:
- platform: amd64
qemu: false
- platform: arm64
qemu: true
- platform: arm
qemu: true
- platform: 386
qemu: true
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Free Disk Space
run: |
echo "Available disk space before cleanup:"
df -h
# Remove pre-installed tools
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
# Clean package managers
sudo apt-get clean
sudo rm -rf /var/lib/apt/lists/*
# Clean Docker aggressively
sudo docker system prune -af --volumes
# Clean Go cache if it exists
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
[ -d /go/pkg ] && rm -rf /go/pkg || true
echo "Available disk space after cleanup:"
df -h
- name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
with:
images: chrislusf/seaweedfs
tags: type=ref,event=tag
flavor: latest=false
- name: Set up QEMU
if: matrix.qemu
uses: docker/setup-qemu-action@v3
- name: Create BuildKit config
run: |
cat > /tmp/buildkitd.toml <<EOF
[registry."docker.io"]
mirrors = ["https://mirror.gcr.io"]
EOF
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
buildkitd-config: /tmp/buildkitd.toml
- name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build ${{ matrix.platform }}
uses: docker/build-push-action@v6
env:
DOCKER_BUILDKIT: 1
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
platforms: linux/${{ matrix.platform }}
tags: ${{ steps.docker_meta.outputs.tags }}-${{ matrix.platform }}
labels: ${{ steps.docker_meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
build-args: |
BUILDKIT_INLINE_CACHE=1
BRANCH=${{ github.sha }}
- name: Clean up build artifacts
if: always()
run: |
# Clean up Docker build cache and temporary files
sudo docker system prune -f
# Remove Go build cache
sudo rm -rf /tmp/go-build*
create-manifest:
runs-on: ubuntu-latest
needs: [build]
if: github.event_name != 'pull_request'
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
with:
images: chrislusf/seaweedfs
tags: type=ref,event=tag
flavor: latest=false
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Create and push manifest
run: |
# Function to retry command with exponential backoff
retry_with_backoff() {
local max_attempts=5
local timeout=1
local attempt=1
local exit_code=0
while [ $attempt -le $max_attempts ]; do
if "$@"; then
return 0
else
exit_code=$?
fi
if [ $attempt -lt $max_attempts ]; then
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
sleep $timeout
timeout=$((timeout * 2))
fi
attempt=$((attempt + 1))
done
echo "Command failed after $max_attempts attempts" >&2
return $exit_code
}
# Create manifest with retry
retry_with_backoff docker buildx imagetools create -t ${{ steps.docker_meta.outputs.tags }} \
${{ steps.docker_meta.outputs.tags }}-amd64 \
${{ steps.docker_meta.outputs.tags }}-arm64 \
${{ steps.docker_meta.outputs.tags }}-arm \
${{ steps.docker_meta.outputs.tags }}-386
-153
View File
@@ -1,153 +0,0 @@
name: "docker: build release containers for large volume"
# DISABLED: Merged into container_release_unified.yml
on:
workflow_dispatch:
inputs:
force_run:
description: 'This workflow is disabled. Use container_release_unified.yml instead'
required: true
default: 'disabled'
permissions:
contents: read
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
platform: [amd64, arm64, arm, 386]
include:
- platform: amd64
qemu: false
- platform: arm64
qemu: true
- platform: arm
qemu: true
- platform: 386
qemu: true
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Free Disk Space
run: |
echo "Available disk space before cleanup:"
df -h
# Remove pre-installed tools
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
# Clean package managers
sudo apt-get clean
sudo rm -rf /var/lib/apt/lists/*
# Clean Docker aggressively
sudo docker system prune -af --volumes
# Clean Go cache if it exists
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
[ -d /go/pkg ] && rm -rf /go/pkg || true
echo "Available disk space after cleanup:"
df -h
- name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
with:
images: chrislusf/seaweedfs
tags: type=ref,event=tag,suffix=_large_disk
flavor: latest=false
- name: Set up QEMU
if: matrix.qemu
uses: docker/setup-qemu-action@v3
- name: Create BuildKit config
run: |
cat > /tmp/buildkitd.toml <<EOF
[registry."docker.io"]
mirrors = ["https://mirror.gcr.io"]
EOF
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
buildkitd-config: /tmp/buildkitd.toml
- name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build ${{ matrix.platform }}
uses: docker/build-push-action@v6
env:
DOCKER_BUILDKIT: 1
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
build-args: |
TAGS=5BytesOffset
BUILDKIT_INLINE_CACHE=1
BRANCH=${{ github.sha }}
platforms: linux/${{ matrix.platform }}
tags: ${{ steps.docker_meta.outputs.tags }}-${{ matrix.platform }}
labels: ${{ steps.docker_meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Clean up build artifacts
if: always()
run: |
# Clean up Docker build cache and temporary files
sudo docker system prune -f
# Remove Go build cache
sudo rm -rf /tmp/go-build*
create-manifest:
runs-on: ubuntu-latest
needs: [build]
if: github.event_name != 'pull_request'
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
with:
images: chrislusf/seaweedfs
tags: type=ref,event=tag,suffix=_large_disk
flavor: latest=false
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Create and push manifest
run: |
# Function to retry command with exponential backoff
retry_with_backoff() {
local max_attempts=5
local timeout=1
local attempt=1
local exit_code=0
while [ $attempt -le $max_attempts ]; do
if "$@"; then
return 0
else
exit_code=$?
fi
if [ $attempt -lt $max_attempts ]; then
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
sleep $timeout
timeout=$((timeout * 2))
fi
attempt=$((attempt + 1))
done
echo "Command failed after $max_attempts attempts" >&2
return $exit_code
}
# Create manifest with retry
retry_with_backoff docker buildx imagetools create -t ${{ steps.docker_meta.outputs.tags }} \
${{ steps.docker_meta.outputs.tags }}-amd64 \
${{ steps.docker_meta.outputs.tags }}-arm64 \
${{ steps.docker_meta.outputs.tags }}-arm \
${{ steps.docker_meta.outputs.tags }}-386
-73
View File
@@ -1,73 +0,0 @@
name: "docker: build release containers for rocksdb"
# DISABLED: Merged into container_release_unified.yml
on:
workflow_dispatch:
inputs:
force_run:
description: 'This workflow is disabled. Use container_release_unified.yml instead'
required: true
default: 'disabled'
permissions:
contents: read
jobs:
build-large-release-container_rocksdb:
runs-on: [ubuntu-latest]
steps:
-
name: Checkout
uses: actions/checkout@v6
-
name: Free Disk Space
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
sudo docker system prune -af
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
with:
images: chrislusf/seaweedfs
tags: type=ref,event=tag,suffix=_large_disk_rocksdb
flavor: latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Create BuildKit config
run: |
cat > /tmp/buildkitd.toml <<EOF
[registry."docker.io"]
mirrors = ["https://mirror.gcr.io"]
EOF
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
buildkitd-config: /tmp/buildkitd.toml
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@v6
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.rocksdb_large
build-args: |
BRANCH=${{ github.sha }}
platforms: linux/amd64
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
-72
View File
@@ -1,72 +0,0 @@
name: "docker: build release containers for all tags"
# DISABLED: Merged into container_release_unified.yml
on:
workflow_dispatch:
inputs:
force_run:
description: 'This workflow is disabled. Use container_release_unified.yml instead'
required: true
default: 'disabled'
permissions:
contents: read
jobs:
build-default-release-container:
runs-on: [ubuntu-latest]
steps:
-
name: Checkout
uses: actions/checkout@v6
-
name: Free Disk Space
run: |
echo "Before cleanup:"
df -h
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
sudo docker system prune -af
echo "After cleanup:"
df -h
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
with:
images: |
chrislusf/seaweedfs
tags: |
type=ref,event=tag,suffix=_full
flavor: |
latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@v6
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
build-args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
platforms: linux/amd64
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
-62
View File
@@ -1,62 +0,0 @@
name: "docker: build release containers for all tags and large volume"
# DISABLED: Merged into container_release_unified.yml
on:
workflow_dispatch:
inputs:
force_run:
description: 'This workflow is disabled. Use container_release_unified.yml instead'
required: true
default: 'disabled'
permissions:
contents: read
jobs:
build-default-release-container:
runs-on: [ubuntu-latest]
steps:
-
name: Checkout
uses: actions/checkout@v6
-
name: Free Disk Space
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
sudo docker system prune -af
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
with:
images: chrislusf/seaweedfs
tags: type=ref,event=tag,suffix=_large_disk_full
flavor: latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@v6
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
build-args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
platforms: linux/amd64
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
+45 -13
View File
@@ -4,11 +4,35 @@ on:
push:
tags:
- '*'
workflow_dispatch: {}
workflow_dispatch:
inputs:
variant:
description: 'Variant to build manually'
required: true
type: choice
default: all
options:
- all
- normal
- large_disk
- full
- large_disk_full
- rocksdb
release_tag:
description: 'Release tag to publish (e.g. 3.93)'
required: true
default: ''
rocksdb_version:
description: 'RocksDB git tag to use when variant=rocksdb'
required: false
default: 'v10.10.1'
permissions:
contents: read
env:
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
# Limit concurrent builds to avoid rate limits
concurrency:
group: release-${{ github.ref }}
@@ -59,9 +83,11 @@ jobs:
steps:
- name: Checkout
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: actions/checkout@v6
- name: Free Disk Space
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
echo "Available disk space before cleanup:"
df -h
@@ -75,13 +101,14 @@ jobs:
df -h
- name: Docker meta
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
id: docker_meta
uses: docker/metadata-action@v5
with:
images: |
chrislusf/seaweedfs
ghcr.io/chrislusf/seaweedfs
tags: type=ref,event=tag,suffix=${{ matrix.tag_suffix }}
tags: type=raw,value=${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
flavor: latest=false
labels: |
org.opencontainers.image.title=seaweedfs
@@ -89,10 +116,11 @@ jobs:
org.opencontainers.image.vendor=Chris Lu
- name: Set up QEMU
if: contains(matrix.platforms, 'arm')
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && contains(matrix.platforms, 'arm')
uses: docker/setup-qemu-action@v3
- name: Create BuildKit config
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
cat > /tmp/buildkitd.toml <<EOF
[registry."docker.io"]
@@ -100,19 +128,20 @@ jobs:
EOF
- name: Set up Docker Buildx
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/setup-buildx-action@v3
with:
buildkitd-config: /tmp/buildkitd.toml
- name: Login to Docker Hub
if: github.event_name != 'pull_request'
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
if: github.event_name != 'pull_request'
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
registry: ghcr.io
@@ -120,6 +149,7 @@ jobs:
password: ${{ secrets.GHCR_TOKEN }}
- name: Build and push ${{ matrix.variant }}
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/build-push-action@v6
env:
DOCKER_BUILDKIT: 1
@@ -130,7 +160,7 @@ jobs:
platforms: ${{ matrix.platforms }}
# Push to GHCR to avoid Docker Hub rate limits on pulls
tags: |
ghcr.io/chrislusf/seaweedfs:${{ github.ref_name }}${{ matrix.tag_suffix }}
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
labels: ${{ steps.docker_meta.outputs.labels }}
cache-from: type=gha,scope=${{ matrix.variant }}
cache-to: type=gha,mode=max,scope=${{ matrix.variant }}
@@ -138,9 +168,10 @@ jobs:
${{ matrix.build_args }}
BUILDKIT_INLINE_CACHE=1
BRANCH=${{ github.sha }}
${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }}
- name: Clean up build artifacts
if: always()
if: always() && (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant)
run: |
sudo docker system prune -f
sudo rm -rf /tmp/go-build*
@@ -166,12 +197,14 @@ jobs:
steps:
- name: Login to Docker Hub
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/login-action@v3
with:
registry: ghcr.io
@@ -179,6 +212,7 @@ jobs:
password: ${{ secrets.GHCR_TOKEN }}
- name: Install crane
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
cd $(mktemp -d)
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
@@ -186,6 +220,7 @@ jobs:
crane version
- name: Copy ${{ matrix.variant }} from GHCR to Docker Hub
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
run: |
# Function to retry with exponential backoff
retry_with_backoff() {
@@ -218,14 +253,15 @@ jobs:
# This is much more efficient than pulling/pushing individual arch images
echo "Copying ${{ matrix.variant }} from GHCR to Docker Hub..."
retry_with_backoff crane copy \
ghcr.io/chrislusf/seaweedfs:${{ github.ref_name }}${{ matrix.tag_suffix }} \
chrislusf/seaweedfs:${{ github.ref_name }}${{ matrix.tag_suffix }}
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"
helm-release:
runs-on: ubuntu-latest
needs: [copy-to-dockerhub]
if: github.event_name == 'push'
permissions:
contents: write
pages: write
@@ -239,7 +275,3 @@ jobs:
target_dir: helm
branch: gh-pages
helm_version: "3.18.4"
@@ -4,9 +4,9 @@ on:
workflow_dispatch:
inputs:
rocksdb_version:
description: 'RocksDB git tag or branch to build (e.g. v10.5.1)'
description: 'RocksDB git tag or branch to build (e.g. v10.10.1)'
required: true
default: 'v10.5.1'
default: 'v10.10.1'
seaweedfs_ref:
description: 'SeaweedFS git tag, branch, or commit to build'
required: true
+1 -1
View File
@@ -3,7 +3,7 @@ FROM golang:1.24 AS builder
RUN apt-get update
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
ARG ROCKSDB_VERSION=v10.5.1
ARG ROCKSDB_VERSION=v10.10.1
ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
# build RocksDB
+1 -1
View File
@@ -3,7 +3,7 @@ FROM golang:1.24 AS builder
RUN apt-get update
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
ARG ROCKSDB_VERSION=v10.5.1
ARG ROCKSDB_VERSION=v10.10.1
ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
# build RocksDB
-2
View File
@@ -300,7 +300,6 @@ populate-spark: ensure-table-bucket ensure-work-dir ## Populate sample data via
--conf "spark.sql.catalog.iceberg.s3.region=$(AWS_REGION)" \
--conf "spark.sql.catalog.iceberg.rest.sigv4-enabled=true" \
--conf "spark.sql.catalog.iceberg.rest.signing-name=s3tables" \
--conf "spark.sql.catalog.iceberg.rest.signing-region=$(AWS_REGION)" \
--conf "spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkCatalog" \
--conf "spark.sql.catalog.spark_catalog.type=rest" \
--conf "spark.sql.catalog.spark_catalog.uri=$(CATALOG_ENDPOINT_DOCKER)" \
@@ -314,7 +313,6 @@ populate-spark: ensure-table-bucket ensure-work-dir ## Populate sample data via
--conf "spark.sql.catalog.spark_catalog.s3.region=$(AWS_REGION)" \
--conf "spark.sql.catalog.spark_catalog.rest.sigv4-enabled=true" \
--conf "spark.sql.catalog.spark_catalog.rest.signing-name=s3tables" \
--conf "spark.sql.catalog.spark_catalog.rest.signing-region=$(AWS_REGION)" \
-f /work/spark_seed.sql'
populate: populate-trino populate-spark ## Populate sample data through Trino and Spark
@@ -3,7 +3,9 @@
package catalog
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net"
@@ -290,6 +292,182 @@ func TestIcebergNamespaces(t *testing.T) {
}
}
// TestStageCreateAndFinalizeFlow verifies staged create remains invisible until assert-create commit finalizes table creation.
func TestStageCreateAndFinalizeFlow(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
env := NewTestEnvironment(t)
defer env.Cleanup(t)
env.StartSeaweedFS(t)
createTableBucket(t, env, "warehouse")
namespace := "stage_ns"
tableName := "orders"
status, _, err := doIcebergJSONRequest(env, http.MethodPost, "/v1/namespaces", map[string]any{
"namespace": []string{namespace},
})
if err != nil {
t.Fatalf("Create namespace request failed: %v", err)
}
if status != http.StatusOK && status != http.StatusConflict {
t.Fatalf("Create namespace status = %d, want 200 or 409", status)
}
status, badReqResp, err := doIcebergJSONRequest(env, http.MethodPost, fmt.Sprintf("/v1/namespaces/%s/tables", namespace), map[string]any{
"stage-create": true,
})
if err != nil {
t.Fatalf("Stage create missing-name request failed: %v", err)
}
if status != http.StatusBadRequest {
t.Fatalf("Stage create missing-name status = %d, want 400", status)
}
errorObj, _ := badReqResp["error"].(map[string]any)
if got := errorObj["type"]; got != "BadRequestException" {
t.Fatalf("error.type = %v, want BadRequestException", got)
}
msg, _ := errorObj["message"].(string)
if !strings.Contains(strings.ToLower(msg), "table name is required") {
t.Fatalf("error.message = %v, want it to include %q", errorObj["message"], "table name is required")
}
status, stageResp, err := doIcebergJSONRequest(env, http.MethodPost, fmt.Sprintf("/v1/namespaces/%s/tables", namespace), map[string]any{
"name": tableName,
"stage-create": true,
})
if err != nil {
t.Fatalf("Stage create request failed: %v", err)
}
if status != http.StatusOK {
t.Fatalf("Stage create status = %d, want 200", status)
}
stageLocation, _ := stageResp["metadata-location"].(string)
if !strings.HasSuffix(stageLocation, "/metadata/v1.metadata.json") {
t.Fatalf("stage metadata-location = %q, want suffix /metadata/v1.metadata.json", stageLocation)
}
status, _, err = doIcebergJSONRequest(env, http.MethodGet, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName), nil)
if err != nil {
t.Fatalf("Load staged table request failed: %v", err)
}
if status != http.StatusNotFound {
t.Fatalf("Load staged table status = %d, want 404", status)
}
status, commitResp, err := doIcebergJSONRequest(env, http.MethodPost, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName), map[string]any{
"requirements": []map[string]any{
{"type": "assert-create"},
},
"updates": []any{},
})
if err != nil {
t.Fatalf("Finalize commit request failed: %v", err)
}
if status != http.StatusOK {
t.Fatalf("Finalize commit status = %d, want 200", status)
}
commitLocation, _ := commitResp["metadata-location"].(string)
if !strings.HasSuffix(commitLocation, "/metadata/v1.metadata.json") {
t.Fatalf("final metadata-location = %q, want suffix /metadata/v1.metadata.json", commitLocation)
}
status, loadResp, err := doIcebergJSONRequest(env, http.MethodGet, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName), nil)
if err != nil {
t.Fatalf("Load finalized table request failed: %v", err)
}
if status != http.StatusOK {
t.Fatalf("Load finalized table status = %d, want 200", status)
}
loadLocation, _ := loadResp["metadata-location"].(string)
if loadLocation != commitLocation {
t.Fatalf("loaded metadata-location = %q, want %q", loadLocation, commitLocation)
}
}
// TestCommitMissingTableWithoutAssertCreate ensures missing-table commits still require assert-create for creation.
func TestCommitMissingTableWithoutAssertCreate(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
env := NewTestEnvironment(t)
defer env.Cleanup(t)
env.StartSeaweedFS(t)
createTableBucket(t, env, "warehouse")
namespace := "stage_missing_assert_ns"
tableName := "missing_table"
status, _, err := doIcebergJSONRequest(env, http.MethodPost, "/v1/namespaces", map[string]any{
"namespace": []string{namespace},
})
if err != nil {
t.Fatalf("Create namespace request failed: %v", err)
}
if status != http.StatusOK && status != http.StatusConflict {
t.Fatalf("Create namespace status = %d, want 200 or 409", status)
}
status, _, err = doIcebergJSONRequest(env, http.MethodPost, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName), map[string]any{
"requirements": []any{},
"updates": []any{},
})
if err != nil {
t.Fatalf("Commit missing table request failed: %v", err)
}
if status != http.StatusNotFound {
t.Fatalf("Commit missing table status = %d, want 404", status)
}
}
// doIcebergJSONRequest decodes JSON object responses used by catalog tests.
func doIcebergJSONRequest(env *TestEnvironment, method, path string, payload any) (int, map[string]any, error) {
url := env.IcebergURL() + path
var bodyReader io.Reader
if payload != nil {
data, err := json.Marshal(payload)
if err != nil {
return 0, nil, err
}
bodyReader = bytes.NewReader(data)
}
req, err := http.NewRequest(method, url, bodyReader)
if err != nil {
return 0, nil, err
}
if payload != nil {
req.Header.Set("Content-Type", "application/json")
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return 0, nil, err
}
defer resp.Body.Close()
data, err := io.ReadAll(resp.Body)
if err != nil {
return resp.StatusCode, nil, err
}
if len(data) == 0 {
return resp.StatusCode, nil, nil
}
var decoded map[string]any
if err := json.Unmarshal(data, &decoded); err != nil {
return resp.StatusCode, nil, fmt.Errorf("failed to decode %s %s response: %w body=%s", method, path, err, string(data))
}
return resp.StatusCode, decoded, nil
}
// createTableBucket creates a table bucket via the S3Tables REST API
func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
t.Helper()
@@ -326,7 +326,6 @@ spark = (SparkSession.builder
.config("spark.sql.catalog.iceberg.rest.auth.type", "sigv4")
.config("spark.sql.catalog.iceberg.rest.auth.sigv4.delegate-auth-type", "none")
.config("spark.sql.catalog.iceberg.rest.sigv4-enabled", "true")
.config("spark.sql.catalog.iceberg.rest.signing-region", "us-west-2")
.config("spark.sql.catalog.iceberg.rest.signing-name", "s3")
.config("spark.sql.catalog.iceberg.rest.access-key-id", "test")
.config("spark.sql.catalog.iceberg.rest.secret-access-key", "test")
+64 -9
View File
@@ -63,6 +63,19 @@ type tableBucketMetadata struct {
const s3TablesAdminListLimit = 1000
func parseNamespaceInput(namespace string) ([]string, error) {
return s3tables.ParseNamespace(namespace)
}
func (s *AdminServer) parseNamespaceFromGin(c *gin.Context, namespace string) ([]string, bool) {
parts, err := parseNamespaceInput(namespace)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid namespace: " + err.Error()})
return nil, false
}
return parts, true
}
func newS3TablesManager() *s3tables.Manager {
manager := s3tables.NewManager()
manager.SetAccountID(s3_constants.AccountAdminId)
@@ -158,7 +171,11 @@ func (s *AdminServer) GetS3TablesTablesData(ctx context.Context, bucketArn, name
var resp s3tables.ListTablesResponse
var ns []string
if namespace != "" {
ns = []string{namespace}
parts, err := parseNamespaceInput(namespace)
if err != nil {
return S3TablesTablesData{}, err
}
ns = parts
}
req := &s3tables.ListTablesRequest{TableBucketARN: bucketArn, Namespace: ns, MaxTables: s3TablesAdminListLimit}
if err := s.executeS3TablesOperation(ctx, "ListTables", req, &resp); err != nil {
@@ -257,9 +274,13 @@ func (s *AdminServer) GetIcebergTablesData(ctx context.Context, catalogName, buc
// GetIcebergTableDetailsData returns Iceberg table metadata and snapshot information.
func (s *AdminServer) GetIcebergTableDetailsData(ctx context.Context, catalogName, bucketArn, namespace, tableName string) (IcebergTableDetailsData, error) {
var resp s3tables.GetTableResponse
namespaceParts, err := parseNamespaceInput(namespace)
if err != nil {
return IcebergTableDetailsData{}, err
}
req := &s3tables.GetTableRequest{
TableBucketARN: bucketArn,
Namespace: []string{namespace},
Namespace: namespaceParts,
Name: tableName,
}
if err := s.executeS3TablesOperation(ctx, "GetTable", req, &resp); err != nil {
@@ -558,6 +579,9 @@ func (s *AdminServer) ListS3TablesBucketsAPI(c *gin.Context) {
}
func (s *AdminServer) CreateS3TablesBucket(c *gin.Context) {
if !requireSessionCSRFToken(c) {
return
}
var req struct {
Name string `json:"name"`
Tags map[string]string `json:"tags"`
@@ -643,6 +667,9 @@ func (s *AdminServer) SetTableBucketOwner(ctx context.Context, bucketName, owner
}
func (s *AdminServer) DeleteS3TablesBucket(c *gin.Context) {
if !requireSessionCSRFToken(c) {
return
}
bucketArn := c.Query("bucket")
if bucketArn == "" {
c.JSON(400, gin.H{"error": "Bucket ARN is required"})
@@ -686,7 +713,11 @@ func (s *AdminServer) CreateS3TablesNamespace(c *gin.Context) {
c.JSON(400, gin.H{"error": "bucket_arn and name are required"})
return
}
createReq := &s3tables.CreateNamespaceRequest{TableBucketARN: req.BucketARN, Namespace: []string{req.Name}}
namespaceParts, ok := s.parseNamespaceFromGin(c, req.Name)
if !ok {
return
}
createReq := &s3tables.CreateNamespaceRequest{TableBucketARN: req.BucketARN, Namespace: namespaceParts}
var resp s3tables.CreateNamespaceResponse
if err := s.executeS3TablesOperation(c.Request.Context(), "CreateNamespace", createReq, &resp); err != nil {
writeS3TablesError(c, err)
@@ -705,7 +736,11 @@ func (s *AdminServer) DeleteS3TablesNamespace(c *gin.Context) {
c.JSON(400, gin.H{"error": "bucket and name query parameters are required"})
return
}
req := &s3tables.DeleteNamespaceRequest{TableBucketARN: bucketArn, Namespace: []string{namespace}}
namespaceParts, ok := s.parseNamespaceFromGin(c, namespace)
if !ok {
return
}
req := &s3tables.DeleteNamespaceRequest{TableBucketARN: bucketArn, Namespace: namespaceParts}
if err := s.executeS3TablesOperation(c.Request.Context(), "DeleteNamespace", req, nil); err != nil {
writeS3TablesError(c, err)
return
@@ -745,6 +780,10 @@ func (s *AdminServer) CreateS3TablesTable(c *gin.Context) {
c.JSON(400, gin.H{"error": "bucket_arn, namespace, and name are required"})
return
}
namespaceParts, ok := s.parseNamespaceFromGin(c, req.Namespace)
if !ok {
return
}
format := req.Format
if format == "" {
format = "ICEBERG"
@@ -757,7 +796,7 @@ func (s *AdminServer) CreateS3TablesTable(c *gin.Context) {
}
createReq := &s3tables.CreateTableRequest{
TableBucketARN: req.BucketARN,
Namespace: []string{req.Namespace},
Namespace: namespaceParts,
Name: req.Name,
Format: format,
Tags: req.Tags,
@@ -780,7 +819,11 @@ func (s *AdminServer) DeleteS3TablesTable(c *gin.Context) {
c.JSON(400, gin.H{"error": "bucket, namespace, and name query parameters are required"})
return
}
req := &s3tables.DeleteTableRequest{TableBucketARN: bucketArn, Namespace: []string{namespace}, Name: name, VersionToken: version}
namespaceParts, ok := s.parseNamespaceFromGin(c, namespace)
if !ok {
return
}
req := &s3tables.DeleteTableRequest{TableBucketARN: bucketArn, Namespace: namespaceParts, Name: name, VersionToken: version}
if err := s.executeS3TablesOperation(c.Request.Context(), "DeleteTable", req, nil); err != nil {
writeS3TablesError(c, err)
return
@@ -853,7 +896,11 @@ func (s *AdminServer) PutS3TablesTablePolicy(c *gin.Context) {
c.JSON(400, gin.H{"error": "bucket_arn, namespace, name, and policy are required"})
return
}
putReq := &s3tables.PutTablePolicyRequest{TableBucketARN: req.BucketARN, Namespace: []string{req.Namespace}, Name: req.Name, ResourcePolicy: req.Policy}
namespaceParts, ok := s.parseNamespaceFromGin(c, req.Namespace)
if !ok {
return
}
putReq := &s3tables.PutTablePolicyRequest{TableBucketARN: req.BucketARN, Namespace: namespaceParts, Name: req.Name, ResourcePolicy: req.Policy}
if err := s.executeS3TablesOperation(c.Request.Context(), "PutTablePolicy", putReq, nil); err != nil {
writeS3TablesError(c, err)
return
@@ -869,7 +916,11 @@ func (s *AdminServer) GetS3TablesTablePolicy(c *gin.Context) {
c.JSON(400, gin.H{"error": "bucket, namespace, and name query parameters are required"})
return
}
getReq := &s3tables.GetTablePolicyRequest{TableBucketARN: bucketArn, Namespace: []string{namespace}, Name: name}
namespaceParts, ok := s.parseNamespaceFromGin(c, namespace)
if !ok {
return
}
getReq := &s3tables.GetTablePolicyRequest{TableBucketARN: bucketArn, Namespace: namespaceParts, Name: name}
var resp s3tables.GetTablePolicyResponse
if err := s.executeS3TablesOperation(c.Request.Context(), "GetTablePolicy", getReq, &resp); err != nil {
writeS3TablesError(c, err)
@@ -886,7 +937,11 @@ func (s *AdminServer) DeleteS3TablesTablePolicy(c *gin.Context) {
c.JSON(400, gin.H{"error": "bucket, namespace, and name query parameters are required"})
return
}
deleteReq := &s3tables.DeleteTablePolicyRequest{TableBucketARN: bucketArn, Namespace: []string{namespace}, Name: name}
namespaceParts, ok := s.parseNamespaceFromGin(c, namespace)
if !ok {
return
}
deleteReq := &s3tables.DeleteTablePolicyRequest{TableBucketARN: bucketArn, Namespace: namespaceParts, Name: name}
if err := s.executeS3TablesOperation(c.Request.Context(), "DeleteTablePolicy", deleteReq, nil); err != nil {
writeS3TablesError(c, err)
return
+20 -13
View File
@@ -725,20 +725,27 @@ function s3TablesNamespaceNameError(name) {
if (name.includes('/')) {
return "namespace name cannot contain '/'";
}
if (!isLowercaseLetterOrDigit(name[0])) {
return 'Namespace name must start with a letter or digit';
}
if (!isLowercaseLetterOrDigit(name[name.length - 1])) {
return 'Namespace name must end with a letter or digit';
}
for (const ch of name) {
if (isLowercaseLetterOrDigit(ch) || ch === '_') {
continue;
const parts = name.split('.');
for (const part of parts) {
if (!part) {
return 'namespace levels cannot be empty';
}
if (!isLowercaseLetterOrDigit(part[0])) {
return 'Namespace name must start with a letter or digit';
}
if (!isLowercaseLetterOrDigit(part[part.length - 1])) {
return 'Namespace name must end with a letter or digit';
}
for (const ch of part) {
if (isLowercaseLetterOrDigit(ch) || ch === '_') {
continue;
}
return "invalid namespace name: only 'a-z', '0-9', and '_' are allowed";
}
if (part.startsWith('aws')) {
return "namespace name cannot start with reserved prefix 'aws'";
}
return "invalid namespace name: only 'a-z', '0-9', and '_' are allowed";
}
if (name.startsWith('aws')) {
return "namespace name cannot start with reserved prefix 'aws'";
}
return '';
}
+8 -4
View File
@@ -7,8 +7,12 @@ import (
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
)
script changePageSize(path string, lastFileName string) {
window.location.href = '/files?path=' + encodeURIComponent(path) + '&lastFileName=' + encodeURIComponent(lastFileName) + '&limit=' + this.value
script changePageSize(path string, lastFileName string, pageSizeSelectId string) {
const pageSizeSelect = document.getElementById(pageSizeSelectId)
if (pageSizeSelect == null) {
return
}
window.location.href = '/files?path=' + encodeURIComponent(path) + '&lastFileName=' + encodeURIComponent(lastFileName) + '&limit=' + pageSizeSelect.value
}
templ FileBrowser(data dash.FileBrowserData) {
@@ -94,7 +98,7 @@ templ FileBrowser(data dash.FileBrowserData) {
<div class="d-flex align-items-center gap-2 ms-auto">
<div class="d-flex align-items-center">
<label class="me-1 mb-0 small text-muted">Show:</label>
<select class="form-select form-select-sm" style="width: 70px; font-size: 0.875rem;" onchange={ changePageSize(data.CurrentPath, data.CurrentLastFileName) }>
<select id="file-browser-page-size-top" class="form-select form-select-sm" style="width: 70px; font-size: 0.875rem;" onchange={ changePageSize(data.CurrentPath, data.CurrentLastFileName, "file-browser-page-size-top") }>
<option value="20" selected?={ data.PageSize == 20 }>20</option>
<option value="50" selected?={ data.PageSize == 50 }>50</option>
<option value="100" selected?={ data.PageSize == 100 }>100</option>
@@ -221,7 +225,7 @@ templ FileBrowser(data dash.FileBrowserData) {
<div class="col-md-6">
<div class="d-flex align-items-center">
<label class="me-2 mb-0">Show:</label>
<select class="form-select form-select-sm" style="width: auto;" onchange={ changePageSize(data.CurrentPath, data.CurrentLastFileName) }>
<select id="file-browser-page-size-bottom" class="form-select form-select-sm" style="width: auto;" onchange={ changePageSize(data.CurrentPath, data.CurrentLastFileName, "file-browser-page-size-bottom") }>
<option value="20" selected?={ data.PageSize == 20 }>20</option>
<option value="50" selected?={ data.PageSize == 50 }>50</option>
<option value="100" selected?={ data.PageSize == 100 }>100</option>
+43 -39
View File
@@ -15,13 +15,17 @@ import (
"strings"
)
func changePageSize(path string, lastFileName string) templ.ComponentScript {
func changePageSize(path string, lastFileName string, pageSizeSelectId string) templ.ComponentScript {
return templ.ComponentScript{
Name: `__templ_changePageSize_6f57`,
Function: `function __templ_changePageSize_6f57(path, lastFileName){window.location.href = '/files?path=' + encodeURIComponent(path) + '&lastFileName=' + encodeURIComponent(lastFileName) + '&limit=' + this.value
Name: `__templ_changePageSize_43f7`,
Function: `function __templ_changePageSize_43f7(path, lastFileName, pageSizeSelectId){const pageSizeSelect = document.getElementById(pageSizeSelectId)
if (pageSizeSelect == null) {
return
}
window.location.href = '/files?path=' + encodeURIComponent(path) + '&lastFileName=' + encodeURIComponent(lastFileName) + '&limit=' + pageSizeSelect.value
}`,
Call: templ.SafeScript(`__templ_changePageSize_6f57`, path, lastFileName),
CallInline: templ.SafeScriptInline(`__templ_changePageSize_6f57`, path, lastFileName),
Call: templ.SafeScript(`__templ_changePageSize_43f7`, path, lastFileName, pageSizeSelectId),
CallInline: templ.SafeScriptInline(`__templ_changePageSize_43f7`, path, lastFileName, pageSizeSelectId),
}
}
@@ -58,7 +62,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var2 string
templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(data.TableBucketName)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 18, Col: 71}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 22, Col: 71}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
if templ_7745c5c3_Err != nil {
@@ -72,7 +76,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var3 string
templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(data.BucketName)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 20, Col: 62}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 24, Col: 62}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
if templ_7745c5c3_Err != nil {
@@ -112,7 +116,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var4 templ.SafeURL
templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/files?path=%s", crumb.Path)))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 58, Col: 72}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 62, Col: 72}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
if templ_7745c5c3_Err != nil {
@@ -125,7 +129,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var5 string
templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(crumb.Name)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 59, Col: 19}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 63, Col: 19}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
if templ_7745c5c3_Err != nil {
@@ -143,7 +147,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var6 templ.SafeURL
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/files?path=%s", crumb.Path)))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 64, Col: 72}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 68, Col: 72}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
if templ_7745c5c3_Err != nil {
@@ -162,7 +166,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var7 string
templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(crumb.Name)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 68, Col: 19}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 72, Col: 19}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
if templ_7745c5c3_Err != nil {
@@ -196,7 +200,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var8 templ.SafeURL
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/files?path=%s", data.CurrentPath)))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 89, Col: 76}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 93, Col: 76}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
if templ_7745c5c3_Err != nil {
@@ -209,7 +213,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var9 string
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(filepath.Base(data.CurrentPath))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 89, Col: 154}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 93, Col: 154}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
if templ_7745c5c3_Err != nil {
@@ -224,15 +228,15 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templ.RenderScriptItems(ctx, templ_7745c5c3_Buffer, changePageSize(data.CurrentPath, data.CurrentLastFileName))
templ_7745c5c3_Err = templ.RenderScriptItems(ctx, templ_7745c5c3_Buffer, changePageSize(data.CurrentPath, data.CurrentLastFileName, "file-browser-page-size-top"))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "<select class=\"form-select form-select-sm\" style=\"width: 70px; font-size: 0.875rem;\" onchange=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "<select id=\"file-browser-page-size-top\" class=\"form-select form-select-sm\" style=\"width: 70px; font-size: 0.875rem;\" onchange=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var10 templ.ComponentScript = changePageSize(data.CurrentPath, data.CurrentLastFileName)
var templ_7745c5c3_Var10 templ.ComponentScript = changePageSize(data.CurrentPath, data.CurrentLastFileName, "file-browser-page-size-top")
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var10.Call)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
@@ -289,7 +293,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var11 templ.SafeURL
templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/files?path=%s&lastFileName=%s&limit=%d", data.CurrentPath, data.LastFileName, data.PageSize)))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 106, Col: 137}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 110, Col: 137}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
if templ_7745c5c3_Err != nil {
@@ -313,7 +317,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var12 templ.SafeURL
templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/files?path=%s", data.ParentPath)))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 115, Col: 77}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 119, Col: 77}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12))
if templ_7745c5c3_Err != nil {
@@ -341,7 +345,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var13 string
templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(entry.FullPath)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 143, Col: 77}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 147, Col: 77}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
if templ_7745c5c3_Err != nil {
@@ -359,7 +363,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var14 templ.SafeURL
templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/files?path=%s", entry.FullPath)))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 149, Col: 82}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 153, Col: 82}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
if templ_7745c5c3_Err != nil {
@@ -372,7 +376,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var15 string
templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(entry.Name)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 150, Col: 25}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 154, Col: 25}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
if templ_7745c5c3_Err != nil {
@@ -408,7 +412,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var18 string
templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(entry.Name)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 154, Col: 30}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 158, Col: 30}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
if templ_7745c5c3_Err != nil {
@@ -432,7 +436,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var19 string
templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(formatBytes(entry.Size))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 162, Col: 36}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 166, Col: 36}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19))
if templ_7745c5c3_Err != nil {
@@ -452,7 +456,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var20 string
templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(getMimeDisplayName(entry.Mime))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 170, Col: 44}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 174, Col: 44}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20))
if templ_7745c5c3_Err != nil {
@@ -467,7 +471,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var21 string
templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinStringErrs(entry.ModTime.Format("2006-01-02 15:04"))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 176, Col: 53}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 180, Col: 53}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var21))
if templ_7745c5c3_Err != nil {
@@ -486,7 +490,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var22 string
templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinStringErrs(entry.Mode)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 182, Col: 72}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 186, Col: 72}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var22))
if templ_7745c5c3_Err != nil {
@@ -499,7 +503,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var23 string
templ_7745c5c3_Var23, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%t", entry.IsDirectory))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 182, Col: 131}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 186, Col: 131}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var23))
if templ_7745c5c3_Err != nil {
@@ -512,7 +516,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var24 string
templ_7745c5c3_Var24, templ_7745c5c3_Err = templ.JoinStringErrs(entry.Mode)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 182, Col: 146}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 186, Col: 146}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var24))
if templ_7745c5c3_Err != nil {
@@ -530,7 +534,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var25 string
templ_7745c5c3_Var25, templ_7745c5c3_Err = templ.JoinStringErrs(entry.FullPath)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 187, Col: 139}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 191, Col: 139}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var25))
if templ_7745c5c3_Err != nil {
@@ -543,7 +547,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var26 string
templ_7745c5c3_Var26, templ_7745c5c3_Err = templ.JoinStringErrs(entry.FullPath)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 190, Col: 128}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 194, Col: 128}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var26))
if templ_7745c5c3_Err != nil {
@@ -561,7 +565,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var27 string
templ_7745c5c3_Var27, templ_7745c5c3_Err = templ.JoinStringErrs(entry.FullPath)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 194, Col: 144}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 198, Col: 144}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var27))
if templ_7745c5c3_Err != nil {
@@ -574,7 +578,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var28 string
templ_7745c5c3_Var28, templ_7745c5c3_Err = templ.JoinStringErrs(entry.FullPath)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 197, Col: 133}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 201, Col: 133}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var28))
if templ_7745c5c3_Err != nil {
@@ -599,15 +603,15 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templ.RenderScriptItems(ctx, templ_7745c5c3_Buffer, changePageSize(data.CurrentPath, data.CurrentLastFileName))
templ_7745c5c3_Err = templ.RenderScriptItems(ctx, templ_7745c5c3_Buffer, changePageSize(data.CurrentPath, data.CurrentLastFileName, "file-browser-page-size-bottom"))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 67, "<select class=\"form-select form-select-sm\" style=\"width: auto;\" onchange=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 67, "<select id=\"file-browser-page-size-bottom\" class=\"form-select form-select-sm\" style=\"width: auto;\" onchange=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var29 templ.ComponentScript = changePageSize(data.CurrentPath, data.CurrentLastFileName)
var templ_7745c5c3_Var29 templ.ComponentScript = changePageSize(data.CurrentPath, data.CurrentLastFileName, "file-browser-page-size-bottom")
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var29.Call)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
@@ -664,7 +668,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var30 templ.SafeURL
templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/files?path=%s&lastFileName=%s&limit=%d", data.CurrentPath, data.LastFileName, data.PageSize)))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 237, Col: 137}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 241, Col: 137}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30))
if templ_7745c5c3_Err != nil {
@@ -687,7 +691,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var31 string
templ_7745c5c3_Var31, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("2006-01-02 15:04:05"))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 252, Col: 66}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 256, Col: 66}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var31))
if templ_7745c5c3_Err != nil {
@@ -700,7 +704,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var32 string
templ_7745c5c3_Var32, templ_7745c5c3_Err = templ.JoinStringErrs(data.CurrentPath)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 277, Col: 87}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 281, Col: 87}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var32))
if templ_7745c5c3_Err != nil {
@@ -713,7 +717,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
var templ_7745c5c3_Var33 string
templ_7745c5c3_Var33, templ_7745c5c3_Err = templ.JoinStringErrs(data.CurrentPath)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 309, Col: 79}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/file_browser.templ`, Line: 313, Col: 79}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var33))
if templ_7745c5c3_Err != nil {
+1 -1
View File
@@ -195,7 +195,7 @@ templ IcebergNamespaces(data dash.IcebergNamespacesData) {
<div class="mb-3">
<label for="icebergNamespaceName" class="form-label">Namespace</label>
<input type="text" class="form-control" id="icebergNamespaceName" name="name" placeholder="analytics" required/>
<div class="form-text">Use lowercase letters, numbers, and underscores. Nested namespaces are not supported.</div>
<div class="form-text">Use lowercase letters, numbers, and underscores. Use dots for nested namespaces (for example, analytics.daily).</div>
</div>
</div>
<div class="modal-footer">
@@ -327,7 +327,7 @@ func IcebergNamespaces(data dash.IcebergNamespacesData) templ.Component {
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "\"> <input type=\"hidden\" id=\"icebergNamespaceCsrfToken\" name=\"csrf_token\"><div class=\"mb-3\"><label for=\"icebergNamespaceName\" class=\"form-label\">Namespace</label> <input type=\"text\" class=\"form-control\" id=\"icebergNamespaceName\" name=\"name\" placeholder=\"analytics\" required><div class=\"form-text\">Use lowercase letters, numbers, and underscores. Nested namespaces are not supported.</div></div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"submit\" class=\"btn btn-primary\"><i class=\"fas fa-plus me-1\"></i>Create</button></div></form></div></div></div><script>\n\t\tdocument.addEventListener('DOMContentLoaded', function() {\n\t\t\tinitIcebergNamespaces();\n\t\t});\n\t</script>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "\"> <input type=\"hidden\" id=\"icebergNamespaceCsrfToken\" name=\"csrf_token\"><div class=\"mb-3\"><label for=\"icebergNamespaceName\" class=\"form-label\">Namespace</label> <input type=\"text\" class=\"form-control\" id=\"icebergNamespaceName\" name=\"name\" placeholder=\"analytics\" required><div class=\"form-text\">Use lowercase letters, numbers, and underscores. Use dots for nested namespaces (for example, analytics.daily).</div></div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"submit\" class=\"btn btn-primary\"><i class=\"fas fa-plus me-1\"></i>Create</button></div></form></div></div></div><script>\n\t\tdocument.addEventListener('DOMContentLoaded', function() {\n\t\t\tinitIcebergNamespaces();\n\t\t});\n\t</script>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
+48 -36
View File
@@ -154,14 +154,14 @@ templ S3TablesNamespaces(data dash.S3TablesNamespacesData) {
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<form id="createS3TablesNamespaceForm">
<div class="modal-body">
<input type="hidden" name="bucket_arn" value={ data.BucketARN }/>
<div class="mb-3">
<label for="s3tablesNamespaceName" class="form-label">Namespace</label>
<input type="text" class="form-control" id="s3tablesNamespaceName" name="name" placeholder="analytics" required/>
<div class="form-text">Use lowercase letters, numbers, and underscores. Nested namespaces are not supported.</div>
<div class="modal-body">
<input type="hidden" name="bucket_arn" value={ data.BucketARN }/>
<div class="mb-3">
<label for="s3tablesNamespaceName" class="form-label">Namespace</label>
<input type="text" class="form-control" id="s3tablesNamespaceName" name="name" placeholder="analytics" required/>
<div class="form-text">Use lowercase letters, numbers, and underscores. Use dots for nested namespaces (for example, analytics.daily).</div>
</div>
</div>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
<button type="submit" class="btn btn-primary">
@@ -235,41 +235,53 @@ templ S3TablesNamespaces(data dash.S3TablesNamespacesData) {
input.setCustomValidity('');
return '';
}
let message = '';
if (!name) {
message = 'Namespace name is required';
} else if (name.length < 1 || name.length > 255) {
message = 'Namespace name must be between 1 and 255 characters';
} else if (name === '.' || name === '..') {
message = "namespace name cannot be '.' or '..'";
} else if (name.includes('/')) {
message = "namespace name cannot contain '/'";
} else {
const start = name[0];
const end = name[name.length - 1];
const isStartValid = (start >= 'a' && start <= 'z') || (start >= '0' && start <= '9');
const isEndValid = (end >= 'a' && end <= 'z') || (end >= '0' && end <= '9');
if (!isStartValid) {
message = 'Namespace name must start with a letter or digit';
} else if (!isEndValid) {
message = 'Namespace name must end with a letter or digit';
let message = '';
if (!name) {
message = 'Namespace name is required';
} else if (name.length < 1 || name.length > 255) {
message = 'Namespace name must be between 1 and 255 characters';
} else if (name === '.' || name === '..') {
message = "namespace name cannot be '.' or '..'";
} else if (name.includes('/')) {
message = "namespace name cannot contain '/'";
} else {
for (const ch of name) {
const isLower = ch >= 'a' && ch <= 'z';
const isDigit = ch >= '0' && ch <= '9';
if (!(isLower || isDigit || ch === '_')) {
message = "invalid namespace name: only 'a-z', '0-9', and '_' are allowed";
for (const part of name.split('.')) {
if (!part) {
message = 'namespace levels cannot be empty';
break;
}
const start = part[0];
const end = part[part.length - 1];
const isStartValid = (start >= 'a' && start <= 'z') || (start >= '0' && start <= '9');
const isEndValid = (end >= 'a' && end <= 'z') || (end >= '0' && end <= '9');
if (!isStartValid) {
message = 'Namespace name must start with a letter or digit';
break;
}
if (!isEndValid) {
message = 'Namespace name must end with a letter or digit';
break;
}
for (const ch of part) {
const isLower = ch >= 'a' && ch <= 'z';
const isDigit = ch >= '0' && ch <= '9';
if (!(isLower || isDigit || ch === '_')) {
message = "invalid namespace name: only 'a-z', '0-9', and '_' are allowed";
break;
}
}
if (message) {
break;
}
if (part.startsWith('aws')) {
message = "namespace name cannot start with reserved prefix 'aws'";
break;
}
}
if (!message && name.startsWith('aws')) {
message = "namespace name cannot start with reserved prefix 'aws'";
}
}
input.setCustomValidity(message);
return message;
}
input.setCustomValidity(message);
return message;
}
document.addEventListener('DOMContentLoaded', function() {
s3tablesNamespaceDeleteModalInstance = new bootstrap.Modal(document.getElementById('deleteS3TablesNamespaceModal'));
File diff suppressed because one or more lines are too long
+1 -1
View File
@@ -49,7 +49,7 @@ type BucketRegistry struct {
metadataCache map[string]*BucketMetaData
metadataCacheLock sync.RWMutex
tableLocationCache map[string]string // Cache for table location mappings (bucket -> table path)
tableLocationCache map[string]string // Cache for table location mappings (external bucket -> mapped root path)
tableLocationLock sync.RWMutex
notFound map[string]struct{}
+80 -2
View File
@@ -1,7 +1,9 @@
package s3api
import (
"context"
"errors"
"io"
"path"
"strings"
@@ -58,8 +60,13 @@ func (s3a *S3ApiServer) tableLocationDir(bucket string) (string, bool) {
entry, err := s3a.getEntry(s3tables.GetTableLocationMappingDir(), bucket)
tablePath := ""
if err == nil && entry != nil && len(entry.Content) > 0 {
tablePath = strings.TrimSpace(string(entry.Content))
if err == nil && entry != nil {
if entry.IsDirectory {
tablePath, err = s3a.readTableLocationMappingFromDirectory(bucket)
} else if len(entry.Content) > 0 {
// Backward compatibility with legacy single-file mappings.
tablePath = normalizeTableLocationMappingPath(string(entry.Content))
}
}
// Only cache definitive results: successful lookup (tablePath set) or definitive not-found (ErrNotFound)
@@ -82,6 +89,77 @@ func (s3a *S3ApiServer) tableLocationDir(bucket string) (string, bool) {
return tablePath, true
}
func (s3a *S3ApiServer) readTableLocationMappingFromDirectory(bucket string) (string, error) {
mappingDir := s3tables.GetTableLocationMappingPath(bucket)
var mappedPath string
conflict := false
err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
stream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
Directory: mappingDir,
Limit: 4294967295, // math.MaxUint32
})
if err != nil {
return err
}
for {
resp, recvErr := stream.Recv()
if recvErr == io.EOF {
return nil
}
if recvErr != nil {
return recvErr
}
if resp == nil || resp.Entry == nil || resp.Entry.IsDirectory || len(resp.Entry.Content) == 0 {
continue
}
candidate := normalizeTableLocationMappingPath(string(resp.Entry.Content))
if candidate == "" {
continue
}
if mappedPath == "" {
mappedPath = candidate
continue
}
if mappedPath != candidate {
conflict = true
return nil
}
}
})
if err != nil {
return "", err
}
if conflict {
glog.V(1).Infof("table location mapping conflict for %s: multiple mapped roots found", bucket)
return "", nil
}
return mappedPath, nil
}
func normalizeTableLocationMappingPath(rawPath string) string {
rawPath = strings.TrimSpace(rawPath)
if rawPath == "" {
return ""
}
normalized := path.Clean("/" + strings.TrimPrefix(rawPath, "/"))
tablesPrefix := strings.TrimSuffix(s3tables.TablesPath, "/") + "/"
if !strings.HasPrefix(normalized, tablesPrefix) {
return normalized
}
remaining := strings.TrimPrefix(normalized, tablesPrefix)
bucketName, _, _ := strings.Cut(remaining, "/")
if bucketName == "" {
return ""
}
return path.Join(s3tables.TablesPath, bucketName)
}
func (s3a *S3ApiServer) bucketRoot(bucket string) string {
// Returns the unified buckets root path for all bucket types
return s3a.option.BucketsPath
+45
View File
@@ -0,0 +1,45 @@
package s3api
import "testing"
func TestNormalizeTableLocationMappingPath(t *testing.T) {
testCases := []struct {
name string
raw string
want string
}{
{
name: "legacy table path maps to bucket root",
raw: "/buckets/warehouse/analytics/orders",
want: "/buckets/warehouse",
},
{
name: "already bucket root",
raw: "/buckets/warehouse",
want: "/buckets/warehouse",
},
{
name: "relative buckets path normalized and reduced",
raw: "buckets/warehouse/analytics/orders",
want: "/buckets/warehouse",
},
{
name: "non buckets path preserved",
raw: "/tmp/custom/path",
want: "/tmp/custom/path",
},
{
name: "empty path",
raw: "",
want: "",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
if got := normalizeTableLocationMappingPath(tc.raw); got != tc.want {
t.Fatalf("normalizeTableLocationMappingPath(%q)=%q, want %q", tc.raw, got, tc.want)
}
})
}
}
+229
View File
@@ -0,0 +1,229 @@
package iceberg
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"path"
"strconv"
"strings"
"github.com/apache/iceberg-go/table"
"github.com/google/uuid"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
const requirementAssertCreate = "assert-create"
type icebergRequestError struct {
status int
errType string
message string
}
func (e *icebergRequestError) Error() string {
return e.message
}
type createOnCommitInput struct {
bucketARN string
markerBucket string
namespace []string
tableName string
identityName string
location string
tableUUID uuid.UUID
baseMetadata table.Metadata
baseMetadataLoc string
baseMetadataVer int
updates table.Updates
statisticsUpdates []statisticsUpdate
}
func isS3TablesConflict(err error) bool {
if err == nil {
return false
}
if errors.Is(err, s3tables.ErrVersionTokenMismatch) {
return true
}
var tableErr *s3tables.S3TablesError
return errors.As(err, &tableErr) && tableErr.Type == s3tables.ErrCodeConflict
}
func isS3TablesNotFound(err error) bool {
if err == nil {
return false
}
if strings.Contains(strings.ToLower(err.Error()), "not found") {
return true
}
var tableErr *s3tables.S3TablesError
return errors.As(err, &tableErr) &&
(tableErr.Type == s3tables.ErrCodeNoSuchTable || tableErr.Type == s3tables.ErrCodeNoSuchNamespace || strings.Contains(strings.ToLower(tableErr.Message), "not found"))
}
func hasAssertCreateRequirement(requirements table.Requirements) bool {
for _, requirement := range requirements {
if requirement.GetType() == requirementAssertCreate {
return true
}
}
return false
}
func isS3TablesAlreadyExists(err error) bool {
if err == nil {
return false
}
if strings.Contains(strings.ToLower(err.Error()), "already exists") {
return true
}
var tableErr *s3tables.S3TablesError
return errors.As(err, &tableErr) &&
(tableErr.Type == s3tables.ErrCodeTableAlreadyExists || tableErr.Type == s3tables.ErrCodeNamespaceAlreadyExists || strings.Contains(strings.ToLower(tableErr.Message), "already exists"))
}
func parseMetadataVersionFromLocation(metadataLocation string) int {
base := path.Base(metadataLocation)
if !strings.HasPrefix(base, "v") || !strings.HasSuffix(base, ".metadata.json") {
return 0
}
rawVersion := strings.TrimPrefix(strings.TrimSuffix(base, ".metadata.json"), "v")
version, err := strconv.Atoi(rawVersion)
if err != nil || version <= 0 {
return 0
}
return version
}
func (s *Server) finalizeCreateOnCommit(ctx context.Context, input createOnCommitInput) (*CommitTableResponse, *icebergRequestError) {
builder, err := table.MetadataBuilderFromBase(input.baseMetadata, input.baseMetadataLoc)
if err != nil {
return nil, &icebergRequestError{
status: http.StatusInternalServerError,
errType: "InternalServerError",
message: "Failed to create metadata builder: " + err.Error(),
}
}
for _, update := range input.updates {
if err := update.Apply(builder); err != nil {
return nil, &icebergRequestError{
status: http.StatusBadRequest,
errType: "BadRequestException",
message: "Failed to apply update: " + err.Error(),
}
}
}
newMetadata, err := builder.Build()
if err != nil {
return nil, &icebergRequestError{
status: http.StatusBadRequest,
errType: "BadRequestException",
message: "Failed to build new metadata: " + err.Error(),
}
}
metadataVersion := input.baseMetadataVer + 1
if metadataVersion <= 0 {
metadataVersion = 1
}
metadataFileName := fmt.Sprintf("v%d.metadata.json", metadataVersion)
newMetadataLocation := fmt.Sprintf("%s/metadata/%s", strings.TrimSuffix(input.location, "/"), metadataFileName)
metadataBytes, err := json.Marshal(newMetadata)
if err != nil {
return nil, &icebergRequestError{
status: http.StatusInternalServerError,
errType: "InternalServerError",
message: "Failed to serialize metadata: " + err.Error(),
}
}
metadataBytes, err = applyStatisticsUpdates(metadataBytes, input.statisticsUpdates)
if err != nil {
return nil, &icebergRequestError{
status: http.StatusBadRequest,
errType: "BadRequestException",
message: "Failed to apply statistics updates: " + err.Error(),
}
}
newMetadata, err = table.ParseMetadataBytes(metadataBytes)
if err != nil {
return nil, &icebergRequestError{
status: http.StatusInternalServerError,
errType: "InternalServerError",
message: "Failed to parse committed metadata: " + err.Error(),
}
}
metadataBucket, metadataPath, err := parseS3Location(input.location)
if err != nil {
return nil, &icebergRequestError{
status: http.StatusInternalServerError,
errType: "InternalServerError",
message: "Invalid table location: " + err.Error(),
}
}
if err := s.saveMetadataFile(ctx, metadataBucket, metadataPath, metadataFileName, metadataBytes); err != nil {
return nil, &icebergRequestError{
status: http.StatusInternalServerError,
errType: "InternalServerError",
message: "Failed to save metadata file: " + err.Error(),
}
}
createReq := &s3tables.CreateTableRequest{
TableBucketARN: input.bucketARN,
Namespace: input.namespace,
Name: input.tableName,
Format: "ICEBERG",
Metadata: &s3tables.TableMetadata{
Iceberg: &s3tables.IcebergMetadata{
TableUUID: input.tableUUID.String(),
},
FullMetadata: metadataBytes,
},
MetadataVersion: metadataVersion,
MetadataLocation: newMetadataLocation,
}
createErr := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(ctx, mgrClient, "CreateTable", createReq, nil, input.identityName)
})
if createErr != nil {
if cleanupErr := s.deleteMetadataFile(ctx, metadataBucket, metadataPath, metadataFileName); cleanupErr != nil {
glog.V(1).Infof("Iceberg: failed to cleanup metadata file %s after create-on-commit failure: %v", newMetadataLocation, cleanupErr)
}
if isS3TablesConflict(createErr) || isS3TablesAlreadyExists(createErr) {
return nil, &icebergRequestError{
status: http.StatusConflict,
errType: "CommitFailedException",
message: "Table was created concurrently",
}
}
glog.Errorf("Iceberg: CommitTable CreateTable error: %v", createErr)
return nil, &icebergRequestError{
status: http.StatusInternalServerError,
errType: "InternalServerError",
message: "Failed to commit table creation: " + createErr.Error(),
}
}
markerBucket := input.markerBucket
if markerBucket == "" {
markerBucket = metadataBucket
}
if markerErr := s.deleteStageCreateMarkers(ctx, markerBucket, input.namespace, input.tableName); markerErr != nil {
glog.V(1).Infof("Iceberg: failed to cleanup stage-create markers for %s.%s after finalize: %v", encodeNamespace(input.namespace), input.tableName, markerErr)
}
return &CommitTableResponse{
MetadataLocation: newMetadataLocation,
Metadata: newMetadata,
}, nil
}
+173
View File
@@ -0,0 +1,173 @@
package iceberg
import (
"encoding/json"
"errors"
"github.com/apache/iceberg-go/table"
)
type statisticsUpdate struct {
set *table.StatisticsFile
remove *int64
}
var ErrIncompleteSetStatistics = errors.New("set-statistics requires snapshot-id, statistics-path, file-size-in-bytes, and file-footer-size-in-bytes")
type commitAction struct {
Action string `json:"action"`
}
type setStatisticsUpdate struct {
Action string `json:"action"`
SnapshotID *int64 `json:"snapshot-id,omitempty"`
StatisticsPath string `json:"statistics-path,omitempty"`
FileSizeInBytes *int64 `json:"file-size-in-bytes,omitempty"`
FileFooterSizeInBytes *int64 `json:"file-footer-size-in-bytes,omitempty"`
KeyMetadata *string `json:"key-metadata,omitempty"`
BlobMetadata []table.BlobMetadata `json:"blob-metadata,omitempty"`
Statistics *table.StatisticsFile `json:"statistics,omitempty"`
}
func (u *setStatisticsUpdate) asStatisticsFile() (*table.StatisticsFile, error) {
if u.Statistics != nil {
if u.Statistics.BlobMetadata == nil {
u.Statistics.BlobMetadata = []table.BlobMetadata{}
}
return u.Statistics, nil
}
if u.SnapshotID == nil || u.StatisticsPath == "" || u.FileSizeInBytes == nil || u.FileFooterSizeInBytes == nil {
return nil, ErrIncompleteSetStatistics
}
stats := &table.StatisticsFile{
SnapshotID: *u.SnapshotID,
StatisticsPath: u.StatisticsPath,
FileSizeInBytes: *u.FileSizeInBytes,
FileFooterSizeInBytes: *u.FileFooterSizeInBytes,
KeyMetadata: u.KeyMetadata,
BlobMetadata: u.BlobMetadata,
}
if stats.BlobMetadata == nil {
stats.BlobMetadata = []table.BlobMetadata{}
}
return stats, nil
}
type removeStatisticsUpdate struct {
Action string `json:"action"`
SnapshotID int64 `json:"snapshot-id"`
}
func parseCommitUpdates(rawUpdates []json.RawMessage) (table.Updates, []statisticsUpdate, error) {
filtered := make([]json.RawMessage, 0, len(rawUpdates))
statisticsUpdates := make([]statisticsUpdate, 0)
for _, raw := range rawUpdates {
var action commitAction
if err := json.Unmarshal(raw, &action); err != nil {
return nil, nil, err
}
switch action.Action {
case "set-statistics":
var setUpdate setStatisticsUpdate
if err := json.Unmarshal(raw, &setUpdate); err != nil {
return nil, nil, err
}
stats, err := setUpdate.asStatisticsFile()
if err != nil {
return nil, nil, err
}
statisticsUpdates = append(statisticsUpdates, statisticsUpdate{set: stats})
case "remove-statistics":
var removeUpdate removeStatisticsUpdate
if err := json.Unmarshal(raw, &removeUpdate); err != nil {
return nil, nil, err
}
snapshotID := removeUpdate.SnapshotID
statisticsUpdates = append(statisticsUpdates, statisticsUpdate{remove: &snapshotID})
default:
filtered = append(filtered, raw)
}
}
if len(filtered) == 0 {
return nil, statisticsUpdates, nil
}
data, err := json.Marshal(filtered)
if err != nil {
return nil, nil, err
}
var updates table.Updates
if err := json.Unmarshal(data, &updates); err != nil {
return nil, nil, err
}
return updates, statisticsUpdates, nil
}
func applyStatisticsUpdates(metadataBytes []byte, updates []statisticsUpdate) ([]byte, error) {
if len(updates) == 0 {
return metadataBytes, nil
}
var metadata map[string]json.RawMessage
if err := json.Unmarshal(metadataBytes, &metadata); err != nil {
return nil, err
}
var statistics []table.StatisticsFile
if rawStatistics, ok := metadata["statistics"]; ok && len(rawStatistics) > 0 {
if err := json.Unmarshal(rawStatistics, &statistics); err != nil {
return nil, err
}
}
statisticsBySnapshot := make(map[int64]table.StatisticsFile, len(statistics))
orderedSnapshotIDs := make([]int64, 0, len(statistics))
inOrder := make(map[int64]bool, len(statistics))
for _, stat := range statistics {
statisticsBySnapshot[stat.SnapshotID] = stat
if !inOrder[stat.SnapshotID] {
orderedSnapshotIDs = append(orderedSnapshotIDs, stat.SnapshotID)
inOrder[stat.SnapshotID] = true
}
}
for _, update := range updates {
if update.set != nil {
statisticsBySnapshot[update.set.SnapshotID] = *update.set
if !inOrder[update.set.SnapshotID] {
orderedSnapshotIDs = append(orderedSnapshotIDs, update.set.SnapshotID)
inOrder[update.set.SnapshotID] = true
}
continue
}
if update.remove != nil {
delete(statisticsBySnapshot, *update.remove)
}
}
statistics = make([]table.StatisticsFile, 0, len(statisticsBySnapshot))
for _, snapshotID := range orderedSnapshotIDs {
stat, ok := statisticsBySnapshot[snapshotID]
if !ok {
continue
}
statistics = append(statistics, stat)
}
if len(statistics) == 0 {
delete(metadata, "statistics")
} else {
data, err := json.Marshal(statistics)
if err != nil {
return nil, err
}
metadata["statistics"] = data
}
return json.Marshal(metadata)
}
+332
View File
@@ -0,0 +1,332 @@
package iceberg
import (
"encoding/json"
"errors"
"fmt"
"math/rand/v2"
"net/http"
"path"
"strings"
"time"
"github.com/apache/iceberg-go/table"
"github.com/google/uuid"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
// handleUpdateTable commits updates to a table.
// Implements the Iceberg REST Catalog commit protocol.
func (s *Server) handleUpdateTable(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
namespace := parseNamespace(vars["namespace"])
tableName := vars["table"]
if len(namespace) == 0 || tableName == "" {
writeError(w, http.StatusBadRequest, "BadRequestException", "Namespace and table name are required")
return
}
bucketName := getBucketFromPrefix(r)
bucketARN := buildTableBucketARN(bucketName)
// Extract identity from context
identityName := s3_constants.GetIdentityNameFromContext(r)
// Parse commit request and keep statistics updates separate because iceberg-go v0.4.0
// does not decode set/remove-statistics update actions yet.
var raw struct {
Identifier *TableIdentifier `json:"identifier,omitempty"`
Requirements json.RawMessage `json:"requirements"`
Updates []json.RawMessage `json:"updates"`
}
if err := json.NewDecoder(r.Body).Decode(&raw); err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", "Invalid request body: "+err.Error())
return
}
var req CommitTableRequest
req.Identifier = raw.Identifier
var statisticsUpdates []statisticsUpdate
if len(raw.Requirements) > 0 {
if err := json.Unmarshal(raw.Requirements, &req.Requirements); err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", "Invalid requirements: "+err.Error())
return
}
}
if len(raw.Updates) > 0 {
var err error
req.Updates, statisticsUpdates, err = parseCommitUpdates(raw.Updates)
if err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", "Invalid updates: "+err.Error())
return
}
}
maxCommitAttempts := 3
generatedLegacyUUID := uuid.New()
stageCreateEnabled := isStageCreateEnabled()
for attempt := 1; attempt <= maxCommitAttempts; attempt++ {
getReq := &s3tables.GetTableRequest{
TableBucketARN: bucketARN,
Namespace: namespace,
Name: tableName,
}
var getResp s3tables.GetTableResponse
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(r.Context(), mgrClient, "GetTable", getReq, &getResp, identityName)
})
if err != nil {
if isS3TablesNotFound(err) {
location := fmt.Sprintf("s3://%s/%s/%s", bucketName, encodeNamespace(namespace), tableName)
tableUUID := generatedLegacyUUID
baseMetadataVersion := 0
baseMetadataLocation := ""
var baseMetadata table.Metadata
var latestMarker *stageCreateMarker
if stageCreateEnabled {
var markerErr error
latestMarker, markerErr = s.loadLatestStageCreateMarker(r.Context(), bucketName, namespace, tableName)
if markerErr != nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to load stage-create marker: "+markerErr.Error())
return
}
}
if latestMarker != nil {
if latestMarker.Location != "" {
location = strings.TrimSuffix(latestMarker.Location, "/")
}
if latestMarker.TableUUID != "" {
if parsedUUID, parseErr := uuid.Parse(latestMarker.TableUUID); parseErr == nil {
tableUUID = parsedUUID
}
}
stagedMetadataLocation := latestMarker.StagedMetadataLocation
if stagedMetadataLocation == "" {
stagedMetadataLocation = fmt.Sprintf("%s/metadata/v1.metadata.json", strings.TrimSuffix(location, "/"))
}
stagedLocation := tableLocationFromMetadataLocation(stagedMetadataLocation)
stagedFileName := path.Base(stagedMetadataLocation)
stagedBucket, stagedPath, parseLocationErr := parseS3Location(stagedLocation)
if parseLocationErr != nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Invalid staged metadata location: "+parseLocationErr.Error())
return
}
stagedMetadataBytes, loadErr := s.loadMetadataFile(r.Context(), stagedBucket, stagedPath, stagedFileName)
if loadErr != nil {
if !errors.Is(loadErr, filer_pb.ErrNotFound) {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to load staged metadata: "+loadErr.Error())
return
}
} else if len(stagedMetadataBytes) > 0 {
stagedMetadata, parseErr := table.ParseMetadataBytes(stagedMetadataBytes)
if parseErr != nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to parse staged metadata: "+parseErr.Error())
return
}
// Staged metadata is only a template for table creation; commit starts from version 1.
baseMetadata = stagedMetadata
baseMetadataLocation = ""
baseMetadataVersion = 0
if stagedMetadata.TableUUID() != uuid.Nil {
tableUUID = stagedMetadata.TableUUID()
}
}
}
hasAssertCreate := hasAssertCreateRequirement(req.Requirements)
hasStagedTemplate := baseMetadata != nil
if !(stageCreateEnabled && (hasAssertCreate || hasStagedTemplate)) {
writeError(w, http.StatusNotFound, "NoSuchTableException", fmt.Sprintf("Table does not exist: %s", tableName))
return
}
for _, requirement := range req.Requirements {
validateAgainst := table.Metadata(nil)
if hasStagedTemplate && requirement.GetType() != requirementAssertCreate {
validateAgainst = baseMetadata
}
if requirementErr := requirement.Validate(validateAgainst); requirementErr != nil {
writeError(w, http.StatusConflict, "CommitFailedException", "Requirement failed: "+requirementErr.Error())
return
}
}
if baseMetadata == nil {
baseMetadata = newTableMetadata(tableUUID, location, nil, nil, nil, nil)
if baseMetadata == nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to build current metadata")
return
}
}
result, reqErr := s.finalizeCreateOnCommit(r.Context(), createOnCommitInput{
bucketARN: bucketARN,
markerBucket: bucketName,
namespace: namespace,
tableName: tableName,
identityName: identityName,
location: location,
tableUUID: tableUUID,
baseMetadata: baseMetadata,
baseMetadataLoc: baseMetadataLocation,
baseMetadataVer: baseMetadataVersion,
updates: req.Updates,
statisticsUpdates: statisticsUpdates,
})
if reqErr != nil {
writeError(w, reqErr.status, reqErr.errType, reqErr.message)
return
}
writeJSON(w, http.StatusOK, result)
return
}
glog.V(1).Infof("Iceberg: CommitTable GetTable error: %v", err)
writeError(w, http.StatusInternalServerError, "InternalServerError", err.Error())
return
}
location := tableLocationFromMetadataLocation(getResp.MetadataLocation)
if location == "" {
location = fmt.Sprintf("s3://%s/%s/%s", bucketName, encodeNamespace(namespace), tableName)
}
tableUUID := uuid.Nil
if getResp.Metadata != nil && getResp.Metadata.Iceberg != nil && getResp.Metadata.Iceberg.TableUUID != "" {
if parsed, parseErr := uuid.Parse(getResp.Metadata.Iceberg.TableUUID); parseErr == nil {
tableUUID = parsed
}
}
if tableUUID == uuid.Nil {
tableUUID = generatedLegacyUUID
}
var currentMetadata table.Metadata
if getResp.Metadata != nil && len(getResp.Metadata.FullMetadata) > 0 {
currentMetadata, err = table.ParseMetadataBytes(getResp.Metadata.FullMetadata)
if err != nil {
glog.Errorf("Iceberg: Failed to parse current metadata for %s: %v", tableName, err)
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to parse current metadata")
return
}
} else {
currentMetadata = newTableMetadata(tableUUID, location, nil, nil, nil, nil)
}
if currentMetadata == nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to build current metadata")
return
}
for _, requirement := range req.Requirements {
if err := requirement.Validate(currentMetadata); err != nil {
writeError(w, http.StatusConflict, "CommitFailedException", "Requirement failed: "+err.Error())
return
}
}
builder, err := table.MetadataBuilderFromBase(currentMetadata, getResp.MetadataLocation)
if err != nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to create metadata builder: "+err.Error())
return
}
for _, update := range req.Updates {
if err := update.Apply(builder); err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", "Failed to apply update: "+err.Error())
return
}
}
newMetadata, err := builder.Build()
if err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", "Failed to build new metadata: "+err.Error())
return
}
metadataVersion := getResp.MetadataVersion + 1
metadataFileName := fmt.Sprintf("v%d.metadata.json", metadataVersion)
newMetadataLocation := fmt.Sprintf("%s/metadata/%s", strings.TrimSuffix(location, "/"), metadataFileName)
metadataBytes, err := json.Marshal(newMetadata)
if err != nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to serialize metadata: "+err.Error())
return
}
// iceberg-go does not currently support set/remove-statistics updates in MetadataBuilder.
// Patch the encoded metadata JSON and parse it back to keep the response object consistent.
metadataBytes, err = applyStatisticsUpdates(metadataBytes, statisticsUpdates)
if err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", "Failed to apply statistics updates: "+err.Error())
return
}
newMetadata, err = table.ParseMetadataBytes(metadataBytes)
if err != nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to parse committed metadata: "+err.Error())
return
}
metadataBucket, metadataPath, err := parseS3Location(location)
if err != nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Invalid table location: "+err.Error())
return
}
if err := s.saveMetadataFile(r.Context(), metadataBucket, metadataPath, metadataFileName, metadataBytes); err != nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to save metadata file: "+err.Error())
return
}
updateReq := &s3tables.UpdateTableRequest{
TableBucketARN: bucketARN,
Namespace: namespace,
Name: tableName,
VersionToken: getResp.VersionToken,
Metadata: &s3tables.TableMetadata{
Iceberg: &s3tables.IcebergMetadata{
TableUUID: tableUUID.String(),
},
FullMetadata: metadataBytes,
},
MetadataVersion: metadataVersion,
MetadataLocation: newMetadataLocation,
}
err = s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(r.Context(), mgrClient, "UpdateTable", updateReq, nil, identityName)
})
if err == nil {
result := CommitTableResponse{
MetadataLocation: newMetadataLocation,
Metadata: newMetadata,
}
writeJSON(w, http.StatusOK, result)
return
}
if isS3TablesConflict(err) {
if cleanupErr := s.deleteMetadataFile(r.Context(), metadataBucket, metadataPath, metadataFileName); cleanupErr != nil {
glog.V(1).Infof("Iceberg: failed to cleanup metadata file %s on conflict: %v", newMetadataLocation, cleanupErr)
}
if attempt < maxCommitAttempts {
glog.V(1).Infof("Iceberg: CommitTable conflict for %s (attempt %d/%d), retrying", tableName, attempt, maxCommitAttempts)
jitter := time.Duration(rand.Int64N(int64(25 * time.Millisecond)))
time.Sleep(time.Duration(50*attempt)*time.Millisecond + jitter)
continue
}
writeError(w, http.StatusConflict, "CommitFailedException", "Version token mismatch")
return
}
if cleanupErr := s.deleteMetadataFile(r.Context(), metadataBucket, metadataPath, metadataFileName); cleanupErr != nil {
glog.V(1).Infof("Iceberg: failed to cleanup metadata file %s after update failure: %v", newMetadataLocation, cleanupErr)
}
glog.Errorf("Iceberg: CommitTable UpdateTable error: %v", err)
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to commit table update: "+err.Error())
return
}
}
+247
View File
@@ -0,0 +1,247 @@
package iceberg
import (
"encoding/json"
"fmt"
"net/http"
"strings"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
// handleConfig returns catalog configuration.
func (s *Server) handleConfig(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
config := CatalogConfig{
Defaults: map[string]string{},
Overrides: map[string]string{},
}
if err := json.NewEncoder(w).Encode(config); err != nil {
glog.Warningf("handleConfig: Failed to encode config: %v", err)
}
}
// handleListNamespaces lists namespaces in a catalog.
func (s *Server) handleListNamespaces(w http.ResponseWriter, r *http.Request) {
bucketName := getBucketFromPrefix(r)
bucketARN := buildTableBucketARN(bucketName)
// Extract identity from context
identityName := s3_constants.GetIdentityNameFromContext(r)
pageToken, pageSize, err := parsePagination(r)
if err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", err.Error())
return
}
// Use S3 Tables manager to list namespaces
var resp s3tables.ListNamespacesResponse
req := &s3tables.ListNamespacesRequest{
TableBucketARN: bucketARN,
ContinuationToken: pageToken,
MaxNamespaces: pageSize,
}
err = s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(r.Context(), mgrClient, "ListNamespaces", req, &resp, identityName)
})
if err != nil {
glog.Infof("Iceberg: ListNamespaces error: %v", err)
writeError(w, http.StatusInternalServerError, "InternalServerError", err.Error())
return
}
// Convert to Iceberg format
namespaces := make([]Namespace, 0, len(resp.Namespaces))
for _, ns := range resp.Namespaces {
namespaces = append(namespaces, Namespace(ns.Namespace))
}
result := ListNamespacesResponse{
NextPageToken: resp.ContinuationToken,
Namespaces: namespaces,
}
writeJSON(w, http.StatusOK, result)
}
// handleCreateNamespace creates a new namespace.
func (s *Server) handleCreateNamespace(w http.ResponseWriter, r *http.Request) {
bucketName := getBucketFromPrefix(r)
bucketARN := buildTableBucketARN(bucketName)
// Extract identity from context
identityName := s3_constants.GetIdentityNameFromContext(r)
var req CreateNamespaceRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", "Invalid request body")
return
}
if len(req.Namespace) == 0 {
writeError(w, http.StatusBadRequest, "BadRequestException", "Namespace is required")
return
}
// Use S3 Tables manager to create namespace
createReq := &s3tables.CreateNamespaceRequest{
TableBucketARN: bucketARN,
Namespace: req.Namespace,
Properties: normalizeNamespaceProperties(req.Properties),
}
var createResp s3tables.CreateNamespaceResponse
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
glog.V(2).Infof("Iceberg: handleCreateNamespace calling Execute with identityName=%s", identityName)
return s.tablesManager.Execute(r.Context(), mgrClient, "CreateNamespace", createReq, &createResp, identityName)
})
if err != nil {
if strings.Contains(err.Error(), "already exists") {
writeError(w, http.StatusConflict, "AlreadyExistsException", err.Error())
return
}
glog.Errorf("Iceberg: CreateNamespace error: %v", err)
writeError(w, http.StatusInternalServerError, "InternalServerError", err.Error())
return
}
result := CreateNamespaceResponse{
Namespace: Namespace(createResp.Namespace),
Properties: normalizeNamespaceProperties(createResp.Properties),
}
writeJSON(w, http.StatusOK, result)
}
// handleGetNamespace gets namespace metadata.
func (s *Server) handleGetNamespace(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
namespace := parseNamespace(vars["namespace"])
if len(namespace) == 0 {
writeError(w, http.StatusBadRequest, "BadRequestException", "Namespace is required")
return
}
bucketName := getBucketFromPrefix(r)
bucketARN := buildTableBucketARN(bucketName)
// Extract identity from context
identityName := s3_constants.GetIdentityNameFromContext(r)
// Use S3 Tables manager to get namespace
getReq := &s3tables.GetNamespaceRequest{
TableBucketARN: bucketARN,
Namespace: namespace,
}
var getResp s3tables.GetNamespaceResponse
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(r.Context(), mgrClient, "GetNamespace", getReq, &getResp, identityName)
})
if err != nil {
if strings.Contains(err.Error(), "not found") {
writeError(w, http.StatusNotFound, "NoSuchNamespaceException", fmt.Sprintf("Namespace does not exist: %v", namespace))
return
}
glog.V(1).Infof("Iceberg: GetNamespace error: %v", err)
writeError(w, http.StatusInternalServerError, "InternalServerError", err.Error())
return
}
result := GetNamespaceResponse{
Namespace: Namespace(getResp.Namespace),
Properties: normalizeNamespaceProperties(getResp.Properties),
}
writeJSON(w, http.StatusOK, result)
}
// handleNamespaceExists checks if a namespace exists.
func (s *Server) handleNamespaceExists(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
namespace := parseNamespace(vars["namespace"])
if len(namespace) == 0 {
w.WriteHeader(http.StatusBadRequest)
return
}
bucketName := getBucketFromPrefix(r)
bucketARN := buildTableBucketARN(bucketName)
// Extract identity from context
identityName := s3_constants.GetIdentityNameFromContext(r)
getReq := &s3tables.GetNamespaceRequest{
TableBucketARN: bucketARN,
Namespace: namespace,
}
var getResp s3tables.GetNamespaceResponse
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(r.Context(), mgrClient, "GetNamespace", getReq, &getResp, identityName)
})
if err != nil {
if strings.Contains(err.Error(), "not found") {
w.WriteHeader(http.StatusNotFound)
return
}
glog.V(1).Infof("Iceberg: NamespaceExists error: %v", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusNoContent)
}
// handleDropNamespace deletes a namespace.
func (s *Server) handleDropNamespace(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
namespace := parseNamespace(vars["namespace"])
if len(namespace) == 0 {
writeError(w, http.StatusBadRequest, "BadRequestException", "Namespace is required")
return
}
bucketName := getBucketFromPrefix(r)
bucketARN := buildTableBucketARN(bucketName)
// Extract identity from context
identityName := s3_constants.GetIdentityNameFromContext(r)
deleteReq := &s3tables.DeleteNamespaceRequest{
TableBucketARN: bucketARN,
Namespace: namespace,
}
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(r.Context(), mgrClient, "DeleteNamespace", deleteReq, nil, identityName)
})
if err != nil {
if strings.Contains(err.Error(), "not found") {
writeError(w, http.StatusNotFound, "NoSuchNamespaceException", fmt.Sprintf("Namespace does not exist: %v", namespace))
return
}
if strings.Contains(err.Error(), "not empty") {
writeError(w, http.StatusConflict, "NamespaceNotEmptyException", "Namespace is not empty")
return
}
glog.V(1).Infof("Iceberg: DropNamespace error: %v", err)
writeError(w, http.StatusInternalServerError, "InternalServerError", err.Error())
return
}
w.WriteHeader(http.StatusNoContent)
}
+487
View File
@@ -0,0 +1,487 @@
package iceberg
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"path"
"strings"
"github.com/apache/iceberg-go"
"github.com/apache/iceberg-go/table"
"github.com/google/uuid"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
var errTableNameRequired = errors.New("table name is required")
func validateCreateTableRequest(req CreateTableRequest) error {
if req.Name == "" {
return errTableNameRequired
}
return nil
}
// handleListTables lists tables in a namespace.
func (s *Server) handleListTables(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
namespace := parseNamespace(vars["namespace"])
if len(namespace) == 0 {
writeError(w, http.StatusBadRequest, "BadRequestException", "Namespace is required")
return
}
bucketName := getBucketFromPrefix(r)
bucketARN := buildTableBucketARN(bucketName)
// Extract identity from context
identityName := s3_constants.GetIdentityNameFromContext(r)
pageToken, pageSize, err := parsePagination(r)
if err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", err.Error())
return
}
listReq := &s3tables.ListTablesRequest{
TableBucketARN: bucketARN,
Namespace: namespace,
ContinuationToken: pageToken,
MaxTables: pageSize,
}
var listResp s3tables.ListTablesResponse
err = s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(r.Context(), mgrClient, "ListTables", listReq, &listResp, identityName)
})
if err != nil {
if strings.Contains(err.Error(), "not found") {
writeError(w, http.StatusNotFound, "NoSuchNamespaceException", fmt.Sprintf("Namespace does not exist: %v", namespace))
return
}
glog.V(1).Infof("Iceberg: ListTables error: %v", err)
writeError(w, http.StatusInternalServerError, "InternalServerError", err.Error())
return
}
// Convert to Iceberg format
identifiers := make([]TableIdentifier, 0, len(listResp.Tables))
for _, t := range listResp.Tables {
identifiers = append(identifiers, TableIdentifier{
Namespace: namespace,
Name: t.Name,
})
}
result := ListTablesResponse{
NextPageToken: listResp.ContinuationToken,
Identifiers: identifiers,
}
writeJSON(w, http.StatusOK, result)
}
// handleCreateTable creates a new table.
func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
namespace := parseNamespace(vars["namespace"])
if len(namespace) == 0 {
writeError(w, http.StatusBadRequest, "BadRequestException", "Namespace is required")
return
}
var req CreateTableRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", "Invalid request body")
return
}
if err := validateCreateTableRequest(req); err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", err.Error())
return
}
if req.StageCreate && !isStageCreateEnabled() {
writeError(w, http.StatusNotImplemented, "NotImplementedException", "stage-create is disabled")
return
}
bucketName := getBucketFromPrefix(r)
bucketARN := buildTableBucketARN(bucketName)
// Extract identity from context
identityName := s3_constants.GetIdentityNameFromContext(r)
// Generate UUID for the new table
tableUUID := uuid.New()
tablePath := path.Join(encodeNamespace(namespace), req.Name)
location := strings.TrimSuffix(req.Location, "/")
if location == "" {
if req.Properties != nil {
if warehouse := strings.TrimSuffix(req.Properties["warehouse"], "/"); warehouse != "" {
location = fmt.Sprintf("%s/%s", warehouse, tablePath)
}
}
if location == "" {
if warehouse := strings.TrimSuffix(os.Getenv("ICEBERG_WAREHOUSE"), "/"); warehouse != "" {
location = fmt.Sprintf("%s/%s", warehouse, tablePath)
}
}
if location == "" {
location = fmt.Sprintf("s3://%s/%s", bucketName, tablePath)
}
} else {
parsedBucket, parsedPath, err := parseS3Location(location)
if err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", "Invalid table location: "+err.Error())
return
}
if parsedPath == "" {
location = fmt.Sprintf("s3://%s/%s", parsedBucket, tablePath)
}
}
// Build proper Iceberg table metadata using iceberg-go types
metadata := newTableMetadata(tableUUID, location, req.Schema, req.PartitionSpec, req.WriteOrder, req.Properties)
if metadata == nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to build table metadata")
return
}
// Serialize metadata to JSON
metadataBytes, err := json.Marshal(metadata)
if err != nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to serialize metadata: "+err.Error())
return
}
tableName := req.Name
metadataFileName := "v1.metadata.json" // Initial version is always 1
metadataLocation := fmt.Sprintf("%s/metadata/%s", location, metadataFileName)
metadataBucket, metadataPath, err := parseS3Location(location)
if err != nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Invalid table location: "+err.Error())
return
}
// Stage-create persists metadata in the internal staged area and skips S3Tables registration.
if req.StageCreate {
stagedTablePath := stageCreateStagedTablePath(namespace, tableName, tableUUID)
if err := s.saveMetadataFile(r.Context(), metadataBucket, stagedTablePath, metadataFileName, metadataBytes); err != nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to save staged metadata file: "+err.Error())
return
}
stagedMetadataLocation := fmt.Sprintf("s3://%s/%s/metadata/%s", metadataBucket, stagedTablePath, metadataFileName)
if markerErr := s.writeStageCreateMarker(r.Context(), bucketName, namespace, tableName, tableUUID, location, stagedMetadataLocation); markerErr != nil {
glog.V(1).Infof("Iceberg: failed to persist stage-create marker for %s.%s: %v", encodeNamespace(namespace), tableName, markerErr)
}
result := LoadTableResult{
MetadataLocation: metadataLocation,
Metadata: metadata,
Config: make(iceberg.Properties),
}
writeJSON(w, http.StatusOK, result)
return
}
if err := s.saveMetadataFile(r.Context(), metadataBucket, metadataPath, metadataFileName, metadataBytes); err != nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to save metadata file: "+err.Error())
return
}
// Use S3 Tables manager to create table
createReq := &s3tables.CreateTableRequest{
TableBucketARN: bucketARN,
Namespace: namespace,
Name: tableName,
Format: "ICEBERG",
Metadata: &s3tables.TableMetadata{
Iceberg: &s3tables.IcebergMetadata{
TableUUID: tableUUID.String(),
},
FullMetadata: metadataBytes,
},
MetadataLocation: metadataLocation,
MetadataVersion: 1,
}
var createResp s3tables.CreateTableResponse
err = s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(r.Context(), mgrClient, "CreateTable", createReq, &createResp, identityName)
})
if err != nil {
if tableErr, ok := err.(*s3tables.S3TablesError); ok && tableErr.Type == s3tables.ErrCodeTableAlreadyExists {
getReq := &s3tables.GetTableRequest{
TableBucketARN: bucketARN,
Namespace: namespace,
Name: tableName,
}
var getResp s3tables.GetTableResponse
getErr := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(r.Context(), mgrClient, "GetTable", getReq, &getResp, identityName)
})
if getErr != nil {
writeError(w, http.StatusConflict, "AlreadyExistsException", err.Error())
return
}
result := buildLoadTableResult(getResp, bucketName, namespace, tableName)
writeJSON(w, http.StatusOK, result)
return
}
if strings.Contains(err.Error(), "already exists") {
getReq := &s3tables.GetTableRequest{
TableBucketARN: bucketARN,
Namespace: namespace,
Name: tableName,
}
var getResp s3tables.GetTableResponse
getErr := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(r.Context(), mgrClient, "GetTable", getReq, &getResp, identityName)
})
if getErr != nil {
writeError(w, http.StatusConflict, "AlreadyExistsException", err.Error())
return
}
result := buildLoadTableResult(getResp, bucketName, namespace, tableName)
writeJSON(w, http.StatusOK, result)
return
}
glog.V(1).Infof("Iceberg: CreateTable error: %v", err)
writeError(w, http.StatusInternalServerError, "InternalServerError", err.Error())
return
}
// Use returned location if available, otherwise fallback to local one
finalLocation := createResp.MetadataLocation
if finalLocation == "" {
finalLocation = metadataLocation
}
if markerErr := s.deleteStageCreateMarkers(r.Context(), bucketName, namespace, tableName); markerErr != nil {
glog.V(1).Infof("Iceberg: failed to cleanup stage-create markers for %s.%s after create: %v", encodeNamespace(namespace), tableName, markerErr)
}
result := LoadTableResult{
MetadataLocation: finalLocation,
Metadata: metadata,
Config: make(iceberg.Properties),
}
writeJSON(w, http.StatusOK, result)
}
// handleLoadTable loads table metadata.
func (s *Server) handleLoadTable(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
namespace := parseNamespace(vars["namespace"])
tableName := vars["table"]
if len(namespace) == 0 || tableName == "" {
writeError(w, http.StatusBadRequest, "BadRequestException", "Namespace and table name are required")
return
}
bucketName := getBucketFromPrefix(r)
bucketARN := buildTableBucketARN(bucketName)
// Extract identity from context
identityName := s3_constants.GetIdentityNameFromContext(r)
getReq := &s3tables.GetTableRequest{
TableBucketARN: bucketARN,
Namespace: namespace,
Name: tableName,
}
var getResp s3tables.GetTableResponse
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(r.Context(), mgrClient, "GetTable", getReq, &getResp, identityName)
})
if err != nil {
if strings.Contains(err.Error(), "not found") {
writeError(w, http.StatusNotFound, "NoSuchTableException", fmt.Sprintf("Table does not exist: %s", tableName))
return
}
glog.V(1).Infof("Iceberg: LoadTable error: %v", err)
writeError(w, http.StatusInternalServerError, "InternalServerError", err.Error())
return
}
result := buildLoadTableResult(getResp, bucketName, namespace, tableName)
writeJSON(w, http.StatusOK, result)
}
func buildLoadTableResult(getResp s3tables.GetTableResponse, bucketName string, namespace []string, tableName string) LoadTableResult {
location := tableLocationFromMetadataLocation(getResp.MetadataLocation)
if location == "" {
location = fmt.Sprintf("s3://%s/%s/%s", bucketName, encodeNamespace(namespace), tableName)
}
tableUUID := uuid.Nil
if getResp.Metadata != nil && getResp.Metadata.Iceberg != nil && getResp.Metadata.Iceberg.TableUUID != "" {
if parsed, err := uuid.Parse(getResp.Metadata.Iceberg.TableUUID); err == nil {
tableUUID = parsed
}
}
// Use Nil UUID if not found in storage (legacy table)
// Stability is guaranteed by not generating random UUIDs on read
var metadata table.Metadata
if getResp.Metadata != nil && len(getResp.Metadata.FullMetadata) > 0 {
var err error
metadata, err = table.ParseMetadataBytes(getResp.Metadata.FullMetadata)
if err != nil {
glog.Warningf("Iceberg: Failed to parse persisted metadata for %s: %v", tableName, err)
// Attempt to reconstruct from IcebergMetadata if available, otherwise synthetic
// TODO: Extract schema/spec from getResp.Metadata.Iceberg if FullMetadata fails but partial info exists?
// For now, fallback to empty metadata
metadata = newTableMetadata(tableUUID, location, nil, nil, nil, nil)
}
} else {
// No full metadata, create synthetic
// TODO: If we had stored schema in IcebergMetadata, we would pass it here
metadata = newTableMetadata(tableUUID, location, nil, nil, nil, nil)
}
return LoadTableResult{
MetadataLocation: getResp.MetadataLocation,
Metadata: metadata,
Config: make(iceberg.Properties),
}
}
// handleTableExists checks if a table exists.
func (s *Server) handleTableExists(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
namespace := parseNamespace(vars["namespace"])
tableName := vars["table"]
if len(namespace) == 0 || tableName == "" {
w.WriteHeader(http.StatusBadRequest)
return
}
bucketName := getBucketFromPrefix(r)
bucketARN := buildTableBucketARN(bucketName)
// Extract identity from context
identityName := s3_constants.GetIdentityNameFromContext(r)
getReq := &s3tables.GetTableRequest{
TableBucketARN: bucketARN,
Namespace: namespace,
Name: tableName,
}
var getResp s3tables.GetTableResponse
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(r.Context(), mgrClient, "GetTable", getReq, &getResp, identityName)
})
if err != nil {
w.WriteHeader(http.StatusNotFound)
return
}
w.WriteHeader(http.StatusNoContent)
}
// handleDropTable deletes a table.
func (s *Server) handleDropTable(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
namespace := parseNamespace(vars["namespace"])
tableName := vars["table"]
if len(namespace) == 0 || tableName == "" {
writeError(w, http.StatusBadRequest, "BadRequestException", "Namespace and table name are required")
return
}
bucketName := getBucketFromPrefix(r)
bucketARN := buildTableBucketARN(bucketName)
// Extract identity from context
identityName := s3_constants.GetIdentityNameFromContext(r)
deleteReq := &s3tables.DeleteTableRequest{
TableBucketARN: bucketARN,
Namespace: namespace,
Name: tableName,
}
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(r.Context(), mgrClient, "DeleteTable", deleteReq, nil, identityName)
})
if err != nil {
if strings.Contains(err.Error(), "not found") {
writeError(w, http.StatusNotFound, "NoSuchTableException", fmt.Sprintf("Table does not exist: %s", tableName))
return
}
glog.V(1).Infof("Iceberg: DropTable error: %v", err)
writeError(w, http.StatusInternalServerError, "InternalServerError", err.Error())
return
}
w.WriteHeader(http.StatusNoContent)
}
// newTableMetadata creates a new table.Metadata object with the given parameters.
// Uses iceberg-go's MetadataBuilder pattern for proper spec compliance.
func newTableMetadata(
tableUUID uuid.UUID,
location string,
schema *iceberg.Schema,
partitionSpec *iceberg.PartitionSpec,
sortOrder *table.SortOrder,
props iceberg.Properties,
) table.Metadata {
// Add schema - use provided or create empty schema
var s *iceberg.Schema
if schema != nil {
s = schema
} else {
s = iceberg.NewSchema(0)
}
// Add partition spec
var pSpec *iceberg.PartitionSpec
if partitionSpec != nil {
pSpec = partitionSpec
} else {
unpartitioned := iceberg.NewPartitionSpecID(0)
pSpec = &unpartitioned
}
// Add sort order
var so table.SortOrder
if sortOrder != nil {
so = *sortOrder
} else {
so = table.UnsortedSortOrder
}
// Create properties map if nil
if props == nil {
props = make(iceberg.Properties)
}
// Create metadata directly using the constructor which ensures spec compliance for V2
metadata, err := table.NewMetadataWithUUID(s, pSpec, so, location, props, tableUUID)
if err != nil {
glog.Errorf("Failed to create metadata: %v", err)
return nil
}
return metadata
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,103 @@
package iceberg
import (
"encoding/json"
"errors"
"testing"
"github.com/apache/iceberg-go/table"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
func TestParseCommitUpdatesSeparatesStatistics(t *testing.T) {
raw := []json.RawMessage{
json.RawMessage(`{"action":"set-statistics","snapshot-id":10,"statistics-path":"s3://bucket/table/metadata/stats.puffin","file-size-in-bytes":100,"file-footer-size-in-bytes":20,"blob-metadata":[]}`),
json.RawMessage(`{"action":"set-properties","updates":{"k":"v"}}`),
}
updates, stats, err := parseCommitUpdates(raw)
if err != nil {
t.Fatalf("parseCommitUpdates() error = %v", err)
}
if len(stats) != 1 {
t.Fatalf("statistics updates = %d, want 1", len(stats))
}
if stats[0].set == nil || stats[0].set.SnapshotID != 10 {
t.Fatalf("unexpected statistics update: %#v", stats[0])
}
if len(updates) != 1 {
t.Fatalf("decoded updates = %d, want 1", len(updates))
}
}
func TestParseCommitUpdatesRejectsIncompleteSetStatistics(t *testing.T) {
raw := []json.RawMessage{
json.RawMessage(`{"action":"set-statistics","snapshot-id":10}`),
}
_, _, err := parseCommitUpdates(raw)
if err == nil {
t.Fatalf("parseCommitUpdates() expected error")
}
if !errors.Is(err, ErrIncompleteSetStatistics) {
t.Fatalf("parseCommitUpdates() error = %v, want ErrIncompleteSetStatistics", err)
}
}
func TestApplyStatisticsUpdatesUpsertAndRemove(t *testing.T) {
metadata := []byte(`{"format-version":2,"statistics":[{"snapshot-id":1,"statistics-path":"s3://bucket/stats-1.puffin","file-size-in-bytes":10,"file-footer-size-in-bytes":1,"blob-metadata":[]},{"snapshot-id":2,"statistics-path":"s3://bucket/stats-2.puffin","file-size-in-bytes":20,"file-footer-size-in-bytes":2,"blob-metadata":[]}]} `)
snapshotID := int64(2)
setUpdates := []statisticsUpdate{
{
set: &statisticsFileForTest,
},
{
remove: &snapshotID,
},
}
updated, err := applyStatisticsUpdates(metadata, setUpdates)
if err != nil {
t.Fatalf("applyStatisticsUpdates() error = %v", err)
}
var decoded map[string]json.RawMessage
if err := json.Unmarshal(updated, &decoded); err != nil {
t.Fatalf("json.Unmarshal(updated) error = %v", err)
}
var stats []map[string]any
if err := json.Unmarshal(decoded["statistics"], &stats); err != nil {
t.Fatalf("json.Unmarshal(statistics) error = %v", err)
}
if len(stats) != 1 {
t.Fatalf("statistics length = %d, want 1", len(stats))
}
if got := int64(stats[0]["snapshot-id"].(float64)); got != 1 {
t.Fatalf("remaining snapshot-id = %d, want 1", got)
}
if got := int64(stats[0]["file-size-in-bytes"].(float64)); got != 11 {
t.Fatalf("remaining file-size-in-bytes = %d, want 11", got)
}
}
var statisticsFileForTest = table.StatisticsFile{
SnapshotID: 1,
StatisticsPath: "s3://bucket/stats-1.puffin",
FileSizeInBytes: 11,
FileFooterSizeInBytes: 2,
BlobMetadata: []table.BlobMetadata{},
}
func TestIsS3TablesConflict(t *testing.T) {
if !isS3TablesConflict(s3tables.ErrVersionTokenMismatch) {
t.Fatalf("expected ErrVersionTokenMismatch to be conflict")
}
if !isS3TablesConflict(&s3tables.S3TablesError{Type: s3tables.ErrCodeConflict, Message: "Version token mismatch"}) {
t.Fatalf("expected S3Tables conflict error to be conflict")
}
if isS3TablesConflict(errors.New("other")) {
t.Fatalf("unexpected conflict for non-conflict error")
}
}
@@ -0,0 +1,37 @@
package iceberg
import (
"errors"
"testing"
)
func TestValidateCreateTableRequestRequiresName(t *testing.T) {
err := validateCreateTableRequest(CreateTableRequest{})
if !errors.Is(err, errTableNameRequired) {
t.Fatalf("validateCreateTableRequest() error = %v, want errTableNameRequired", err)
}
}
func TestValidateCreateTableRequestAcceptsWithName(t *testing.T) {
err := validateCreateTableRequest(CreateTableRequest{Name: "orders"})
if err != nil {
t.Fatalf("validateCreateTableRequest() error = %v, want nil", err)
}
}
func TestIsStageCreateEnabledDefaultsToTrue(t *testing.T) {
t.Setenv("ICEBERG_ENABLE_STAGE_CREATE", "")
if !isStageCreateEnabled() {
t.Fatalf("isStageCreateEnabled() = false, want true")
}
}
func TestIsStageCreateEnabledFalseValues(t *testing.T) {
falseValues := []string{"0", "false", "FALSE", "no", "off"}
for _, value := range falseValues {
t.Setenv("ICEBERG_ENABLE_STAGE_CREATE", value)
if isStageCreateEnabled() {
t.Fatalf("isStageCreateEnabled() = true for value %q, want false", value)
}
}
}
@@ -0,0 +1,29 @@
package iceberg
import "testing"
func TestNormalizeNamespacePropertiesNil(t *testing.T) {
properties := normalizeNamespaceProperties(nil)
if properties == nil {
t.Fatalf("normalizeNamespaceProperties(nil) returned nil map")
}
if len(properties) != 0 {
t.Fatalf("normalizeNamespaceProperties(nil) length = %d, want 0", len(properties))
}
}
func TestNormalizeNamespacePropertiesReturnsInputWhenSet(t *testing.T) {
input := map[string]string{
"owner": "analytics",
}
properties := normalizeNamespaceProperties(input)
if properties["owner"] != "analytics" {
t.Fatalf("normalized properties value = %q, want %q", properties["owner"], "analytics")
}
input["owner"] = "updated"
if properties["owner"] != "updated" {
t.Fatalf("normalizeNamespaceProperties should reuse the input map when non-nil")
}
}
@@ -0,0 +1,69 @@
package iceberg
import (
"net/http/httptest"
"testing"
)
func TestParsePaginationDefaultValues(t *testing.T) {
req := httptest.NewRequest("GET", "/v1/namespaces", nil)
pageToken, pageSize, err := parsePagination(req)
if err != nil {
t.Fatalf("parsePagination() error = %v", err)
}
if pageToken != "" {
t.Fatalf("pageToken = %q, want empty", pageToken)
}
if pageSize != defaultListPageSize {
t.Fatalf("pageSize = %d, want %d", pageSize, defaultListPageSize)
}
}
func TestParsePaginationUsesCamelCaseParameters(t *testing.T) {
req := httptest.NewRequest("GET", "/v1/namespaces?pageToken=abc&pageSize=25", nil)
pageToken, pageSize, err := parsePagination(req)
if err != nil {
t.Fatalf("parsePagination() error = %v", err)
}
if pageToken != "abc" {
t.Fatalf("pageToken = %q, want %q", pageToken, "abc")
}
if pageSize != 25 {
t.Fatalf("pageSize = %d, want %d", pageSize, 25)
}
}
func TestParsePaginationSupportsHyphenatedFallback(t *testing.T) {
req := httptest.NewRequest("GET", "/v1/namespaces?page-token=abc&page-size=17", nil)
pageToken, pageSize, err := parsePagination(req)
if err != nil {
t.Fatalf("parsePagination() error = %v", err)
}
if pageToken != "abc" {
t.Fatalf("pageToken = %q, want %q", pageToken, "abc")
}
if pageSize != 17 {
t.Fatalf("pageSize = %d, want %d", pageSize, 17)
}
}
func TestParsePaginationRejectsInvalidPageSize(t *testing.T) {
testCases := []string{
"/v1/namespaces?pageSize=0",
"/v1/namespaces?pageSize=-1",
"/v1/namespaces?pageSize=foo",
"/v1/namespaces?pageSize=1001",
}
for _, rawURL := range testCases {
t.Run(rawURL, func(t *testing.T) {
req := httptest.NewRequest("GET", rawURL, nil)
if _, _, err := parsePagination(req); err == nil {
t.Fatalf("parsePagination() expected error for url %q", rawURL)
}
})
}
}
@@ -0,0 +1,76 @@
package iceberg
import (
"strings"
"testing"
"github.com/apache/iceberg-go/table"
"github.com/google/uuid"
)
func TestHasAssertCreateRequirement(t *testing.T) {
requirements := table.Requirements{table.AssertCreate()}
if !hasAssertCreateRequirement(requirements) {
t.Fatalf("hasAssertCreateRequirement() = false, want true")
}
requirements = table.Requirements{table.AssertDefaultSortOrderID(0)}
if hasAssertCreateRequirement(requirements) {
t.Fatalf("hasAssertCreateRequirement() = true, want false")
}
}
func TestParseMetadataVersionFromLocation(t *testing.T) {
testCases := []struct {
location string
version int
}{
{location: "s3://b/ns/t/metadata/v1.metadata.json", version: 1},
{location: "s3://b/ns/t/metadata/v25.metadata.json", version: 25},
{location: "v1.metadata.json", version: 1},
{location: "s3://b/ns/t/metadata/v0.metadata.json", version: 0},
{location: "s3://b/ns/t/metadata/v-1.metadata.json", version: 0},
{location: "s3://b/ns/t/metadata/vABC.metadata.json", version: 0},
{location: "s3://b/ns/t/metadata/current.json", version: 0},
{location: "", version: 0},
}
for _, tc := range testCases {
t.Run(tc.location, func(t *testing.T) {
if got := parseMetadataVersionFromLocation(tc.location); got != tc.version {
t.Errorf("parseMetadataVersionFromLocation(%q) = %d, want %d", tc.location, got, tc.version)
}
})
}
}
func TestStageCreateMarkerNamespaceKey(t *testing.T) {
key := stageCreateMarkerNamespaceKey([]string{"a", "b"})
if key == "a\x1fb" {
t.Fatalf("stageCreateMarkerNamespaceKey() returned unescaped namespace key %q", key)
}
if !strings.Contains(key, "%1F") {
t.Fatalf("stageCreateMarkerNamespaceKey() = %q, want escaped unit separator", key)
}
}
func TestStageCreateMarkerDir(t *testing.T) {
dir := stageCreateMarkerDir("warehouse", []string{"ns"}, "orders")
if !strings.Contains(dir, stageCreateMarkerDirName) {
t.Fatalf("stageCreateMarkerDir() = %q, want marker dir segment %q", dir, stageCreateMarkerDirName)
}
if !strings.HasSuffix(dir, "/orders") {
t.Fatalf("stageCreateMarkerDir() = %q, want suffix /orders", dir)
}
}
func TestStageCreateStagedTablePath(t *testing.T) {
tableUUID := uuid.MustParse("11111111-2222-3333-4444-555555555555")
stagedPath := stageCreateStagedTablePath([]string{"ns"}, "orders", tableUUID)
if !strings.Contains(stagedPath, stageCreateMarkerDirName) {
t.Fatalf("stageCreateStagedTablePath() = %q, want marker dir segment %q", stagedPath, stageCreateMarkerDirName)
}
if !strings.HasSuffix(stagedPath, "/"+tableUUID.String()) {
t.Fatalf("stageCreateStagedTablePath() = %q, want UUID suffix %q", stagedPath, tableUUID.String())
}
}
+157
View File
@@ -0,0 +1,157 @@
package iceberg
import (
"context"
"fmt"
"os"
"path"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
// saveMetadataFile saves the Iceberg metadata JSON file to the filer.
// It constructs the filer path from the S3 location components.
func (s *Server) saveMetadataFile(ctx context.Context, bucketName, tablePath, metadataFileName string, content []byte) error {
// Create context with timeout for file operations
opCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
return s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
bucketsPath := s3tables.TablesPath
ensureDir := func(parent, name, errorContext string) error {
_, err := filer_pb.LookupEntry(opCtx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: parent,
Name: name,
})
if err == nil {
return nil
}
if err != filer_pb.ErrNotFound {
return fmt.Errorf("lookup %s failed: %w", errorContext, err)
}
// If lookup fails with ErrNotFound, try to create the directory.
resp, createErr := client.CreateEntry(opCtx, &filer_pb.CreateEntryRequest{
Directory: parent,
Entry: &filer_pb.Entry{
Name: name,
IsDirectory: true,
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Now().Unix(),
Crtime: time.Now().Unix(),
FileMode: uint32(0755 | os.ModeDir),
},
},
})
if createErr != nil {
return fmt.Errorf("failed to create %s: %w", errorContext, createErr)
}
if resp.Error != "" && !strings.Contains(resp.Error, "exist") {
return fmt.Errorf("failed to create %s: %s", errorContext, resp.Error)
}
return nil
}
bucketDir := path.Join(bucketsPath, bucketName)
// 1. Ensure bucket directory exists: <bucketsPath>/<bucket>
if err := ensureDir(bucketsPath, bucketName, "bucket directory"); err != nil {
return err
}
// 2. Ensure table path exists under the bucket directory
tableDir := bucketDir
if tablePath != "" {
segments := strings.Split(tablePath, "/")
for _, segment := range segments {
if segment == "" {
continue
}
if err := ensureDir(tableDir, segment, "table directory"); err != nil {
return err
}
tableDir = path.Join(tableDir, segment)
}
}
// 3. Ensure metadata directory exists: <bucketsPath>/<bucket>/<tablePath>/metadata
metadataDir := path.Join(tableDir, "metadata")
if err := ensureDir(tableDir, "metadata", "metadata directory"); err != nil {
return err
}
// 4. Write the file
resp, err := client.CreateEntry(opCtx, &filer_pb.CreateEntryRequest{
Directory: metadataDir,
Entry: &filer_pb.Entry{
Name: metadataFileName,
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Now().Unix(),
Crtime: time.Now().Unix(),
FileMode: uint32(0644),
FileSize: uint64(len(content)),
},
Content: content,
Extended: map[string][]byte{
"Mime-Type": []byte("application/json"),
},
},
})
if err != nil {
return fmt.Errorf("failed to write metadata file: %w", err)
}
if resp.Error != "" {
return fmt.Errorf("failed to write metadata file: %s", resp.Error)
}
return nil
})
}
func (s *Server) deleteMetadataFile(ctx context.Context, bucketName, tablePath, metadataFileName string) error {
opCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
metadataDir := path.Join(s3tables.TablesPath, bucketName)
if tablePath != "" {
metadataDir = path.Join(metadataDir, tablePath)
}
metadataDir = path.Join(metadataDir, "metadata")
return s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return filer_pb.DoRemove(opCtx, client, metadataDir, metadataFileName, true, false, true, false, nil)
})
}
func (s *Server) loadMetadataFile(ctx context.Context, bucketName, tablePath, metadataFileName string) ([]byte, error) {
opCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
metadataDir := path.Join(s3tables.TablesPath, bucketName)
if tablePath != "" {
metadataDir = path.Join(metadataDir, tablePath)
}
metadataDir = path.Join(metadataDir, "metadata")
var content []byte
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
resp, err := filer_pb.LookupEntry(opCtx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: metadataDir,
Name: metadataFileName,
})
if err != nil {
return err
}
if resp == nil || resp.Entry == nil {
return fmt.Errorf("lookup returned nil entry for %s/%s", metadataDir, metadataFileName)
}
content = append([]byte(nil), resp.Entry.Content...)
return nil
})
if err != nil {
return nil, err
}
return content, nil
}
+160
View File
@@ -0,0 +1,160 @@
package iceberg
import (
"net/http"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
// FilerClient provides access to the filer for storage operations.
type FilerClient interface {
WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error
}
type S3Authenticator interface {
AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode)
}
// Server implements the Iceberg REST Catalog API.
type Server struct {
filerClient FilerClient
tablesManager *s3tables.Manager
prefix string // optional prefix for routes
authenticator S3Authenticator
}
// NewServer creates a new Iceberg REST Catalog server.
func NewServer(filerClient FilerClient, authenticator S3Authenticator) *Server {
manager := s3tables.NewManager()
return &Server{
filerClient: filerClient,
tablesManager: manager,
prefix: "",
authenticator: authenticator,
}
}
// RegisterRoutes registers Iceberg REST API routes on the provided router.
func (s *Server) RegisterRoutes(router *mux.Router) {
// Add middleware to log all requests/responses
router.Use(loggingMiddleware)
// Configuration endpoint - no auth needed for config
router.HandleFunc("/v1/config", s.handleConfig).Methods(http.MethodGet)
// Namespace endpoints - wrapped with Auth middleware
router.HandleFunc("/v1/namespaces", s.Auth(s.handleListNamespaces)).Methods(http.MethodGet)
router.HandleFunc("/v1/namespaces", s.Auth(s.handleCreateNamespace)).Methods(http.MethodPost)
router.HandleFunc("/v1/namespaces/{namespace}", s.Auth(s.handleGetNamespace)).Methods(http.MethodGet)
router.HandleFunc("/v1/namespaces/{namespace}", s.Auth(s.handleNamespaceExists)).Methods(http.MethodHead)
router.HandleFunc("/v1/namespaces/{namespace}", s.Auth(s.handleDropNamespace)).Methods(http.MethodDelete)
// Table endpoints - wrapped with Auth middleware
router.HandleFunc("/v1/namespaces/{namespace}/tables", s.Auth(s.handleListTables)).Methods(http.MethodGet)
router.HandleFunc("/v1/namespaces/{namespace}/tables", s.Auth(s.handleCreateTable)).Methods(http.MethodPost)
router.HandleFunc("/v1/namespaces/{namespace}/tables/{table}", s.Auth(s.handleLoadTable)).Methods(http.MethodGet)
router.HandleFunc("/v1/namespaces/{namespace}/tables/{table}", s.Auth(s.handleTableExists)).Methods(http.MethodHead)
router.HandleFunc("/v1/namespaces/{namespace}/tables/{table}", s.Auth(s.handleDropTable)).Methods(http.MethodDelete)
router.HandleFunc("/v1/namespaces/{namespace}/tables/{table}", s.Auth(s.handleUpdateTable)).Methods(http.MethodPost)
// With prefix support - wrapped with Auth middleware
router.HandleFunc("/v1/{prefix}/namespaces", s.Auth(s.handleListNamespaces)).Methods(http.MethodGet)
router.HandleFunc("/v1/{prefix}/namespaces", s.Auth(s.handleCreateNamespace)).Methods(http.MethodPost)
router.HandleFunc("/v1/{prefix}/namespaces/{namespace}", s.Auth(s.handleGetNamespace)).Methods(http.MethodGet)
router.HandleFunc("/v1/{prefix}/namespaces/{namespace}", s.Auth(s.handleNamespaceExists)).Methods(http.MethodHead)
router.HandleFunc("/v1/{prefix}/namespaces/{namespace}", s.Auth(s.handleDropNamespace)).Methods(http.MethodDelete)
router.HandleFunc("/v1/{prefix}/namespaces/{namespace}/tables", s.Auth(s.handleListTables)).Methods(http.MethodGet)
router.HandleFunc("/v1/{prefix}/namespaces/{namespace}/tables", s.Auth(s.handleCreateTable)).Methods(http.MethodPost)
router.HandleFunc("/v1/{prefix}/namespaces/{namespace}/tables/{table}", s.Auth(s.handleLoadTable)).Methods(http.MethodGet)
router.HandleFunc("/v1/{prefix}/namespaces/{namespace}/tables/{table}", s.Auth(s.handleTableExists)).Methods(http.MethodHead)
router.HandleFunc("/v1/{prefix}/namespaces/{namespace}/tables/{table}", s.Auth(s.handleDropTable)).Methods(http.MethodDelete)
router.HandleFunc("/v1/{prefix}/namespaces/{namespace}/tables/{table}", s.Auth(s.handleUpdateTable)).Methods(http.MethodPost)
// Catch-all for debugging
router.PathPrefix("/").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
glog.V(2).Infof("Catch-all route hit: %s %s", r.Method, r.RequestURI)
writeError(w, http.StatusNotFound, "NotFound", "Path not found")
})
glog.V(2).Infof("Registered Iceberg REST Catalog routes")
}
func loggingMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
glog.V(2).Infof("Iceberg REST request: %s %s from %s", r.Method, r.RequestURI, r.RemoteAddr)
// Log all headers for debugging
glog.V(2).Infof("Iceberg REST headers:")
for name, values := range r.Header {
for _, value := range values {
// Redact sensitive headers
if name == "Authorization" && len(value) > 20 {
glog.V(2).Infof(" %s: %s...%s", name, value[:20], value[len(value)-10:])
} else {
glog.V(2).Infof(" %s: %s", name, value)
}
}
}
// Create a response writer that captures the status code
wrapped := &responseWriter{ResponseWriter: w}
next.ServeHTTP(wrapped, r)
glog.V(2).Infof("Iceberg REST response: %s %s -> %d", r.Method, r.RequestURI, wrapped.statusCode)
})
}
type responseWriter struct {
http.ResponseWriter
statusCode int
}
func (w *responseWriter) WriteHeader(code int) {
w.statusCode = code
w.ResponseWriter.WriteHeader(code)
}
func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if s.authenticator == nil {
writeError(w, http.StatusUnauthorized, "NotAuthorizedException", "Authentication required")
return
}
identityName, identity, errCode := s.authenticator.AuthenticateRequest(r)
if errCode != s3err.ErrNone {
apiErr := s3err.GetAPIError(errCode)
errorType := "RESTException"
switch apiErr.HTTPStatusCode {
case http.StatusForbidden:
errorType = "ForbiddenException"
case http.StatusUnauthorized:
errorType = "NotAuthorizedException"
case http.StatusBadRequest:
errorType = "BadRequestException"
case http.StatusInternalServerError:
errorType = "InternalServerError"
}
writeError(w, apiErr.HTTPStatusCode, errorType, apiErr.Description)
return
}
if identityName != "" || identity != nil {
ctx := r.Context()
if identityName != "" {
ctx = s3_constants.SetIdentityNameInContext(ctx, identityName)
}
if identity != nil {
ctx = s3_constants.SetIdentityInContext(ctx, identity)
}
r = r.WithContext(ctx)
}
handler(w, r)
}
}
+273
View File
@@ -0,0 +1,273 @@
package iceberg
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/url"
"os"
"path"
"strings"
"time"
"github.com/google/uuid"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
const (
stageCreateMarkerDirName = ".iceberg_staged"
// Keep staged markers long enough to avoid deleting in-progress create transactions.
stageCreateMarkerTTL = 72 * time.Hour
)
type stageCreateMarker struct {
TableUUID string `json:"table_uuid"`
Location string `json:"location"`
StagedMetadataLocation string `json:"staged_metadata_location,omitempty"`
CreatedAt string `json:"created_at"`
ExpiresAt string `json:"expires_at"`
}
func stageCreateMarkerNamespaceKey(namespace []string) string {
return url.PathEscape(encodeNamespace(namespace))
}
func stageCreateMarkerDir(bucketName string, namespace []string, tableName string) string {
return path.Join(s3tables.TablesPath, bucketName, stageCreateMarkerDirName, stageCreateMarkerNamespaceKey(namespace), tableName)
}
func stageCreateStagedTablePath(namespace []string, tableName string, tableUUID uuid.UUID) string {
return path.Join(stageCreateMarkerDirName, stageCreateMarkerNamespaceKey(namespace), tableName, tableUUID.String())
}
func (s *Server) pruneExpiredStageCreateMarkers(ctx context.Context, bucketName string, namespace []string, tableName string) error {
opCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
markerDir := stageCreateMarkerDir(bucketName, namespace, tableName)
now := time.Now().UTC()
return s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
stream, err := client.ListEntries(opCtx, &filer_pb.ListEntriesRequest{
Directory: markerDir,
Limit: 1024,
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) || strings.Contains(strings.ToLower(err.Error()), "not found") {
return nil
}
return err
}
for {
resp, recvErr := stream.Recv()
if recvErr == io.EOF {
return nil
}
if recvErr != nil {
return recvErr
}
if resp.Entry == nil || resp.Entry.IsDirectory || len(resp.Entry.Content) == 0 {
continue
}
var marker stageCreateMarker
if err := json.Unmarshal(resp.Entry.Content, &marker); err != nil {
continue
}
expiresAt, err := time.Parse(time.RFC3339Nano, marker.ExpiresAt)
if err != nil || !expiresAt.Before(now) {
continue
}
if err := filer_pb.DoRemove(opCtx, client, markerDir, resp.Entry.Name, true, false, true, false, nil); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
return err
}
}
})
}
func (s *Server) loadLatestStageCreateMarker(ctx context.Context, bucketName string, namespace []string, tableName string) (*stageCreateMarker, error) {
opCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
markerDir := stageCreateMarkerDir(bucketName, namespace, tableName)
now := time.Now().UTC()
var latestMarker *stageCreateMarker
var latestCreatedAt time.Time
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
stream, err := client.ListEntries(opCtx, &filer_pb.ListEntriesRequest{
Directory: markerDir,
Limit: 1024,
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) || strings.Contains(strings.ToLower(err.Error()), "not found") {
return nil
}
return err
}
for {
resp, recvErr := stream.Recv()
if recvErr == io.EOF {
return nil
}
if recvErr != nil {
return recvErr
}
if resp.Entry == nil || resp.Entry.IsDirectory || len(resp.Entry.Content) == 0 {
continue
}
var marker stageCreateMarker
if err := json.Unmarshal(resp.Entry.Content, &marker); err != nil {
continue
}
expiresAt, err := time.Parse(time.RFC3339Nano, marker.ExpiresAt)
if err != nil || !expiresAt.After(now) {
continue
}
createdAt, err := time.Parse(time.RFC3339Nano, marker.CreatedAt)
if err != nil {
createdAt = time.Time{}
}
if latestMarker == nil || createdAt.After(latestCreatedAt) {
candidate := marker
latestMarker = &candidate
latestCreatedAt = createdAt
}
}
})
if err != nil {
return nil, err
}
return latestMarker, nil
}
func (s *Server) writeStageCreateMarker(ctx context.Context, bucketName string, namespace []string, tableName string, tableUUID uuid.UUID, location string, stagedMetadataLocation string) error {
opCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
marker := stageCreateMarker{
TableUUID: tableUUID.String(),
Location: location,
StagedMetadataLocation: stagedMetadataLocation,
CreatedAt: time.Now().UTC().Format(time.RFC3339Nano),
ExpiresAt: time.Now().UTC().Add(stageCreateMarkerTTL).Format(time.RFC3339Nano),
}
content, err := json.Marshal(marker)
if err != nil {
return err
}
if err := s.pruneExpiredStageCreateMarkers(ctx, bucketName, namespace, tableName); err != nil {
return err
}
return s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
ensureDir := func(parent, name, errorContext string) error {
_, lookupErr := filer_pb.LookupEntry(opCtx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: parent,
Name: name,
})
if lookupErr == nil {
return nil
}
if lookupErr != filer_pb.ErrNotFound {
return fmt.Errorf("lookup %s failed: %w", errorContext, lookupErr)
}
resp, createErr := client.CreateEntry(opCtx, &filer_pb.CreateEntryRequest{
Directory: parent,
Entry: &filer_pb.Entry{
Name: name,
IsDirectory: true,
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Now().Unix(),
Crtime: time.Now().Unix(),
FileMode: uint32(0755 | os.ModeDir),
},
},
})
if createErr != nil {
return fmt.Errorf("failed to create %s: %w", errorContext, createErr)
}
if resp.Error != "" && !strings.Contains(resp.Error, "exist") {
return fmt.Errorf("failed to create %s: %s", errorContext, resp.Error)
}
return nil
}
segments := []string{bucketName, stageCreateMarkerDirName, stageCreateMarkerNamespaceKey(namespace), tableName}
currentDir := s3tables.TablesPath
for _, segment := range segments {
if segment == "" {
continue
}
if err := ensureDir(currentDir, segment, "stage-create marker directory"); err != nil {
return err
}
currentDir = path.Join(currentDir, segment)
}
entryName := tableUUID.String() + ".json"
resp, createErr := client.CreateEntry(opCtx, &filer_pb.CreateEntryRequest{
Directory: currentDir,
Entry: &filer_pb.Entry{
Name: entryName,
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Now().Unix(),
Crtime: time.Now().Unix(),
FileMode: uint32(0644),
FileSize: uint64(len(content)),
},
Content: content,
Extended: map[string][]byte{
"Mime-Type": []byte("application/json"),
},
},
})
if createErr != nil {
return createErr
}
if resp.Error != "" {
return errors.New(resp.Error)
}
return nil
})
}
func (s *Server) deleteStageCreateMarkers(ctx context.Context, bucketName string, namespace []string, tableName string) error {
opCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
parentDir := path.Dir(stageCreateMarkerDir(bucketName, namespace, tableName))
targetName := path.Base(stageCreateMarkerDir(bucketName, namespace, tableName))
return s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
err := filer_pb.DoRemove(opCtx, client, parentDir, targetName, true, true, true, false, nil)
if errors.Is(err, filer_pb.ErrNotFound) {
return nil
}
return err
})
}
func isStageCreateEnabled() bool {
value := strings.TrimSpace(strings.ToLower(os.Getenv("ICEBERG_ENABLE_STAGE_CREATE")))
if value == "" {
return true
}
switch value {
case "0", "false", "no", "off":
return false
default:
return true
}
}
+151
View File
@@ -0,0 +1,151 @@
package iceberg
import (
"encoding/json"
"fmt"
"net/http"
"os"
"strconv"
"strings"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
// parseNamespace parses the namespace from path parameter.
// Iceberg uses unit separator (0x1F) for multi-level namespaces.
// Note: mux already decodes URL-encoded path parameters, so we only split by unit separator.
func parseNamespace(encoded string) []string {
if encoded == "" {
return nil
}
parts := strings.Split(encoded, "\x1F")
// Filter empty parts
result := make([]string, 0, len(parts))
for _, p := range parts {
if p != "" {
result = append(result, p)
}
}
return result
}
// encodeNamespace encodes namespace parts for response.
func encodeNamespace(parts []string) string {
return strings.Join(parts, "\x1F")
}
func parseS3Location(location string) (bucketName, tablePath string, err error) {
if !strings.HasPrefix(location, "s3://") {
return "", "", fmt.Errorf("unsupported location: %s", location)
}
trimmed := strings.TrimPrefix(location, "s3://")
trimmed = strings.TrimSuffix(trimmed, "/")
if trimmed == "" {
return "", "", fmt.Errorf("invalid location: %s", location)
}
parts := strings.SplitN(trimmed, "/", 2)
bucketName = parts[0]
if bucketName == "" {
return "", "", fmt.Errorf("invalid location bucket: %s", location)
}
if len(parts) == 2 {
tablePath = parts[1]
}
return bucketName, tablePath, nil
}
func tableLocationFromMetadataLocation(metadataLocation string) string {
trimmed := strings.TrimSuffix(metadataLocation, "/")
if idx := strings.LastIndex(trimmed, "/metadata/"); idx != -1 {
return trimmed[:idx]
}
return trimmed
}
// writeJSON writes a JSON response.
func writeJSON(w http.ResponseWriter, status int, v interface{}) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
if v != nil {
data, err := json.Marshal(v)
if err != nil {
glog.Errorf("Iceberg: failed to encode response: %v", err)
return
}
w.Write(data)
}
}
// writeError writes an Iceberg error response.
func writeError(w http.ResponseWriter, status int, errType, message string) {
resp := ErrorResponse{
Error: ErrorModel{
Message: message,
Type: errType,
Code: status,
},
}
writeJSON(w, status, resp)
}
// getBucketFromPrefix extracts table bucket name from prefix parameter.
// For now, we use the prefix as the table bucket name.
func getBucketFromPrefix(r *http.Request) string {
vars := mux.Vars(r)
if prefix := vars["prefix"]; prefix != "" {
return prefix
}
if bucket := os.Getenv("S3TABLES_DEFAULT_BUCKET"); bucket != "" {
return bucket
}
// Default bucket if no prefix - use "warehouse" for Iceberg
return "warehouse"
}
// buildTableBucketARN builds an ARN for a table bucket.
func buildTableBucketARN(bucketName string) string {
arn, _ := s3tables.BuildBucketARN(s3tables.DefaultRegion, s3_constants.AccountAdminId, bucketName)
return arn
}
const (
defaultListPageSize = 1000
maxListPageSize = 1000
)
func getPaginationQueryParam(r *http.Request, primary, fallback string) string {
if v := strings.TrimSpace(r.URL.Query().Get(primary)); v != "" {
return v
}
return strings.TrimSpace(r.URL.Query().Get(fallback))
}
func parsePagination(r *http.Request) (pageToken string, pageSize int, err error) {
pageToken = getPaginationQueryParam(r, "pageToken", "page-token")
pageSize = defaultListPageSize
pageSizeValue := getPaginationQueryParam(r, "pageSize", "page-size")
if pageSizeValue == "" {
return pageToken, pageSize, nil
}
parsedPageSize, parseErr := strconv.Atoi(pageSizeValue)
if parseErr != nil || parsedPageSize <= 0 {
return pageToken, 0, fmt.Errorf("invalid pageSize %q: must be a positive integer", pageSizeValue)
}
if parsedPageSize > maxListPageSize {
return pageToken, 0, fmt.Errorf("invalid pageSize %q: must be <= %d", pageSizeValue, maxListPageSize)
}
return pageToken, parsedPageSize, nil
}
func normalizeNamespaceProperties(properties map[string]string) map[string]string {
if properties == nil {
return map[string]string{}
}
return properties
}
+24 -19
View File
@@ -351,25 +351,8 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
}
// Adjust nextMarker for CommonPrefixes to include trailing slash (AWS S3 compliance)
if cursor.isTruncated && lastEntryWasCommonPrefix && lastCommonPrefixName != "" {
// For CommonPrefixes, NextMarker should include the trailing slash
if requestDir != "" {
if prefix != "" {
nextMarker = requestDir + "/" + prefix + "/" + lastCommonPrefixName + "/"
} else {
nextMarker = requestDir + "/" + lastCommonPrefixName + "/"
}
} else {
nextMarker = lastCommonPrefixName + "/"
}
} else if cursor.isTruncated {
if requestDir != "" {
if prefix != "" {
nextMarker = requestDir + "/" + prefix + "/" + nextMarker
} else {
nextMarker = requestDir + "/" + nextMarker
}
}
if cursor.isTruncated {
nextMarker = buildTruncatedNextMarker(requestDir, prefix, nextMarker, lastEntryWasCommonPrefix, lastCommonPrefixName)
}
if cursor.isTruncated {
@@ -480,6 +463,28 @@ func toParentAndDescendants(dirAndName string) (dir, name string) {
return
}
func buildTruncatedNextMarker(requestDir, prefix, nextMarker string, lastEntryWasCommonPrefix bool, lastCommonPrefixName string) string {
if lastEntryWasCommonPrefix && lastCommonPrefixName != "" {
// For CommonPrefixes, NextMarker should include the trailing slash
if requestDir != "" {
if prefix != "" {
return requestDir + "/" + prefix + "/" + lastCommonPrefixName + "/"
}
return requestDir + "/" + lastCommonPrefixName + "/"
}
if prefix != "" {
return prefix + "/" + lastCommonPrefixName + "/"
}
return lastCommonPrefixName + "/"
}
if requestDir != "" {
return requestDir + "/" + nextMarker
}
return nextMarker
}
func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, cursor *ListingCursor, marker, delimiter string, inclusiveStartFrom bool, bucket string, eachEntryFn func(dir string, entry *filer_pb.Entry)) (nextMarker string, err error) {
// invariants
// prefix and marker should be under dir, marker may contain "/"
@@ -148,6 +148,26 @@ func Test_normalizePrefixMarker(t *testing.T) {
}
}
func TestBuildTruncatedNextMarker(t *testing.T) {
t.Run("does not duplicate prefix segment in next continuation token", func(t *testing.T) {
prefix := "export_2026-02-10_17-00-23"
nextMarker := "export_2026-02-10_17-00-23/4156000e.jpg"
actual := buildTruncatedNextMarker("xemu", prefix, nextMarker, false, "")
assert.Equal(t, "xemu/export_2026-02-10_17-00-23/4156000e.jpg", actual)
})
t.Run("keeps common prefix marker trailing slash", func(t *testing.T) {
actual := buildTruncatedNextMarker("xemu", "export_2026-02-10_17-00-23", "", true, "nested")
assert.Equal(t, "xemu/export_2026-02-10_17-00-23/nested/", actual)
})
t.Run("includes prefix for common prefix marker when request dir is empty", func(t *testing.T) {
actual := buildTruncatedNextMarker("", "foo", "", true, "bar")
assert.Equal(t, "foo/bar/", actual)
})
}
func TestAllowUnorderedParameterValidation(t *testing.T) {
// Test getListObjectsV1Args with allow-unordered parameter
t.Run("getListObjectsV1Args with allow-unordered", func(t *testing.T) {
+41 -64
View File
@@ -226,16 +226,27 @@ func parseOptionalIntParam(r *http.Request, name string) (int, error) {
return parsed, nil
}
func parseOptionalNamespace(r *http.Request, name string) []string {
func parseOptionalNamespace(r *http.Request, name string) ([]string, error) {
value := r.URL.Query().Get(name)
if value == "" {
return nil
return nil, nil
}
if _, err := s3tables.ValidateNamespace([]string{value}); err != nil {
glog.V(1).Infof("invalid namespace value for %s: %q: %v", name, value, err)
return nil
parts, err := s3tables.ParseNamespace(value)
if err != nil {
return nil, fmt.Errorf("invalid %s: %w", name, err)
}
return []string{value}
return parts, nil
}
func parseRequiredNamespacePathParam(r *http.Request, name string) ([]string, error) {
value, err := getDecodedPathParam(r, name)
if err != nil {
return nil, err
}
if value == "" {
return nil, fmt.Errorf("%s is required", name)
}
return s3tables.ParseNamespace(value)
}
// parseTagKeys handles tag key parsing from query parameters.
@@ -352,19 +363,13 @@ func buildGetNamespaceRequest(r *http.Request) (interface{}, error) {
if err != nil {
return nil, err
}
namespace, err := getDecodedPathParam(r, "namespace")
namespace, err := parseRequiredNamespacePathParam(r, "namespace")
if err != nil {
return nil, err
}
if namespace == "" {
return nil, fmt.Errorf("namespace is required")
}
if _, err := s3tables.ValidateNamespace([]string{namespace}); err != nil {
return nil, err
}
return &s3tables.GetNamespaceRequest{
TableBucketARN: tableBucketARN,
Namespace: []string{namespace},
Namespace: namespace,
}, nil
}
@@ -373,19 +378,13 @@ func buildDeleteNamespaceRequest(r *http.Request) (interface{}, error) {
if err != nil {
return nil, err
}
namespace, err := getDecodedPathParam(r, "namespace")
namespace, err := parseRequiredNamespacePathParam(r, "namespace")
if err != nil {
return nil, err
}
if namespace == "" {
return nil, fmt.Errorf("namespace is required")
}
if _, err := s3tables.ValidateNamespace([]string{namespace}); err != nil {
return nil, err
}
return &s3tables.DeleteNamespaceRequest{
TableBucketARN: tableBucketARN,
Namespace: []string{namespace},
Namespace: namespace,
}, nil
}
@@ -398,18 +397,12 @@ func buildCreateTableRequest(r *http.Request) (interface{}, error) {
if err != nil {
return nil, err
}
namespace, err := getDecodedPathParam(r, "namespace")
namespace, err := parseRequiredNamespacePathParam(r, "namespace")
if err != nil {
return nil, err
}
if namespace == "" {
return nil, fmt.Errorf("namespace is required")
}
if _, err := s3tables.ValidateNamespace([]string{namespace}); err != nil {
return nil, err
}
req.TableBucketARN = tableBucketARN
req.Namespace = []string{namespace}
req.Namespace = namespace
return &req, nil
}
@@ -418,13 +411,17 @@ func buildListTablesRequest(r *http.Request) (interface{}, error) {
if err != nil {
return nil, err
}
namespace, err := parseOptionalNamespace(r, "namespace")
if err != nil {
return nil, err
}
maxTables, err := parseOptionalIntParam(r, "maxTables")
if err != nil {
return nil, err
}
return &s3tables.ListTablesRequest{
TableBucketARN: tableBucketARN,
Namespace: parseOptionalNamespace(r, "namespace"),
Namespace: namespace,
Prefix: r.URL.Query().Get("prefix"),
ContinuationToken: r.URL.Query().Get("continuationToken"),
MaxTables: maxTables,
@@ -439,7 +436,11 @@ func buildGetTableRequest(r *http.Request) (interface{}, error) {
}
if tableARN == "" {
req.TableBucketARN = query.Get("tableBucketARN")
req.Namespace = parseOptionalNamespace(r, "namespace")
namespace, err := parseOptionalNamespace(r, "namespace")
if err != nil {
return nil, err
}
req.Namespace = namespace
req.Name = query.Get("name")
if req.TableBucketARN == "" || len(req.Namespace) == 0 || req.Name == "" {
return nil, fmt.Errorf("either tableArn or (tableBucketARN, namespace, name) must be provided")
@@ -453,16 +454,10 @@ func buildDeleteTableRequest(r *http.Request) (interface{}, error) {
if err != nil {
return nil, err
}
namespace, err := getDecodedPathParam(r, "namespace")
namespace, err := parseRequiredNamespacePathParam(r, "namespace")
if err != nil {
return nil, err
}
if namespace == "" {
return nil, fmt.Errorf("namespace is required")
}
if _, err := s3tables.ValidateNamespace([]string{namespace}); err != nil {
return nil, err
}
name, err := getDecodedPathParam(r, "name")
if err != nil {
return nil, err
@@ -475,7 +470,7 @@ func buildDeleteTableRequest(r *http.Request) (interface{}, error) {
}
return &s3tables.DeleteTableRequest{
TableBucketARN: tableBucketARN,
Namespace: []string{namespace},
Namespace: namespace,
Name: name,
VersionToken: r.URL.Query().Get("versionToken"),
}, nil
@@ -490,16 +485,10 @@ func buildPutTablePolicyRequest(r *http.Request) (interface{}, error) {
if err != nil {
return nil, err
}
namespace, err := getDecodedPathParam(r, "namespace")
namespace, err := parseRequiredNamespacePathParam(r, "namespace")
if err != nil {
return nil, err
}
if namespace == "" {
return nil, fmt.Errorf("namespace is required")
}
if _, err := s3tables.ValidateNamespace([]string{namespace}); err != nil {
return nil, err
}
name, err := getDecodedPathParam(r, "name")
if err != nil {
return nil, err
@@ -511,7 +500,7 @@ func buildPutTablePolicyRequest(r *http.Request) (interface{}, error) {
return nil, err
}
req.TableBucketARN = tableBucketARN
req.Namespace = []string{namespace}
req.Namespace = namespace
req.Name = name
return &req, nil
}
@@ -521,16 +510,10 @@ func buildGetTablePolicyRequest(r *http.Request) (interface{}, error) {
if err != nil {
return nil, err
}
namespace, err := getDecodedPathParam(r, "namespace")
namespace, err := parseRequiredNamespacePathParam(r, "namespace")
if err != nil {
return nil, err
}
if namespace == "" {
return nil, fmt.Errorf("namespace is required")
}
if _, err := s3tables.ValidateNamespace([]string{namespace}); err != nil {
return nil, err
}
name, err := getDecodedPathParam(r, "name")
if err != nil {
return nil, err
@@ -543,7 +526,7 @@ func buildGetTablePolicyRequest(r *http.Request) (interface{}, error) {
}
return &s3tables.GetTablePolicyRequest{
TableBucketARN: tableBucketARN,
Namespace: []string{namespace},
Namespace: namespace,
Name: name,
}, nil
}
@@ -553,16 +536,10 @@ func buildDeleteTablePolicyRequest(r *http.Request) (interface{}, error) {
if err != nil {
return nil, err
}
namespace, err := getDecodedPathParam(r, "namespace")
namespace, err := parseRequiredNamespacePathParam(r, "namespace")
if err != nil {
return nil, err
}
if namespace == "" {
return nil, fmt.Errorf("namespace is required")
}
if _, err := s3tables.ValidateNamespace([]string{namespace}); err != nil {
return nil, err
}
name, err := getDecodedPathParam(r, "name")
if err != nil {
return nil, err
@@ -575,7 +552,7 @@ func buildDeleteTablePolicyRequest(r *http.Request) (interface{}, error) {
}
return &s3tables.DeleteTablePolicyRequest{
TableBucketARN: tableBucketARN,
Namespace: []string{namespace},
Namespace: namespace,
Name: name,
}, nil
}
@@ -0,0 +1,104 @@
package s3api
import (
"encoding/json"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
const testTableBucketARN = "arn:aws:s3tables:us-east-1:123456789012:bucket/test-bucket"
func TestBuildListTablesRequestRejectsInvalidNamespaceQuery(t *testing.T) {
testCases := []struct {
name string
namespace string
}{
{name: "uppercase", namespace: "InvalidNamespace"},
{name: "hyphen", namespace: "invalid-ns"},
{name: "slash", namespace: "a/b"},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "/tables?namespace="+url.QueryEscape(tc.namespace), nil)
req = mux.SetURLVars(req, map[string]string{"tableBucketARN": testTableBucketARN})
_, err := buildListTablesRequest(req)
if err == nil {
t.Fatalf("expected invalid namespace query to return an error")
}
if !strings.Contains(err.Error(), "invalid namespace") {
t.Fatalf("expected invalid namespace error, got %q", err.Error())
}
})
}
}
func TestBuildGetTableRequestRejectsInvalidNamespaceQuery(t *testing.T) {
testCases := []struct {
name string
namespace string
}{
{name: "uppercase", namespace: "InvalidNamespace"},
{name: "hyphen", namespace: "invalid-ns"},
{name: "slash", namespace: "a/b"},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "/get-table?tableBucketARN="+url.QueryEscape(testTableBucketARN)+"&namespace="+url.QueryEscape(tc.namespace)+"&name=table1", nil)
_, err := buildGetTableRequest(req)
if err == nil {
t.Fatalf("expected invalid namespace query to return an error")
}
if !strings.Contains(err.Error(), "invalid namespace") {
t.Fatalf("expected invalid namespace error, got %q", err.Error())
}
})
}
}
func TestHandleRestOperationReturnsBadRequestForInvalidNamespaceQuery(t *testing.T) {
testCases := []struct {
name string
namespace string
}{
{name: "uppercase", namespace: "InvalidNamespace"},
{name: "hyphen", namespace: "invalid-ns"},
{name: "slash", namespace: "a/b"},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "/tables?namespace="+url.QueryEscape(tc.namespace), nil)
req = mux.SetURLVars(req, map[string]string{"tableBucketARN": testTableBucketARN})
rr := httptest.NewRecorder()
st := &S3TablesApiServer{}
st.handleRestOperation("ListTables", buildListTablesRequest).ServeHTTP(rr, req)
if rr.Code != http.StatusBadRequest {
t.Fatalf("expected status %d, got %d", http.StatusBadRequest, rr.Code)
}
var body map[string]string
if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
t.Fatalf("failed to decode error response: %v", err)
}
if got, want := body["__type"], s3tables.ErrCodeInvalidRequest; got != want {
t.Fatalf("expected __type=%q, got %q", want, got)
}
if !strings.Contains(body["message"], "invalid namespace") {
t.Fatalf("expected invalid namespace error message, got %q", body["message"])
}
})
}
}
+3
View File
@@ -147,6 +147,7 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R
Namespace: req.Namespace,
CreatedAt: now,
OwnerAccountID: bucketMetadata.OwnerAccountID,
Properties: req.Properties,
}
metadataBytes, err := json.Marshal(metadata)
@@ -177,6 +178,7 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R
resp := &CreateNamespaceResponse{
Namespace: req.Namespace,
TableBucketARN: req.TableBucketARN,
Properties: req.Properties,
}
h.writeJSON(w, http.StatusOK, resp)
@@ -265,6 +267,7 @@ func (h *S3TablesHandler) handleGetNamespace(w http.ResponseWriter, r *http.Requ
Namespace: metadata.Namespace,
CreatedAt: metadata.CreatedAt,
OwnerAccountID: metadata.OwnerAccountID,
Properties: metadata.Properties,
}
h.writeJSON(w, http.StatusOK, resp)
+78 -9
View File
@@ -408,7 +408,7 @@ func (h *S3TablesHandler) handleGetTable(w http.ResponseWriter, r *http.Request,
resp := &GetTableResponse{
Name: metadata.Name,
TableARN: tableARN,
Namespace: []string{metadata.Namespace},
Namespace: expandNamespace(metadata.Namespace),
Format: metadata.Format,
CreatedAt: metadata.CreatedAt,
ModifiedAt: metadata.ModifiedAt,
@@ -683,7 +683,7 @@ func (h *S3TablesHandler) listTablesWithClient(r *http.Request, client filer_pb.
tables = append(tables, TableSummary{
Name: entry.Entry.Name,
TableARN: tableARN,
Namespace: []string{namespaceName},
Namespace: expandNamespace(namespaceName),
CreatedAt: metadata.CreatedAt,
ModifiedAt: metadata.ModifiedAt,
})
@@ -929,7 +929,7 @@ func (h *S3TablesHandler) handleDeleteTable(w http.ResponseWriter, r *http.Reque
if err := h.deleteDirectory(r.Context(), client, tablePath); err != nil {
return err
}
if err := h.deleteTableLocationMapping(r.Context(), client, metadata.MetadataLocation); err != nil {
if err := h.deleteTableLocationMapping(r.Context(), client, metadata.MetadataLocation, tablePath); err != nil {
glog.V(1).Infof("failed to delete table location mapping for %s: %v", metadata.MetadataLocation, err)
}
return nil
@@ -1139,29 +1139,98 @@ func (h *S3TablesHandler) updateTableLocationMapping(ctx context.Context, client
if !ok {
return nil
}
tableBucketPath, ok := tableBucketPathFromTablePath(tablePath)
if !ok {
return fmt.Errorf("invalid table path for location mapping: %s", tablePath)
}
if err := h.ensureDirectory(ctx, client, GetTableLocationMappingDir()); err != nil {
return err
}
if err := h.ensureTableLocationMappingBucketDir(ctx, client, newTableLocationBucket); err != nil {
return err
}
// If the metadata location changed, delete the stale mapping for the old bucket
// If the metadata location changed, remove this table's stale mapping entry from the old bucket.
if oldMetadataLocation != "" && oldMetadataLocation != newMetadataLocation {
oldTableLocationBucket, ok := parseTableLocationBucket(oldMetadataLocation)
if ok && oldTableLocationBucket != newTableLocationBucket {
oldMappingPath := GetTableLocationMappingPath(oldTableLocationBucket)
if err := h.deleteEntryIfExists(ctx, client, oldMappingPath); err != nil {
if err := h.removeTableLocationMappingEntry(ctx, client, oldTableLocationBucket, tablePath); err != nil {
glog.V(1).Infof("failed to delete stale mapping for %s: %v", oldTableLocationBucket, err)
}
}
}
return h.upsertFile(ctx, client, GetTableLocationMappingPath(newTableLocationBucket), []byte(tablePath))
return h.upsertFile(ctx, client, GetTableLocationMappingEntryPath(newTableLocationBucket, tablePath), []byte(tableBucketPath))
}
func (h *S3TablesHandler) deleteTableLocationMapping(ctx context.Context, client filer_pb.SeaweedFilerClient, metadataLocation string) error {
func (h *S3TablesHandler) deleteTableLocationMapping(ctx context.Context, client filer_pb.SeaweedFilerClient, metadataLocation, tablePath string) error {
tableLocationBucket, ok := parseTableLocationBucket(metadataLocation)
if !ok {
return nil
}
return h.deleteEntryIfExists(ctx, client, GetTableLocationMappingPath(tableLocationBucket))
return h.removeTableLocationMappingEntry(ctx, client, tableLocationBucket, tablePath)
}
func (h *S3TablesHandler) ensureTableLocationMappingBucketDir(ctx context.Context, client filer_pb.SeaweedFilerClient, tableLocationBucket string) error {
mappingDir := GetTableLocationMappingDir()
bucketMappingPath := GetTableLocationMappingPath(tableLocationBucket)
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: mappingDir,
Name: tableLocationBucket,
})
if err == nil {
if resp != nil && resp.Entry != nil && resp.Entry.IsDirectory {
return nil
}
if removeErr := h.deleteEntryIfExists(ctx, client, bucketMappingPath); removeErr != nil && !errors.Is(removeErr, filer_pb.ErrNotFound) {
return removeErr
}
} else if !errors.Is(err, filer_pb.ErrNotFound) {
return err
}
return h.ensureDirectory(ctx, client, bucketMappingPath)
}
func (h *S3TablesHandler) removeTableLocationMappingEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, tableLocationBucket, tablePath string) error {
entryPath := GetTableLocationMappingEntryPath(tableLocationBucket, tablePath)
if err := h.deleteEntryIfExists(ctx, client, entryPath); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
return err
}
return h.removeTableLocationMappingBucketDirIfEmpty(ctx, client, tableLocationBucket)
}
func (h *S3TablesHandler) removeTableLocationMappingBucketDirIfEmpty(ctx context.Context, client filer_pb.SeaweedFilerClient, tableLocationBucket string) error {
bucketMappingPath := GetTableLocationMappingPath(tableLocationBucket)
stream, err := client.ListEntries(ctx, &filer_pb.ListEntriesRequest{
Directory: bucketMappingPath,
Limit: 1,
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
return nil
}
return err
}
for {
resp, recvErr := stream.Recv()
if recvErr == io.EOF {
break
}
if recvErr != nil {
return recvErr
}
if resp != nil && resp.Entry != nil {
return nil
}
}
if err := h.deleteEntryIfExists(ctx, client, bucketMappingPath); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
return err
}
return nil
}
@@ -0,0 +1,77 @@
package s3tables
import (
"strings"
"testing"
)
func TestGetTableLocationMappingEntryPathPerTable(t *testing.T) {
tableLocationBucket := "shared-location--table-s3"
tablePathA := GetTablePath("warehouse", "analytics", "orders")
tablePathB := GetTablePath("warehouse", "analytics", "customers")
entryPathA := GetTableLocationMappingEntryPath(tableLocationBucket, tablePathA)
entryPathARepeat := GetTableLocationMappingEntryPath(tableLocationBucket, tablePathA)
entryPathB := GetTableLocationMappingEntryPath(tableLocationBucket, tablePathB)
if entryPathA != entryPathARepeat {
t.Fatalf("mapping entry path should be deterministic: %q != %q", entryPathA, entryPathARepeat)
}
if entryPathA == entryPathB {
t.Fatalf("mapping entry path should differ per table path: %q == %q", entryPathA, entryPathB)
}
expectedPrefix := GetTableLocationMappingPath(tableLocationBucket) + "/"
if !strings.HasPrefix(entryPathA, expectedPrefix) {
t.Fatalf("mapping entry path %q should start with %q", entryPathA, expectedPrefix)
}
if strings.TrimPrefix(entryPathA, expectedPrefix) == "" {
t.Fatalf("mapping entry name should not be empty: %q", entryPathA)
}
}
func TestTableBucketPathFromTablePath(t *testing.T) {
testCases := []struct {
name string
tablePath string
expected string
ok bool
}{
{
name: "valid table path",
tablePath: GetTablePath("warehouse", "analytics", "orders"),
expected: GetTableBucketPath("warehouse"),
ok: true,
},
{
name: "valid table bucket root",
tablePath: GetTableBucketPath("warehouse"),
expected: GetTableBucketPath("warehouse"),
ok: true,
},
{
name: "invalid non-tables path",
tablePath: "/tmp/warehouse/analytics/orders",
expected: "",
ok: false,
},
{
name: "invalid empty bucket segment",
tablePath: "/buckets/",
expected: "",
ok: false,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
actual, ok := tableBucketPathFromTablePath(tc.tablePath)
if ok != tc.ok {
t.Fatalf("tableBucketPathFromTablePath(%q) ok=%v, want %v", tc.tablePath, ok, tc.ok)
}
if actual != tc.expected {
t.Fatalf("tableBucketPathFromTablePath(%q)=%q, want %q", tc.tablePath, actual, tc.expected)
}
})
}
}
+14 -10
View File
@@ -77,19 +77,22 @@ type DeleteTableBucketPolicyRequest struct {
// Namespace types
type Namespace struct {
Namespace []string `json:"namespace"`
CreatedAt time.Time `json:"createdAt"`
OwnerAccountID string `json:"ownerAccountId"`
Namespace []string `json:"namespace"`
CreatedAt time.Time `json:"createdAt"`
OwnerAccountID string `json:"ownerAccountId"`
Properties map[string]string `json:"properties,omitempty"`
}
type CreateNamespaceRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
Properties map[string]string `json:"properties,omitempty"`
}
type CreateNamespaceResponse struct {
Namespace []string `json:"namespace"`
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
TableBucketARN string `json:"tableBucketARN"`
Properties map[string]string `json:"properties,omitempty"`
}
type GetNamespaceRequest struct {
@@ -98,9 +101,10 @@ type GetNamespaceRequest struct {
}
type GetNamespaceResponse struct {
Namespace []string `json:"namespace"`
CreatedAt time.Time `json:"createdAt"`
OwnerAccountID string `json:"ownerAccountId"`
Namespace []string `json:"namespace"`
CreatedAt time.Time `json:"createdAt"`
OwnerAccountID string `json:"ownerAccountId"`
Properties map[string]string `json:"properties,omitempty"`
}
type ListNamespacesRequest struct {
+90 -25
View File
@@ -2,6 +2,7 @@ package s3tables
import (
"crypto/rand"
"crypto/sha1"
"encoding/hex"
"fmt"
"net/url"
@@ -15,7 +16,7 @@ import (
const (
bucketNamePatternStr = `[a-z0-9-]+`
tableNamespacePatternStr = `[a-z0-9_]+`
tableNamespacePatternStr = `[a-z0-9_.]+`
tableNamePatternStr = `[a-z0-9_]+`
)
@@ -54,7 +55,6 @@ func ParseBucketNameFromARN(arn string) (string, error) {
// parseTableFromARN extracts bucket name, namespace, and table name from ARN
// ARN format: arn:aws:s3tables:{region}:{account}:bucket/{bucket-name}/table/{namespace}/{table-name}
func parseTableFromARN(arn string) (bucketName, namespace, tableName string, err error) {
// Updated regex to align with namespace validation (single-segment)
matches := tableARNPattern.FindStringSubmatch(arn)
if len(matches) != 4 {
return "", "", "", fmt.Errorf("invalid table ARN: %s", arn)
@@ -66,9 +66,7 @@ func parseTableFromARN(arn string) (bucketName, namespace, tableName string, err
return "", "", "", fmt.Errorf("invalid bucket name in ARN: %v", err)
}
// Namespace is already constrained by the regex; validate it directly.
namespace = matches[2]
_, err = validateNamespace([]string{namespace})
namespace, err = validateNamespace([]string{matches[2]})
if err != nil {
return "", "", "", fmt.Errorf("invalid namespace in ARN: %v", err)
}
@@ -121,6 +119,33 @@ func GetTableLocationMappingPath(tableLocationBucket string) string {
return path.Join(GetTableLocationMappingDir(), tableLocationBucket)
}
// GetTableLocationMappingEntryPath returns the filer path for a table-specific mapping entry.
// Each table gets its own entry so multiple tables can share the same external table-location bucket.
func GetTableLocationMappingEntryPath(tableLocationBucket, tablePath string) string {
return path.Join(GetTableLocationMappingPath(tableLocationBucket), tableLocationMappingEntryName(tablePath))
}
func tableLocationMappingEntryName(tablePath string) string {
normalized := path.Clean("/" + strings.TrimSpace(strings.TrimPrefix(tablePath, "/")))
sum := sha1.Sum([]byte(normalized))
return hex.EncodeToString(sum[:])
}
func tableBucketPathFromTablePath(tablePath string) (string, bool) {
normalized := path.Clean("/" + strings.TrimSpace(strings.TrimPrefix(tablePath, "/")))
tablesPrefix := strings.TrimSuffix(TablesPath, "/") + "/"
if !strings.HasPrefix(normalized, tablesPrefix) {
return "", false
}
remaining := strings.TrimPrefix(normalized, tablesPrefix)
bucketName, _, _ := strings.Cut(remaining, "/")
if bucketName == "" {
return "", false
}
return path.Join(TablesPath, bucketName), true
}
// Metadata structures
type tableBucketMetadata struct {
@@ -131,9 +156,10 @@ type tableBucketMetadata struct {
// namespaceMetadata stores metadata for a namespace
type namespaceMetadata struct {
Namespace []string `json:"namespace"`
CreatedAt time.Time `json:"createdAt"`
OwnerAccountID string `json:"ownerAccountId"`
Namespace []string `json:"namespace"`
CreatedAt time.Time `json:"createdAt"`
OwnerAccountID string `json:"ownerAccountId"`
Properties map[string]string `json:"properties,omitempty"`
}
// tableMetadataInternal stores metadata for a table
@@ -326,35 +352,27 @@ func splitPath(p string) (dir, name string) {
return
}
// validateNamespace validates that the namespace provided is supported (single-level)
func validateNamespace(namespace []string) (string, error) {
if len(namespace) == 0 {
return "", fmt.Errorf("namespace is required")
}
if len(namespace) > 1 {
return "", fmt.Errorf("multi-level namespaces are not supported")
}
name := namespace[0]
func validateNamespacePart(name string) error {
if len(name) < 1 || len(name) > 255 {
return "", fmt.Errorf("namespace name must be between 1 and 255 characters")
return fmt.Errorf("namespace name must be between 1 and 255 characters")
}
// Prevent path traversal and multi-segment paths
if name == "." || name == ".." {
return "", fmt.Errorf("namespace name cannot be '.' or '..'")
return fmt.Errorf("namespace name cannot be '.' or '..'")
}
if strings.Contains(name, "/") {
return "", fmt.Errorf("namespace name cannot contain '/'")
return fmt.Errorf("namespace name cannot contain '/'")
}
// Must start and end with a letter or digit
start := name[0]
end := name[len(name)-1]
if !((start >= 'a' && start <= 'z') || (start >= '0' && start <= '9')) {
return "", fmt.Errorf("namespace name must start with a letter or digit")
return fmt.Errorf("namespace name must start with a letter or digit")
}
if !((end >= 'a' && end <= 'z') || (end >= '0' && end <= '9')) {
return "", fmt.Errorf("namespace name must end with a letter or digit")
return fmt.Errorf("namespace name must end with a letter or digit")
}
// Allowed characters: a-z, 0-9, _
@@ -362,15 +380,46 @@ func validateNamespace(namespace []string) (string, error) {
if (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_' {
continue
}
return "", fmt.Errorf("invalid namespace name: only 'a-z', '0-9', and '_' are allowed")
return fmt.Errorf("invalid namespace name: only 'a-z', '0-9', and '_' are allowed")
}
// Reserved prefix
if strings.HasPrefix(name, "aws") {
return "", fmt.Errorf("namespace name cannot start with reserved prefix 'aws'")
return fmt.Errorf("namespace name cannot start with reserved prefix 'aws'")
}
return name, nil
return nil
}
func normalizeNamespace(namespace []string) ([]string, error) {
if len(namespace) == 0 {
return nil, fmt.Errorf("namespace is required")
}
parts := namespace
if len(namespace) == 1 {
parts = strings.Split(namespace[0], ".")
}
normalized := make([]string, 0, len(parts))
for _, part := range parts {
if err := validateNamespacePart(part); err != nil {
return nil, err
}
normalized = append(normalized, part)
}
return normalized, nil
}
// validateNamespace validates namespace identifiers and returns an internal namespace key.
// A single dotted namespace value is interpreted as multi-level namespace for compatibility
// with path-style APIs, for example "analytics.daily" => ["analytics", "daily"].
func validateNamespace(namespace []string) (string, error) {
parts, err := normalizeNamespace(namespace)
if err != nil {
return "", err
}
return flattenNamespace(parts), nil
}
// ValidateNamespace is a wrapper to validate namespace for other packages.
@@ -378,6 +427,11 @@ func ValidateNamespace(namespace []string) (string, error) {
return validateNamespace(namespace)
}
// ParseNamespace parses a namespace string into namespace parts.
func ParseNamespace(namespace string) ([]string, error) {
return normalizeNamespace([]string{namespace})
}
// validateTableName validates a table name
func validateTableName(name string) (string, error) {
if len(name) < 1 || len(name) > 255 {
@@ -415,3 +469,14 @@ func flattenNamespace(namespace []string) string {
}
return strings.Join(namespace, ".")
}
func expandNamespace(namespace string) []string {
if namespace == "" {
return nil
}
parts, err := ParseNamespace(namespace)
if err != nil {
return []string{namespace}
}
return parts
}
+186
View File
@@ -0,0 +1,186 @@
package s3tables
import (
"encoding/json"
"reflect"
"strings"
"testing"
)
func TestValidateNamespaceSupportsMultiLevel(t *testing.T) {
got, err := validateNamespace([]string{"analytics", "daily"})
if err != nil {
t.Fatalf("validateNamespace returned error: %v", err)
}
if got != "analytics.daily" {
t.Fatalf("validateNamespace = %q, want %q", got, "analytics.daily")
}
}
func TestValidateNamespaceSupportsDottedInput(t *testing.T) {
got, err := validateNamespace([]string{"analytics.daily"})
if err != nil {
t.Fatalf("validateNamespace returned error: %v", err)
}
if got != "analytics.daily" {
t.Fatalf("validateNamespace = %q, want %q", got, "analytics.daily")
}
}
func TestValidateNamespaceRejectsEmptyDottedSegment(t *testing.T) {
_, err := validateNamespace([]string{"analytics..daily"})
if err == nil {
t.Fatalf("expected validateNamespace to fail for empty dotted segment")
}
}
func TestParseNamespace(t *testing.T) {
tests := []struct {
name string
namespace string
want []string
wantErr bool
}{
{
name: "single level",
namespace: "analytics",
want: []string{"analytics"},
},
{
name: "multi level dotted",
namespace: "analytics.daily",
want: []string{"analytics", "daily"},
},
{
name: "invalid reserved prefix",
namespace: "analytics.awsprod",
wantErr: true,
},
{
name: "invalid empty segment",
namespace: "analytics..daily",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := ParseNamespace(tt.namespace)
if tt.wantErr {
if err == nil {
t.Fatalf("ParseNamespace(%q) expected error", tt.namespace)
}
return
}
if err != nil {
t.Fatalf("ParseNamespace(%q) unexpected error: %v", tt.namespace, err)
}
if len(got) != len(tt.want) {
t.Fatalf("ParseNamespace(%q) = %v, want %v", tt.namespace, got, tt.want)
}
for i := range got {
if got[i] != tt.want[i] {
t.Fatalf("ParseNamespace(%q) = %v, want %v", tt.namespace, got, tt.want)
}
}
})
}
}
func TestParseTableFromARNWithMultiLevelNamespace(t *testing.T) {
arn := "arn:aws:s3tables:us-east-1:123456789012:bucket/testbucket/table/analytics.daily/events"
bucket, namespace, table, err := parseTableFromARN(arn)
if err != nil {
t.Fatalf("parseTableFromARN returned error: %v", err)
}
if bucket != "testbucket" {
t.Fatalf("bucket = %q, want %q", bucket, "testbucket")
}
if namespace != "analytics.daily" {
t.Fatalf("namespace = %q, want %q", namespace, "analytics.daily")
}
if table != "events" {
t.Fatalf("table = %q, want %q", table, "events")
}
}
func TestBuildTableARNWithDottedNamespace(t *testing.T) {
arn, err := BuildTableARN("us-east-1", "123456789012", "testbucket", "analytics.daily", "events")
if err != nil {
t.Fatalf("BuildTableARN returned error: %v", err)
}
if !strings.Contains(arn, "/table/analytics.daily/events") {
t.Fatalf("BuildTableARN returned %q, missing normalized namespace/table path", arn)
}
}
func TestExpandNamespace(t *testing.T) {
got := expandNamespace("analytics.daily")
want := []string{"analytics", "daily"}
if len(got) != len(want) {
t.Fatalf("expandNamespace = %v, want %v", got, want)
}
for i := range got {
if got[i] != want[i] {
t.Fatalf("expandNamespace = %v, want %v", got, want)
}
}
}
func TestNamespaceMetadataPropertiesRoundTrip(t *testing.T) {
testCases := []struct {
name string
metadata namespaceMetadata
}{
{
name: "with properties",
metadata: namespaceMetadata{
Namespace: []string{"analytics"},
Properties: map[string]string{"owner": "finance"},
OwnerAccountID: "123456789012",
},
},
{
name: "nil properties",
metadata: namespaceMetadata{
Namespace: []string{"analytics"},
Properties: nil,
OwnerAccountID: "123456789012",
},
},
{
name: "empty properties",
metadata: namespaceMetadata{
Namespace: []string{"analytics"},
Properties: map[string]string{},
OwnerAccountID: "123456789012",
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
data, err := json.Marshal(tc.metadata)
if err != nil {
t.Fatalf("json.Marshal(metadata) returned error: %v", err)
}
var decoded namespaceMetadata
if err := json.Unmarshal(data, &decoded); err != nil {
t.Fatalf("json.Unmarshal(data) returned error: %v", err)
}
// Due to `omitempty`, nil and empty maps are unmarshaled as nil.
if len(tc.metadata.Properties) == 0 {
if decoded.Properties != nil {
t.Fatalf("expected nil properties for empty/nil input, got %v", decoded.Properties)
}
return
}
if !reflect.DeepEqual(decoded.Properties, tc.metadata.Properties) {
t.Fatalf("decoded.Properties = %v, want %v", decoded.Properties, tc.metadata.Properties)
}
})
}
}
@@ -54,4 +54,3 @@ func TestCheckEcVolumeStatusCountOnlyDataShards(t *testing.T) {
t.Fatalf("expected shardCount=3, got %d", shardCount)
}
}