feat: migrate policies to multi-file layout and fix identity duplicated content

This commit is contained in:
Chris Lu
2026-01-25 18:43:05 -08:00
parent fe6a95c89b
commit 6977188029
2 changed files with 176 additions and 344 deletions
@@ -462,259 +462,3 @@ func listEntries(ctx context.Context, client filer_pb.SeaweedFilerClient, dir st
}
return entries, nil
}
import (
"bytes"
"context"
"encoding/json"
"fmt"
"strings"
"github.com/seaweedfs/seaweedfs/weed/credential"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
)
const (
IamIdentitiesDirectory = "identities"
IamConfigurationFile = "configuration.json"
IamLegacyIdentityFile = "identity.json"
IamLegacyIdentityOldFile = "identity.json.old"
)
func (store *FilerEtcStore) LoadConfiguration(ctx context.Context) (*iam_pb.S3ApiConfiguration, error) {
s3cfg := &iam_pb.S3ApiConfiguration{}
// 1. Load from legacy single file (low priority)
content, foundLegacy, err := store.readInsideFiler(filer.IamConfigDirectory, IamLegacyIdentityFile)
if err != nil {
return s3cfg, err
}
if foundLegacy && len(content) > 0 {
if err := filer.ParseS3ConfigurationFromBytes(content, s3cfg); err != nil {
glog.Errorf("Failed to parse legacy IAM configuration: %v", err)
return s3cfg, err
}
}
// 2. Load from multi-file structure (high priority, overrides legacy)
// This will merge identities into s3cfg
if _, err := store.loadFromMultiFile(ctx, s3cfg); err != nil {
return s3cfg, err
}
// 3. Perform migration if we loaded legacy config
// This ensures that all identities (including legacy ones) are written to individual files
// and the legacy file is renamed.
if foundLegacy {
if err := store.migrateToMultiFile(ctx, s3cfg); err != nil {
glog.Errorf("Failed to migrate IAM configuration to multi-file layout: %v", err)
return s3cfg, nil
}
}
return s3cfg, nil
}
func (store *FilerEtcStore) loadFromMultiFile(ctx context.Context, s3cfg *iam_pb.S3ApiConfiguration) (bool, error) {
var hasIdentities bool
// Helper to find existing identity index
findIdentity := func(name string) int {
for i, identity := range s3cfg.Identities {
if identity.Name == name {
return i
}
}
return -1
}
// 1. List identities
err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
dir := filer.IamConfigDirectory + "/" + IamIdentitiesDirectory
entries, err := listEntries(ctx, client, dir)
if err != nil {
// If directory doesn't exist, it's not multi-file yet
return nil
}
for _, entry := range entries {
if entry.IsDirectory {
continue
}
hasIdentities = true
var content []byte
if len(entry.Content) > 0 {
content = entry.Content
} else {
c, err := filer.ReadInsideFiler(client, dir, entry.Name)
if err != nil {
glog.Warningf("Failed to read identity file %s: %v", entry.Name, err)
continue
}
content = c
}
if len(content) > 0 {
identity := &iam_pb.Identity{}
if err := json.Unmarshal(content, identity); err != nil {
glog.Warningf("Failed to unmarshal identity %s: %v", entry.Name, err)
continue
}
// Merge logic: Overwrite existing or Append
idx := findIdentity(identity.Name)
if idx != -1 {
s3cfg.Identities[idx] = identity
} else {
s3cfg.Identities = append(s3cfg.Identities, identity)
}
}
}
return nil
})
if err != nil {
return false, err
}
// 2. Load configuration.json (Accounts, etc.)
content, found, err := store.readInsideFiler(filer.IamConfigDirectory, IamConfigurationFile)
if err != nil {
return false, err
}
if found && len(content) > 0 {
tempCfg := &iam_pb.S3ApiConfiguration{}
if err := filer.ParseS3ConfigurationFromBytes(content, tempCfg); err == nil {
// Overwrite accounts from configuration.json (high priority)
s3cfg.Accounts = tempCfg.Accounts
}
return true, nil
}
return hasIdentities, nil
}
func (store *FilerEtcStore) migrateToMultiFile(ctx context.Context, s3cfg *iam_pb.S3ApiConfiguration) error {
glog.Infof("Migrating IAM configuration to multi-file layout...")
// 1. Save all identities
for _, identity := range s3cfg.Identities {
if err := store.saveIdentity(ctx, identity); err != nil {
return err
}
}
// 2. Save rest of configuration
if err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
// Create config with only accounts
cleanCfg := &iam_pb.S3ApiConfiguration{
Accounts: s3cfg.Accounts,
}
var buf bytes.Buffer
if err := filer.ProtoToText(&buf, cleanCfg); err != nil {
return err
}
return filer.SaveInsideFiler(client, filer.IamConfigDirectory, IamConfigurationFile, buf.Bytes())
}); err != nil {
return err
}
// 3. Rename legacy file
return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
// We use atomic rename if possible, but Filer 'AtomicRenameEntry' exists in filer_pb
// util.JoinPath(filer.IamConfigDirectory, IamLegacyIdentityFile)
_, err := client.AtomicRenameEntry(context.Background(), &filer_pb.AtomicRenameEntryRequest{
OldDirectory: filer.IamConfigDirectory,
OldName: IamLegacyIdentityFile,
NewDirectory: filer.IamConfigDirectory,
NewName: IamLegacyIdentityOldFile,
})
return err
})
}
func (store *FilerEtcStore) SaveConfiguration(ctx context.Context, config *iam_pb.S3ApiConfiguration) error {
// 1. Save all identities
for _, identity := range config.Identities {
if err := store.saveIdentity(ctx, identity); err != nil {
return err
}
}
// 2. Save configuration file (accounts)
err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
cleanCfg := &iam_pb.S3ApiConfiguration{
Accounts: config.Accounts,
}
var buf bytes.Buffer
if err := filer.ProtoToText(&buf, cleanCfg); err != nil {
return err
}
return filer.SaveInsideFiler(client, filer.IamConfigDirectory, IamConfigurationFile, buf.Bytes())
})
if err != nil {
return err
}
// 3. Cleanup removed identities (Full Sync)
// Get list of existing identity files
// Compare with config.Identities
// Delete unknown ones
return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
dir := filer.IamConfigDirectory + "/" + IamIdentitiesDirectory
entries, err := listEntries(ctx, client, dir)
if err != nil {
return nil // Should exist by now
}
validNames := make(map[string]bool)
for _, id := range config.Identities {
validNames[id.Name+".json"] = true
}
for _, entry := range entries {
if !entry.IsDirectory && !validNames[entry.Name] {
// Delete obsolete identity file
client.DeleteEntry(context.Background(), &filer_pb.DeleteEntryRequest{
Directory: dir,
Name: entry.Name,
})
}
}
return nil
})
}
func (store *FilerEtcStore) CreateUser(ctx context.Context, identity *iam_pb.Identity) error {
// Check if user exists (read specific file)
existing, err := store.GetUser(ctx, identity.Name)
if err == nil && existing != nil {
return credential.ErrUserAlreadyExists
}
return store.saveIdentity(ctx, identity)
}
func (store *FilerEtcStore) GetUser(ctx context.Context, username string) (*iam_pb.Identity, error) {
var identity *iam_pb.Identity
err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
data, err := filer.ReadInsideFiler(client, filer.IamConfigDirectory+"/"+IamIdentitiesDirectory, username+".json")
if err != nil {
if err == filer_pb.ErrNotFound {
return credential.ErrUserNotFound
}
return err
}
if len(data) == 0 {
return credential.ErrUserNotFound
}
identity = &iam_pb.Identity{}
return json.Unmarshal(data, identity)
})
return identity, err
}
+176 -88
View File
@@ -3,6 +3,7 @@ package filer_etc
import (
"context"
"encoding/json"
"strings"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -10,15 +11,17 @@ import (
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
)
const (
IamPoliciesDirectory = "policies"
)
type PoliciesCollection struct {
Policies map[string]policy_engine.PolicyDocument `json:"policies"`
}
// GetPolicies retrieves all IAM policies from the filer
func (store *FilerEtcStore) GetPolicies(ctx context.Context) (map[string]policy_engine.PolicyDocument, error) {
policiesCollection := &PoliciesCollection{
Policies: make(map[string]policy_engine.PolicyDocument),
}
policies := make(map[string]policy_engine.PolicyDocument)
// Check if filer client is configured (with mutex protection)
store.mu.RLock()
@@ -27,125 +30,210 @@ func (store *FilerEtcStore) GetPolicies(ctx context.Context) (map[string]policy_
if !configured {
glog.V(1).Infof("Filer client not configured for policy retrieval, returning empty policies")
// Return empty policies if filer client is not configured
return policiesCollection.Policies, nil
return policies, nil
}
glog.V(2).Infof("Loading IAM policies from %s/%s (using current active filer)",
filer.IamConfigDirectory, filer.IamPoliciesFile)
err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
// Use ReadInsideFiler instead of ReadEntry since policies.json is small
// and stored inline. ReadEntry requires a master client for chunked files,
// but ReadInsideFiler only reads inline content.
content, err := filer.ReadInsideFiler(client, filer.IamConfigDirectory, filer.IamPoliciesFile)
if err != nil {
if err == filer_pb.ErrNotFound {
glog.V(1).Infof("Policies file not found at %s/%s, returning empty policies",
filer.IamConfigDirectory, filer.IamPoliciesFile)
// If file doesn't exist, return empty collection
return nil
}
glog.Errorf("Failed to read IAM policies file from %s/%s: %v",
filer.IamConfigDirectory, filer.IamPoliciesFile, err)
return err
}
if len(content) == 0 {
glog.V(2).Infof("IAM policies file at %s/%s is empty",
filer.IamConfigDirectory, filer.IamPoliciesFile)
return nil
}
glog.V(2).Infof("Read %d bytes from %s/%s",
len(content), filer.IamConfigDirectory, filer.IamPoliciesFile)
if err := json.Unmarshal(content, policiesCollection); err != nil {
glog.Errorf("Failed to parse IAM policies from %s/%s: %v",
filer.IamConfigDirectory, filer.IamPoliciesFile, err)
return err
}
glog.V(1).Infof("Successfully loaded %d IAM policies", len(policiesCollection.Policies))
return nil
})
// 1. Load from legacy file (low priority)
content, foundLegacy, err := store.readInsideFiler(filer.IamConfigDirectory, filer.IamPoliciesFile)
if err != nil {
return nil, err
}
// Log policy names for debugging
if glog.V(2) && len(policiesCollection.Policies) > 0 {
for policyName := range policiesCollection.Policies {
glog.V(2).Infof(" Policy: %s", policyName)
if foundLegacy && len(content) > 0 {
legacyCollection := &PoliciesCollection{}
if err := json.Unmarshal(content, legacyCollection); err != nil {
glog.Errorf("Failed to parse legacy IAM policies: %v", err)
} else {
for k, v := range legacyCollection.Policies {
policies[k] = v
}
}
}
return policiesCollection.Policies, nil
// 2. Load from multi-file structure (high priority, overrides legacy)
if err := store.loadPoliciesFromMultiFile(ctx, policies); err != nil {
return nil, err
}
// 3. Perform migration if we loaded legacy config
if foundLegacy {
if err := store.migratePoliciesToMultiFile(ctx, policies); err != nil {
glog.Errorf("Failed to migrate IAM policies to multi-file layout: %v", err)
return policies, err
}
}
return policies, nil
}
func (store *FilerEtcStore) loadPoliciesFromMultiFile(ctx context.Context, policies map[string]policy_engine.PolicyDocument) error {
return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
dir := filer.IamConfigDirectory + "/" + IamPoliciesDirectory
entries, err := listEntries(ctx, client, dir)
if err != nil {
if err == filer_pb.ErrNotFound {
return nil
}
return err
}
for _, entry := range entries {
if entry.IsDirectory {
continue
}
// We iterate, so we can lazily read or used passed content.
// Safest to read if content missing or use helper.
var content []byte
if len(entry.Content) > 0 {
content = entry.Content
} else {
c, err := filer.ReadInsideFiler(client, dir, entry.Name)
if err != nil {
glog.Warningf("Failed to read policy file %s: %v", entry.Name, err)
continue
}
content = c
}
if len(content) > 0 {
policy := policy_engine.PolicyDocument{}
if err := json.Unmarshal(content, &policy); err != nil {
glog.Warningf("Failed to unmarshal policy %s: %v", entry.Name, err)
continue
}
// Policies are stored as <name>.json
name := entry.Name
if strings.HasSuffix(name, ".json") {
name = name[:len(name)-5]
}
policies[name] = policy
}
}
return nil
})
}
func (store *FilerEtcStore) migratePoliciesToMultiFile(ctx context.Context, policies map[string]policy_engine.PolicyDocument) error {
glog.Infof("Migrating IAM policies to multi-file layout...")
// 1. Save all policies to individual files
for name, policy := range policies {
if err := store.savePolicyfile(ctx, name, policy); err != nil {
return err
}
}
// 2. Rename legacy file
return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
// Using .old extension for legacy file
oldName := filer.IamPoliciesFile + ".old"
_, err := client.AtomicRenameEntry(ctx, &filer_pb.AtomicRenameEntryRequest{
OldDirectory: filer.IamConfigDirectory,
OldName: filer.IamPoliciesFile,
NewDirectory: filer.IamConfigDirectory,
NewName: oldName,
})
return err
})
}
func (store *FilerEtcStore) savePolicyfile(ctx context.Context, name string, policy policy_engine.PolicyDocument) error {
return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
data, err := json.Marshal(policy)
if err != nil {
return err
}
return filer.SaveInsideFiler(client, filer.IamConfigDirectory+"/"+IamPoliciesDirectory, name+".json", data)
})
}
// CreatePolicy creates a new IAM policy in the filer
func (store *FilerEtcStore) CreatePolicy(ctx context.Context, name string, document policy_engine.PolicyDocument) error {
return store.updatePolicies(ctx, func(policies map[string]policy_engine.PolicyDocument) {
policies[name] = document
})
// Check if already exists? GetPolicy logic.
// Spec says PutPolicy is create/update. CreatePolicy might strictly be create?
// The store interface usually implies overwrite for Put, but Create implies existence check.
// But let's check current implementation: it was just map assignment.
// Safe to overwrite or do we want to check?
// Existing: "policies[name] = document". So it overwrites.
return store.savePolicyfile(ctx, name, document)
}
// UpdatePolicy updates an existing IAM policy in the filer
func (store *FilerEtcStore) UpdatePolicy(ctx context.Context, name string, document policy_engine.PolicyDocument) error {
return store.updatePolicies(ctx, func(policies map[string]policy_engine.PolicyDocument) {
policies[name] = document
})
return store.savePolicyfile(ctx, name, document)
}
// PutPolicy creates or updates an IAM policy in the filer
func (store *FilerEtcStore) PutPolicy(ctx context.Context, name string, document policy_engine.PolicyDocument) error {
return store.UpdatePolicy(ctx, name, document)
return store.savePolicyfile(ctx, name, document)
}
// DeletePolicy deletes an IAM policy from the filer
func (store *FilerEtcStore) DeletePolicy(ctx context.Context, name string) error {
return store.updatePolicies(ctx, func(policies map[string]policy_engine.PolicyDocument) {
delete(policies, name)
})
}
// updatePolicies is a helper method to update policies atomically
func (store *FilerEtcStore) updatePolicies(ctx context.Context, updateFunc func(map[string]policy_engine.PolicyDocument)) error {
// Load existing policies
policies, err := store.GetPolicies(ctx)
if err != nil {
return err
}
// Apply update
updateFunc(policies)
// Save back to filer
policiesCollection := &PoliciesCollection{
Policies: policies,
}
data, err := json.Marshal(policiesCollection)
if err != nil {
return err
}
return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
return filer.SaveInsideFiler(client, filer.IamConfigDirectory, filer.IamPoliciesFile, data)
_, err := client.DeleteEntry(ctx, &filer_pb.DeleteEntryRequest{
Directory: filer.IamConfigDirectory + "/" + IamPoliciesDirectory,
Name: name + ".json",
})
if err != nil && !strings.Contains(err.Error(), filer_pb.ErrNotFound.Error()) {
return err
}
return nil
})
}
// GetPolicy retrieves a specific IAM policy by name from the filer
func (store *FilerEtcStore) GetPolicy(ctx context.Context, name string) (*policy_engine.PolicyDocument, error) {
policies, err := store.GetPolicies(ctx)
// Optimization: Read directly from file
var policy *policy_engine.PolicyDocument
err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
data, err := filer.ReadInsideFiler(client, filer.IamConfigDirectory+"/"+IamPoliciesDirectory, name+".json")
if err != nil {
if err == filer_pb.ErrNotFound {
return nil
}
return err
}
if len(data) == 0 {
return nil
}
policy = &policy_engine.PolicyDocument{}
return json.Unmarshal(data, policy)
})
if err != nil {
return nil, err
}
if policy, exists := policies[name]; exists {
return &policy, nil
// If not found in multi-file, check legacy?
// If migration hasn't happened yet, we might miss it.
// But `GetPolicies` triggers migration.
// If we access `GetPolicy` directly before `GetPolicies` (server start), migration might not have run.
// Ideally `LoadConfiguration` triggers migration on start.
// For policies, we don't have a "LoadPoliciesConfiguration" that runs on start always.
// But accessing via `GetPolicy` directly is rare without listing?
// Actually `GetPolicy` is used by verifying access.
// To be safe and transparent: if not found in multi-file, we COULD check legacy.
// Or we force migration on first access?
// Or simplistic approach: just use GetPolicies logic which handles migration.
// BUT `GetPolicies` reads ALL files. `GetPolicy` should be fast.
// Let's implement fallback: if file read fails (not found), read `policies.json`.
if policy != nil {
return policy, nil
}
return nil, nil // Policy not found
// Fallback to legacy check (inefficient but safe for transition if migration missed)
legacyPolicies, err := store.GetPolicies(ctx) // This triggers migration if needed!
if err != nil {
return nil, err
}
if p, ok := legacyPolicies[name]; ok {
return &p, nil
}
return nil, nil // Not found
}