diff --git a/weed/credential/filer_etc/filer_etc_identity.go b/weed/credential/filer_etc/filer_etc_identity.go index 102ea8513..de0a8715f 100644 --- a/weed/credential/filer_etc/filer_etc_identity.go +++ b/weed/credential/filer_etc/filer_etc_identity.go @@ -462,259 +462,3 @@ func listEntries(ctx context.Context, client filer_pb.SeaweedFilerClient, dir st } return entries, nil } - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "strings" - - "github.com/seaweedfs/seaweedfs/weed/credential" - "github.com/seaweedfs/seaweedfs/weed/filer" - "github.com/seaweedfs/seaweedfs/weed/glog" - "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" - "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" -) - -const ( - IamIdentitiesDirectory = "identities" - IamConfigurationFile = "configuration.json" - IamLegacyIdentityFile = "identity.json" - IamLegacyIdentityOldFile = "identity.json.old" -) - -func (store *FilerEtcStore) LoadConfiguration(ctx context.Context) (*iam_pb.S3ApiConfiguration, error) { - s3cfg := &iam_pb.S3ApiConfiguration{} - - // 1. Load from legacy single file (low priority) - content, foundLegacy, err := store.readInsideFiler(filer.IamConfigDirectory, IamLegacyIdentityFile) - if err != nil { - return s3cfg, err - } - if foundLegacy && len(content) > 0 { - if err := filer.ParseS3ConfigurationFromBytes(content, s3cfg); err != nil { - glog.Errorf("Failed to parse legacy IAM configuration: %v", err) - return s3cfg, err - } - } - - // 2. Load from multi-file structure (high priority, overrides legacy) - // This will merge identities into s3cfg - if _, err := store.loadFromMultiFile(ctx, s3cfg); err != nil { - return s3cfg, err - } - - // 3. Perform migration if we loaded legacy config - // This ensures that all identities (including legacy ones) are written to individual files - // and the legacy file is renamed. - if foundLegacy { - if err := store.migrateToMultiFile(ctx, s3cfg); err != nil { - glog.Errorf("Failed to migrate IAM configuration to multi-file layout: %v", err) - return s3cfg, nil - } - } - - return s3cfg, nil -} - -func (store *FilerEtcStore) loadFromMultiFile(ctx context.Context, s3cfg *iam_pb.S3ApiConfiguration) (bool, error) { - var hasIdentities bool - - // Helper to find existing identity index - findIdentity := func(name string) int { - for i, identity := range s3cfg.Identities { - if identity.Name == name { - return i - } - } - return -1 - } - - // 1. List identities - err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - dir := filer.IamConfigDirectory + "/" + IamIdentitiesDirectory - entries, err := listEntries(ctx, client, dir) - if err != nil { - // If directory doesn't exist, it's not multi-file yet - return nil - } - - for _, entry := range entries { - if entry.IsDirectory { - continue - } - hasIdentities = true - - var content []byte - if len(entry.Content) > 0 { - content = entry.Content - } else { - c, err := filer.ReadInsideFiler(client, dir, entry.Name) - if err != nil { - glog.Warningf("Failed to read identity file %s: %v", entry.Name, err) - continue - } - content = c - } - - if len(content) > 0 { - identity := &iam_pb.Identity{} - if err := json.Unmarshal(content, identity); err != nil { - glog.Warningf("Failed to unmarshal identity %s: %v", entry.Name, err) - continue - } - - // Merge logic: Overwrite existing or Append - idx := findIdentity(identity.Name) - if idx != -1 { - s3cfg.Identities[idx] = identity - } else { - s3cfg.Identities = append(s3cfg.Identities, identity) - } - } - } - return nil - }) - - if err != nil { - return false, err - } - - // 2. Load configuration.json (Accounts, etc.) - content, found, err := store.readInsideFiler(filer.IamConfigDirectory, IamConfigurationFile) - if err != nil { - return false, err - } - if found && len(content) > 0 { - tempCfg := &iam_pb.S3ApiConfiguration{} - if err := filer.ParseS3ConfigurationFromBytes(content, tempCfg); err == nil { - // Overwrite accounts from configuration.json (high priority) - s3cfg.Accounts = tempCfg.Accounts - } - return true, nil - } - - return hasIdentities, nil -} - -func (store *FilerEtcStore) migrateToMultiFile(ctx context.Context, s3cfg *iam_pb.S3ApiConfiguration) error { - glog.Infof("Migrating IAM configuration to multi-file layout...") - - // 1. Save all identities - for _, identity := range s3cfg.Identities { - if err := store.saveIdentity(ctx, identity); err != nil { - return err - } - } - - // 2. Save rest of configuration - if err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - // Create config with only accounts - cleanCfg := &iam_pb.S3ApiConfiguration{ - Accounts: s3cfg.Accounts, - } - var buf bytes.Buffer - if err := filer.ProtoToText(&buf, cleanCfg); err != nil { - return err - } - return filer.SaveInsideFiler(client, filer.IamConfigDirectory, IamConfigurationFile, buf.Bytes()) - }); err != nil { - return err - } - - // 3. Rename legacy file - return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - // We use atomic rename if possible, but Filer 'AtomicRenameEntry' exists in filer_pb - // util.JoinPath(filer.IamConfigDirectory, IamLegacyIdentityFile) - - _, err := client.AtomicRenameEntry(context.Background(), &filer_pb.AtomicRenameEntryRequest{ - OldDirectory: filer.IamConfigDirectory, - OldName: IamLegacyIdentityFile, - NewDirectory: filer.IamConfigDirectory, - NewName: IamLegacyIdentityOldFile, - }) - return err - }) -} - -func (store *FilerEtcStore) SaveConfiguration(ctx context.Context, config *iam_pb.S3ApiConfiguration) error { - // 1. Save all identities - for _, identity := range config.Identities { - if err := store.saveIdentity(ctx, identity); err != nil { - return err - } - } - - // 2. Save configuration file (accounts) - err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - cleanCfg := &iam_pb.S3ApiConfiguration{ - Accounts: config.Accounts, - } - var buf bytes.Buffer - if err := filer.ProtoToText(&buf, cleanCfg); err != nil { - return err - } - return filer.SaveInsideFiler(client, filer.IamConfigDirectory, IamConfigurationFile, buf.Bytes()) - }) - if err != nil { - return err - } - - // 3. Cleanup removed identities (Full Sync) - // Get list of existing identity files - // Compare with config.Identities - // Delete unknown ones - - return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - dir := filer.IamConfigDirectory + "/" + IamIdentitiesDirectory - entries, err := listEntries(ctx, client, dir) - if err != nil { - return nil // Should exist by now - } - - validNames := make(map[string]bool) - for _, id := range config.Identities { - validNames[id.Name+".json"] = true - } - - for _, entry := range entries { - if !entry.IsDirectory && !validNames[entry.Name] { - // Delete obsolete identity file - client.DeleteEntry(context.Background(), &filer_pb.DeleteEntryRequest{ - Directory: dir, - Name: entry.Name, - }) - } - } - return nil - }) -} - -func (store *FilerEtcStore) CreateUser(ctx context.Context, identity *iam_pb.Identity) error { - // Check if user exists (read specific file) - existing, err := store.GetUser(ctx, identity.Name) - if err == nil && existing != nil { - return credential.ErrUserAlreadyExists - } - return store.saveIdentity(ctx, identity) -} - -func (store *FilerEtcStore) GetUser(ctx context.Context, username string) (*iam_pb.Identity, error) { - var identity *iam_pb.Identity - err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - data, err := filer.ReadInsideFiler(client, filer.IamConfigDirectory+"/"+IamIdentitiesDirectory, username+".json") - if err != nil { - if err == filer_pb.ErrNotFound { - return credential.ErrUserNotFound - } - return err - } - if len(data) == 0 { - return credential.ErrUserNotFound - } - identity = &iam_pb.Identity{} - return json.Unmarshal(data, identity) - }) - return identity, err -} diff --git a/weed/credential/filer_etc/filer_etc_policy.go b/weed/credential/filer_etc/filer_etc_policy.go index c1a5b9e80..a0548060d 100644 --- a/weed/credential/filer_etc/filer_etc_policy.go +++ b/weed/credential/filer_etc/filer_etc_policy.go @@ -3,6 +3,7 @@ package filer_etc import ( "context" "encoding/json" + "strings" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" @@ -10,15 +11,17 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine" ) +const ( + IamPoliciesDirectory = "policies" +) + type PoliciesCollection struct { Policies map[string]policy_engine.PolicyDocument `json:"policies"` } // GetPolicies retrieves all IAM policies from the filer func (store *FilerEtcStore) GetPolicies(ctx context.Context) (map[string]policy_engine.PolicyDocument, error) { - policiesCollection := &PoliciesCollection{ - Policies: make(map[string]policy_engine.PolicyDocument), - } + policies := make(map[string]policy_engine.PolicyDocument) // Check if filer client is configured (with mutex protection) store.mu.RLock() @@ -27,125 +30,210 @@ func (store *FilerEtcStore) GetPolicies(ctx context.Context) (map[string]policy_ if !configured { glog.V(1).Infof("Filer client not configured for policy retrieval, returning empty policies") - // Return empty policies if filer client is not configured - return policiesCollection.Policies, nil + return policies, nil } - glog.V(2).Infof("Loading IAM policies from %s/%s (using current active filer)", - filer.IamConfigDirectory, filer.IamPoliciesFile) - - err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - // Use ReadInsideFiler instead of ReadEntry since policies.json is small - // and stored inline. ReadEntry requires a master client for chunked files, - // but ReadInsideFiler only reads inline content. - content, err := filer.ReadInsideFiler(client, filer.IamConfigDirectory, filer.IamPoliciesFile) - if err != nil { - if err == filer_pb.ErrNotFound { - glog.V(1).Infof("Policies file not found at %s/%s, returning empty policies", - filer.IamConfigDirectory, filer.IamPoliciesFile) - // If file doesn't exist, return empty collection - return nil - } - glog.Errorf("Failed to read IAM policies file from %s/%s: %v", - filer.IamConfigDirectory, filer.IamPoliciesFile, err) - return err - } - - if len(content) == 0 { - glog.V(2).Infof("IAM policies file at %s/%s is empty", - filer.IamConfigDirectory, filer.IamPoliciesFile) - return nil - } - - glog.V(2).Infof("Read %d bytes from %s/%s", - len(content), filer.IamConfigDirectory, filer.IamPoliciesFile) - - if err := json.Unmarshal(content, policiesCollection); err != nil { - glog.Errorf("Failed to parse IAM policies from %s/%s: %v", - filer.IamConfigDirectory, filer.IamPoliciesFile, err) - return err - } - - glog.V(1).Infof("Successfully loaded %d IAM policies", len(policiesCollection.Policies)) - return nil - }) - + // 1. Load from legacy file (low priority) + content, foundLegacy, err := store.readInsideFiler(filer.IamConfigDirectory, filer.IamPoliciesFile) if err != nil { return nil, err } - - // Log policy names for debugging - if glog.V(2) && len(policiesCollection.Policies) > 0 { - for policyName := range policiesCollection.Policies { - glog.V(2).Infof(" Policy: %s", policyName) + if foundLegacy && len(content) > 0 { + legacyCollection := &PoliciesCollection{} + if err := json.Unmarshal(content, legacyCollection); err != nil { + glog.Errorf("Failed to parse legacy IAM policies: %v", err) + } else { + for k, v := range legacyCollection.Policies { + policies[k] = v + } } } - return policiesCollection.Policies, nil + // 2. Load from multi-file structure (high priority, overrides legacy) + if err := store.loadPoliciesFromMultiFile(ctx, policies); err != nil { + return nil, err + } + + // 3. Perform migration if we loaded legacy config + if foundLegacy { + if err := store.migratePoliciesToMultiFile(ctx, policies); err != nil { + glog.Errorf("Failed to migrate IAM policies to multi-file layout: %v", err) + return policies, err + } + } + + return policies, nil +} + +func (store *FilerEtcStore) loadPoliciesFromMultiFile(ctx context.Context, policies map[string]policy_engine.PolicyDocument) error { + return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + dir := filer.IamConfigDirectory + "/" + IamPoliciesDirectory + entries, err := listEntries(ctx, client, dir) + if err != nil { + if err == filer_pb.ErrNotFound { + return nil + } + return err + } + + for _, entry := range entries { + if entry.IsDirectory { + continue + } + + // We iterate, so we can lazily read or used passed content. + // Safest to read if content missing or use helper. + + var content []byte + if len(entry.Content) > 0 { + content = entry.Content + } else { + c, err := filer.ReadInsideFiler(client, dir, entry.Name) + if err != nil { + glog.Warningf("Failed to read policy file %s: %v", entry.Name, err) + continue + } + content = c + } + + if len(content) > 0 { + policy := policy_engine.PolicyDocument{} + if err := json.Unmarshal(content, &policy); err != nil { + glog.Warningf("Failed to unmarshal policy %s: %v", entry.Name, err) + continue + } + + // Policies are stored as .json + name := entry.Name + if strings.HasSuffix(name, ".json") { + name = name[:len(name)-5] + } + policies[name] = policy + } + } + return nil + }) +} + +func (store *FilerEtcStore) migratePoliciesToMultiFile(ctx context.Context, policies map[string]policy_engine.PolicyDocument) error { + glog.Infof("Migrating IAM policies to multi-file layout...") + + // 1. Save all policies to individual files + for name, policy := range policies { + if err := store.savePolicyfile(ctx, name, policy); err != nil { + return err + } + } + + // 2. Rename legacy file + return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + // Using .old extension for legacy file + oldName := filer.IamPoliciesFile + ".old" + _, err := client.AtomicRenameEntry(ctx, &filer_pb.AtomicRenameEntryRequest{ + OldDirectory: filer.IamConfigDirectory, + OldName: filer.IamPoliciesFile, + NewDirectory: filer.IamConfigDirectory, + NewName: oldName, + }) + return err + }) +} + +func (store *FilerEtcStore) savePolicyfile(ctx context.Context, name string, policy policy_engine.PolicyDocument) error { + return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + data, err := json.Marshal(policy) + if err != nil { + return err + } + return filer.SaveInsideFiler(client, filer.IamConfigDirectory+"/"+IamPoliciesDirectory, name+".json", data) + }) } // CreatePolicy creates a new IAM policy in the filer func (store *FilerEtcStore) CreatePolicy(ctx context.Context, name string, document policy_engine.PolicyDocument) error { - return store.updatePolicies(ctx, func(policies map[string]policy_engine.PolicyDocument) { - policies[name] = document - }) + // Check if already exists? GetPolicy logic. + // Spec says PutPolicy is create/update. CreatePolicy might strictly be create? + // The store interface usually implies overwrite for Put, but Create implies existence check. + // But let's check current implementation: it was just map assignment. + // Safe to overwrite or do we want to check? + // Existing: "policies[name] = document". So it overwrites. + + return store.savePolicyfile(ctx, name, document) } // UpdatePolicy updates an existing IAM policy in the filer func (store *FilerEtcStore) UpdatePolicy(ctx context.Context, name string, document policy_engine.PolicyDocument) error { - return store.updatePolicies(ctx, func(policies map[string]policy_engine.PolicyDocument) { - policies[name] = document - }) + return store.savePolicyfile(ctx, name, document) } // PutPolicy creates or updates an IAM policy in the filer func (store *FilerEtcStore) PutPolicy(ctx context.Context, name string, document policy_engine.PolicyDocument) error { - return store.UpdatePolicy(ctx, name, document) + return store.savePolicyfile(ctx, name, document) } // DeletePolicy deletes an IAM policy from the filer func (store *FilerEtcStore) DeletePolicy(ctx context.Context, name string) error { - return store.updatePolicies(ctx, func(policies map[string]policy_engine.PolicyDocument) { - delete(policies, name) - }) -} - -// updatePolicies is a helper method to update policies atomically -func (store *FilerEtcStore) updatePolicies(ctx context.Context, updateFunc func(map[string]policy_engine.PolicyDocument)) error { - // Load existing policies - policies, err := store.GetPolicies(ctx) - if err != nil { - return err - } - - // Apply update - updateFunc(policies) - - // Save back to filer - policiesCollection := &PoliciesCollection{ - Policies: policies, - } - - data, err := json.Marshal(policiesCollection) - if err != nil { - return err - } - return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - return filer.SaveInsideFiler(client, filer.IamConfigDirectory, filer.IamPoliciesFile, data) + _, err := client.DeleteEntry(ctx, &filer_pb.DeleteEntryRequest{ + Directory: filer.IamConfigDirectory + "/" + IamPoliciesDirectory, + Name: name + ".json", + }) + if err != nil && !strings.Contains(err.Error(), filer_pb.ErrNotFound.Error()) { + return err + } + return nil }) } // GetPolicy retrieves a specific IAM policy by name from the filer func (store *FilerEtcStore) GetPolicy(ctx context.Context, name string) (*policy_engine.PolicyDocument, error) { - policies, err := store.GetPolicies(ctx) + // Optimization: Read directly from file + var policy *policy_engine.PolicyDocument + err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + data, err := filer.ReadInsideFiler(client, filer.IamConfigDirectory+"/"+IamPoliciesDirectory, name+".json") + if err != nil { + if err == filer_pb.ErrNotFound { + return nil + } + return err + } + if len(data) == 0 { + return nil + } + policy = &policy_engine.PolicyDocument{} + return json.Unmarshal(data, policy) + }) + if err != nil { return nil, err } - if policy, exists := policies[name]; exists { - return &policy, nil + // If not found in multi-file, check legacy? + // If migration hasn't happened yet, we might miss it. + // But `GetPolicies` triggers migration. + // If we access `GetPolicy` directly before `GetPolicies` (server start), migration might not have run. + // Ideally `LoadConfiguration` triggers migration on start. + // For policies, we don't have a "LoadPoliciesConfiguration" that runs on start always. + // But accessing via `GetPolicy` directly is rare without listing? + // Actually `GetPolicy` is used by verifying access. + // To be safe and transparent: if not found in multi-file, we COULD check legacy. + // Or we force migration on first access? + // Or simplistic approach: just use GetPolicies logic which handles migration. + // BUT `GetPolicies` reads ALL files. `GetPolicy` should be fast. + // Let's implement fallback: if file read fails (not found), read `policies.json`. + + if policy != nil { + return policy, nil } - return nil, nil // Policy not found + // Fallback to legacy check (inefficient but safe for transition if migration missed) + legacyPolicies, err := store.GetPolicies(ctx) // This triggers migration if needed! + if err != nil { + return nil, err + } + if p, ok := legacyPolicies[name]; ok { + return &p, nil + } + + return nil, nil // Not found }