mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-09-19 06:26:44 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into batch-delete-snapshot

Browse Source
This commit is contained in:
Lyndon-Li
2024-02-18 14:54:01 +08:00
parent 32d92ca964 3b8370e13c
commit 57879357fc
164 changed files with 1121 additions and 489 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/repository/config/aws.go
+67 -5
View File
@@ -19,7 +19,13 @@ package config
import (
"context"
"fmt"
"os"
"time"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
"github.com/aws/aws-sdk-go-v2/service/sts"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
@@ -31,13 +37,13 @@ import (
const (
// AWS specific environment variable
awsProfileEnvVar = "AWS_PROFILE"
awsRoleEnvVar = "AWS_ROLE_ARN"
awsKeyIDEnvVar = "AWS_ACCESS_KEY_ID"
awsSecretKeyEnvVar = "AWS_SECRET_ACCESS_KEY"
awsSessTokenEnvVar = "AWS_SESSION_TOKEN"
awsProfileKey = "profile"
awsCredentialsFileEnvVar = "AWS_SHARED_CREDENTIALS_FILE"
awsConfigFileEnvVar = "AWS_CONFIG_FILE"
awsDefaultProfile = "default"
)
// GetS3ResticEnvVars gets the environment variables that restic
@@ -72,10 +78,6 @@ func GetS3ResticEnvVars(config map[string]string) (map[string]string, error) {
// GetS3Credentials gets the S3 credential values according to the information
// of the provided config or the system's environment variables
func GetS3Credentials(config map[string]string) (*aws.Credentials, error) {
if os.Getenv(awsRoleEnvVar) != "" {
return nil, nil
}
var opts []func(*awsconfig.LoadOptions) error
credentialsFile := config[CredentialsFileKey]
if credentialsFile == "" {
@@ -93,6 +95,25 @@ func GetS3Credentials(config map[string]string) (*aws.Credentials, error) {
if err != nil {
return nil, err
}
if credentialsFile != "" && os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE") != "" && os.Getenv("AWS_ROLE_ARN") != "" {
// Reset the config to use the credentials from the credentials/config file
profile := config[awsProfileKey]
if profile == "" {
profile = awsDefaultProfile
}
sfp, err := awsconfig.LoadSharedConfigProfile(context.Background(), profile, func(o *awsconfig.LoadSharedConfigOptions) {
o.ConfigFiles = []string{credentialsFile}
o.CredentialsFiles = []string{credentialsFile}
})
if err != nil {
return nil, fmt.Errorf("error loading config profile '%s': %v", profile, err)
}
if err := resolveCredsFromProfile(&cfg, &sfp); err != nil {
return nil, fmt.Errorf("error resolving creds from profile '%s': %v", profile, err)
}
}
creds, err := cfg.Credentials.Retrieve(context.Background())
return &creds, err
@@ -115,3 +136,44 @@ func GetAWSBucketRegion(bucket string) (string, error) {
}
return region, nil
}
func resolveCredsFromProfile(cfg *aws.Config, sharedConfig *awsconfig.SharedConfig) error {
var err error
switch {
case sharedConfig.Source != nil:
// Assume IAM role with credentials source from a different profile.
err = resolveCredsFromProfile(cfg, sharedConfig.Source)
case sharedConfig.Credentials.HasKeys():
// Static Credentials from Shared Config/Credentials file.
cfg.Credentials = credentials.StaticCredentialsProvider{
Value: sharedConfig.Credentials,
}
}
if err != nil {
return err
}
if len(sharedConfig.RoleARN) > 0 {
credsFromAssumeRole(cfg, sharedConfig)
}
return nil
}
func credsFromAssumeRole(cfg *aws.Config, sharedCfg *awsconfig.SharedConfig) {
optFns := []func(*stscreds.AssumeRoleOptions){
func(options *stscreds.AssumeRoleOptions) {
options.RoleSessionName = sharedCfg.RoleSessionName
if sharedCfg.RoleDurationSeconds != nil {
if *sharedCfg.RoleDurationSeconds/time.Minute > 15 {
options.Duration = *sharedCfg.RoleDurationSeconds
}
}
if len(sharedCfg.ExternalID) > 0 {
options.ExternalID = aws.String(sharedCfg.ExternalID)
}
if len(sharedCfg.MFASerial) != 0 {
options.SerialNumber = aws.String(sharedCfg.MFASerial)
}
},
}
cfg.Credentials = stscreds.NewAssumeRoleProvider(sts.NewFromConfig(*cfg), sharedCfg.RoleARN, optFns...)
}
pkg/repository/provider/unified_repo.go
+7 -14
View File
@@ -53,7 +53,7 @@ var getGCPCredentials = repoconfig.GetGCPCredentials
var getS3BucketRegion = repoconfig.GetAWSBucketRegion
type localFuncTable struct {
getStorageVariables func(*velerov1api.BackupStorageLocation, string, string, credentials.FileStore) (map[string]string, error)
getStorageVariables func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error)
getStorageCredentials func(*velerov1api.BackupStorageLocation, credentials.FileStore) (map[string]string, error)
}
@@ -397,7 +397,7 @@ func (urp *unifiedRepoProvider) GetStoreOptions(param interface{}) (map[string]s
return map[string]string{}, errors.Errorf("invalid parameter, expect %T, actual %T", RepoParam{}, param)
}
storeVar, err := funcTable.getStorageVariables(repoParam.BackupLocation, urp.repoBackend, repoParam.BackupRepo.Spec.VolumeNamespace, urp.credentialGetter.FromFile)
storeVar, err := funcTable.getStorageVariables(repoParam.BackupLocation, urp.repoBackend, repoParam.BackupRepo.Spec.VolumeNamespace)
if err != nil {
return map[string]string{}, errors.Wrap(err, "error to get storage variables")
}
@@ -488,8 +488,9 @@ func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, cr
result[udmrepo.StoreOptionS3Token] = credValue.SessionToken
}
case repoconfig.AzureBackend:
// do nothing here, will retrieve the credential in Azure Storage
return nil, nil
if config[repoconfig.CredentialsFileKey] != "" {
result[repoconfig.CredentialsFileKey] = config[repoconfig.CredentialsFileKey]
}
case repoconfig.GCPBackend:
result[udmrepo.StoreOptionCredentialFile] = getGCPCredentials(config)
}
@@ -497,8 +498,7 @@ func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, cr
return result, nil
}
func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repoBackend string, repoName string,
credentialFileStore credentials.FileStore) (map[string]string, error) {
func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repoBackend string, repoName string) (map[string]string, error) {
result := make(map[string]string)
backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider, backupLocation.Spec.Config)
@@ -510,13 +510,6 @@ func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repo
if config == nil {
config = map[string]string{}
}
if backupLocation.Spec.Credential != nil {
credsFile, err := credentialFileStore.Path(backupLocation.Spec.Credential)
if err != nil {
return map[string]string{}, errors.WithStack(err)
}
config[repoconfig.CredentialsFileKey] = credsFile
}
bucket := strings.Trim(config["bucket"], "/")
prefix := strings.Trim(config["prefix"], "/")
@@ -555,7 +548,7 @@ func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repo
}
s3URL = url.Host
disableTLS = (url.Scheme == "http")
disableTLS = url.Scheme == "http"
}
result[udmrepo.StoreOptionS3Endpoint] = strings.Trim(s3URL, "/")
pkg/repository/provider/unified_repo_test.go
+20 -21
View File
@@ -168,7 +168,7 @@ func TestGetStorageCredentials(t *testing.T) {
},
},
credFileStore: new(credmock.FileStore),
expected: nil,
expected: map[string]string{},
},
{
name: "gcp, Credential section not exists in BSL",
@@ -437,12 +437,11 @@ func TestGetStorageVariables(t *testing.T) {
},
}
credFileStore := new(credmock.FileStore)
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
getS3BucketRegion = tc.getS3BucketRegion
actual, err := getStorageVariables(&tc.backupLocation, tc.repoBackend, tc.repoName, credFileStore)
actual, err := getStorageVariables(&tc.backupLocation, tc.repoBackend, tc.repoName)
require.Equal(t, tc.expected, actual)
@@ -531,7 +530,7 @@ func TestGetStoreOptions(t *testing.T) {
BackupRepo: &velerov1api.BackupRepository{},
},
funcTable: localFuncTable{
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
return map[string]string{}, errors.New("fake-error-2")
},
},
@@ -545,7 +544,7 @@ func TestGetStoreOptions(t *testing.T) {
BackupRepo: &velerov1api.BackupRepository{},
},
funcTable: localFuncTable{
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
return map[string]string{}, nil
},
getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -605,7 +604,7 @@ func TestPrepareRepo(t *testing.T) {
repoService: new(reposervicenmocks.BackupRepoService),
credStoreReturn: "fake-password",
funcTable: localFuncTable{
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
return map[string]string{}, errors.New("fake-store-option-error")
},
},
@@ -616,7 +615,7 @@ func TestPrepareRepo(t *testing.T) {
getter: new(credmock.SecretStore),
credStoreReturn: "fake-password",
funcTable: localFuncTable{
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
return map[string]string{}, nil
},
getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -636,7 +635,7 @@ func TestPrepareRepo(t *testing.T) {
getter: new(credmock.SecretStore),
credStoreReturn: "fake-password",
funcTable: localFuncTable{
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
return map[string]string{}, nil
},
getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -657,7 +656,7 @@ func TestPrepareRepo(t *testing.T) {
getter: new(credmock.SecretStore),
credStoreReturn: "fake-password",
funcTable: localFuncTable{
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
return map[string]string{}, nil
},
getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -734,7 +733,7 @@ func TestForget(t *testing.T) {
getter: new(credmock.SecretStore),
credStoreReturn: "fake-password",
funcTable: localFuncTable{
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
return map[string]string{}, nil
},
getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -758,7 +757,7 @@ func TestForget(t *testing.T) {
getter: new(credmock.SecretStore),
credStoreReturn: "fake-password",
funcTable: localFuncTable{
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
return map[string]string{}, nil
},
getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -786,7 +785,7 @@ func TestForget(t *testing.T) {
getter: new(credmock.SecretStore),
credStoreReturn: "fake-password",
funcTable: localFuncTable{
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
return map[string]string{}, nil
},
getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -878,7 +877,7 @@ func TestInitRepo(t *testing.T) {
getter: new(credmock.SecretStore),
credStoreReturn: "fake-password",
funcTable: localFuncTable{
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
return map[string]string{}, nil
},
getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -896,7 +895,7 @@ func TestInitRepo(t *testing.T) {
getter: new(credmock.SecretStore),
credStoreReturn: "fake-password",
funcTable: localFuncTable{
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
return map[string]string{}, nil
},
getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -966,7 +965,7 @@ func TestConnectToRepo(t *testing.T) {
getter: new(credmock.SecretStore),
credStoreReturn: "fake-password",
funcTable: localFuncTable{
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
return map[string]string{}, nil
},
getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -984,7 +983,7 @@ func TestConnectToRepo(t *testing.T) {
getter: new(credmock.SecretStore),
credStoreReturn: "fake-password",
funcTable: localFuncTable{
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
return map[string]string{}, nil
},
getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -1058,7 +1057,7 @@ func TestBoostRepoConnect(t *testing.T) {
getter: new(credmock.SecretStore),
credStoreReturn: "fake-password",
funcTable: localFuncTable{
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
return map[string]string{}, nil
},
getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -1085,7 +1084,7 @@ func TestBoostRepoConnect(t *testing.T) {
getter: new(credmock.SecretStore),
credStoreReturn: "fake-password",
funcTable: localFuncTable{
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
return map[string]string{}, nil
},
getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -1111,7 +1110,7 @@ func TestBoostRepoConnect(t *testing.T) {
getter: new(credmock.SecretStore),
credStoreReturn: "fake-password",
funcTable: localFuncTable{
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
return map[string]string{}, nil
},
getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -1198,7 +1197,7 @@ func TestPruneRepo(t *testing.T) {
getter: new(credmock.SecretStore),
credStoreReturn: "fake-password",
funcTable: localFuncTable{
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
return map[string]string{}, nil
},
getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {
@@ -1216,7 +1215,7 @@ func TestPruneRepo(t *testing.T) {
getter: new(credmock.SecretStore),
credStoreReturn: "fake-password",
funcTable: localFuncTable{
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, velerocredentials.FileStore) (map[string]string, error) {
getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string) (map[string]string, error) {
return map[string]string{}, nil
},
getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) {