From c47eab1a5db7292080f4bc03d3a6afac65df6c72 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 14 May 2026 10:51:04 -0700 Subject: [PATCH] admin: attach admin-signed Bearer token on filer IAM gRPC calls (#9498) * admin: attach admin-signed Bearer token on filer IAM gRPC calls PR #9442 added Bearer-JWT enforcement on the filer's IAM gRPC service but didn't update its only production client, IamGrpcStore. The admin UI Users/Groups pages went through that client and started failing in 4.24 with either Unimplemented (filer refuses to register the service when jwt.filer_signing.key is empty) or Unauthenticated (the client sent no token). Issues #9495 and #9496 both trace to this gap. Plumb jwt.filer_signing.key into IamGrpcStore via a new SetAdminSigning hook called from the admin server, and append a freshly minted Bearer token to outgoing metadata on every call. The mint helper security.GenJwtForFilerAdmin existed since #9442 but had no production caller; this wires it up. Add an integration test alongside grpc_store.go that runs a real IamGrpcServer over a real grpc.Server listener and exercises the store end-to-end: matching key succeeds, wrong key returns Unauthenticated, no key returns Unauthenticated. Without the client-side token attach the success path fails, so the regression cannot land again. * address review: include adminSigningExpiresAfterSec in mu comment --- weed/admin/dash/admin_server.go | 20 ++++ weed/credential/grpc/grpc_identity.go | 20 ++-- weed/credential/grpc/grpc_policy.go | 10 +- weed/credential/grpc/grpc_service_account.go | 12 +-- weed/credential/grpc/grpc_store.go | 44 +++++++- weed/credential/grpc/grpc_store_test.go | 108 +++++++++++++++++++ 6 files changed, 189 insertions(+), 25 deletions(-) create mode 100644 weed/credential/grpc/grpc_store_test.go diff --git a/weed/admin/dash/admin_server.go b/weed/admin/dash/admin_server.go index 478caaa94..ed856da3f 100644 --- a/weed/admin/dash/admin_server.go +++ b/weed/admin/dash/admin_server.go @@ -193,6 +193,26 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string, } else { glog.V(0).Infof("Credential store %s does not support filer address function", store.GetName()) } + + // The filer's IAM gRPC service rejects every RPC without an + // admin-signed Bearer token (PR #9442). Mirror the filer's + // jwt.filer_signing.key here so the admin UI's Users/Groups + // pages can talk to it; without this they fail with either + // Unimplemented (filer refuses to register the service) or + // Unauthenticated (missing authorization metadata). + if signer, ok := store.(interface { + SetAdminSigning(security.SigningKey, int) + }); ok { + viper := util.GetViper() + key := security.SigningKey(viper.GetString("jwt.filer_signing.key")) + expires := viper.GetInt("jwt.filer_signing.expires_after_seconds") + signer.SetAdminSigning(key, expires) + if len(key) == 0 { + glog.Warningf("jwt.filer_signing.key is empty in security.toml; the admin UI Users/Groups pages will fail until this is set on both the filer and the admin server") + } else { + glog.V(0).Infof("Credential store configured with admin Bearer token signing") + } + } } } diff --git a/weed/credential/grpc/grpc_identity.go b/weed/credential/grpc/grpc_identity.go index 72ca90365..831bad617 100644 --- a/weed/credential/grpc/grpc_identity.go +++ b/weed/credential/grpc/grpc_identity.go @@ -12,7 +12,7 @@ import ( func (store *IamGrpcStore) LoadConfiguration(ctx context.Context) (*iam_pb.S3ApiConfiguration, error) { var config *iam_pb.S3ApiConfiguration - err := store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + err := store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { resp, err := client.GetConfiguration(ctx, &iam_pb.GetConfigurationRequest{}) if err != nil { return err @@ -24,7 +24,7 @@ func (store *IamGrpcStore) LoadConfiguration(ctx context.Context) (*iam_pb.S3Api } func (store *IamGrpcStore) SaveConfiguration(ctx context.Context, config *iam_pb.S3ApiConfiguration) error { - return store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + return store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { _, err := client.PutConfiguration(ctx, &iam_pb.PutConfigurationRequest{ Configuration: config, }) @@ -33,7 +33,7 @@ func (store *IamGrpcStore) SaveConfiguration(ctx context.Context, config *iam_pb } func (store *IamGrpcStore) CreateUser(ctx context.Context, identity *iam_pb.Identity) error { - return store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + return store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { _, err := client.CreateUser(ctx, &iam_pb.CreateUserRequest{ Identity: identity, }) @@ -43,7 +43,7 @@ func (store *IamGrpcStore) CreateUser(ctx context.Context, identity *iam_pb.Iden func (store *IamGrpcStore) GetUser(ctx context.Context, username string) (*iam_pb.Identity, error) { var identity *iam_pb.Identity - err := store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + err := store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { resp, err := client.GetUser(ctx, &iam_pb.GetUserRequest{ Username: username, }) @@ -63,7 +63,7 @@ func (store *IamGrpcStore) GetUser(ctx context.Context, username string) (*iam_p } func (store *IamGrpcStore) UpdateUser(ctx context.Context, username string, identity *iam_pb.Identity) error { - return store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + return store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { _, err := client.UpdateUser(ctx, &iam_pb.UpdateUserRequest{ Username: username, Identity: identity, @@ -73,7 +73,7 @@ func (store *IamGrpcStore) UpdateUser(ctx context.Context, username string, iden } func (store *IamGrpcStore) DeleteUser(ctx context.Context, username string) error { - return store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + return store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { _, err := client.DeleteUser(ctx, &iam_pb.DeleteUserRequest{ Username: username, }) @@ -83,7 +83,7 @@ func (store *IamGrpcStore) DeleteUser(ctx context.Context, username string) erro func (store *IamGrpcStore) ListUsers(ctx context.Context) ([]string, error) { var usernames []string - err := store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + err := store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { resp, err := client.ListUsers(ctx, &iam_pb.ListUsersRequest{}) if err != nil { return err @@ -96,7 +96,7 @@ func (store *IamGrpcStore) ListUsers(ctx context.Context) ([]string, error) { func (store *IamGrpcStore) GetUserByAccessKey(ctx context.Context, accessKey string) (*iam_pb.Identity, error) { var identity *iam_pb.Identity - err := store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + err := store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { resp, err := client.GetUserByAccessKey(ctx, &iam_pb.GetUserByAccessKeyRequest{ AccessKey: accessKey, }) @@ -110,7 +110,7 @@ func (store *IamGrpcStore) GetUserByAccessKey(ctx context.Context, accessKey str } func (store *IamGrpcStore) CreateAccessKey(ctx context.Context, username string, credential *iam_pb.Credential) error { - return store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + return store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { _, err := client.CreateAccessKey(ctx, &iam_pb.CreateAccessKeyRequest{ Username: username, Credential: credential, @@ -120,7 +120,7 @@ func (store *IamGrpcStore) CreateAccessKey(ctx context.Context, username string, } func (store *IamGrpcStore) DeleteAccessKey(ctx context.Context, username string, accessKey string) error { - return store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + return store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { _, err := client.DeleteAccessKey(ctx, &iam_pb.DeleteAccessKeyRequest{ Username: username, AccessKey: accessKey, diff --git a/weed/credential/grpc/grpc_policy.go b/weed/credential/grpc/grpc_policy.go index 52ab9e773..4ed9ce3b8 100644 --- a/weed/credential/grpc/grpc_policy.go +++ b/weed/credential/grpc/grpc_policy.go @@ -13,7 +13,7 @@ import ( func (store *IamGrpcStore) GetPolicies(ctx context.Context) (map[string]policy_engine.PolicyDocument, error) { policies := make(map[string]policy_engine.PolicyDocument) - err := store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + err := store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { resp, err := client.ListPolicies(ctx, &iam_pb.ListPoliciesRequest{}) if err != nil { return err @@ -35,7 +35,7 @@ func (store *IamGrpcStore) PutPolicy(ctx context.Context, name string, document if err != nil { return err } - return store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + return store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { _, err := client.PutPolicy(ctx, &iam_pb.PutPolicyRequest{ Name: name, Content: string(content), @@ -45,7 +45,7 @@ func (store *IamGrpcStore) PutPolicy(ctx context.Context, name string, document } func (store *IamGrpcStore) DeletePolicy(ctx context.Context, name string) error { - return store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + return store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { _, err := client.DeletePolicy(ctx, &iam_pb.DeletePolicyRequest{ Name: name, }) @@ -55,7 +55,7 @@ func (store *IamGrpcStore) DeletePolicy(ctx context.Context, name string) error func (store *IamGrpcStore) GetPolicy(ctx context.Context, name string) (*policy_engine.PolicyDocument, error) { var doc policy_engine.PolicyDocument - err := store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + err := store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { resp, err := client.GetPolicy(ctx, &iam_pb.GetPolicyRequest{ Name: name, }) @@ -82,7 +82,7 @@ func (store *IamGrpcStore) CreatePolicy(ctx context.Context, name string, docume // ListPolicyNames retrieves names of all IAM policies via gRPC. func (store *IamGrpcStore) ListPolicyNames(ctx context.Context) ([]string, error) { var names []string - err := store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + err := store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { resp, err := client.ListPolicies(ctx, &iam_pb.ListPoliciesRequest{}) if err != nil { return err diff --git a/weed/credential/grpc/grpc_service_account.go b/weed/credential/grpc/grpc_service_account.go index 956b52daa..e8172e971 100644 --- a/weed/credential/grpc/grpc_service_account.go +++ b/weed/credential/grpc/grpc_service_account.go @@ -7,7 +7,7 @@ import ( ) func (store *IamGrpcStore) CreateServiceAccount(ctx context.Context, sa *iam_pb.ServiceAccount) error { - return store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + return store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { _, err := client.CreateServiceAccount(ctx, &iam_pb.CreateServiceAccountRequest{ ServiceAccount: sa, }) @@ -16,7 +16,7 @@ func (store *IamGrpcStore) CreateServiceAccount(ctx context.Context, sa *iam_pb. } func (store *IamGrpcStore) UpdateServiceAccount(ctx context.Context, id string, sa *iam_pb.ServiceAccount) error { - return store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + return store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { _, err := client.UpdateServiceAccount(ctx, &iam_pb.UpdateServiceAccountRequest{ Id: id, ServiceAccount: sa, @@ -26,7 +26,7 @@ func (store *IamGrpcStore) UpdateServiceAccount(ctx context.Context, id string, } func (store *IamGrpcStore) DeleteServiceAccount(ctx context.Context, id string) error { - return store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + return store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { _, err := client.DeleteServiceAccount(ctx, &iam_pb.DeleteServiceAccountRequest{ Id: id, }) @@ -36,7 +36,7 @@ func (store *IamGrpcStore) DeleteServiceAccount(ctx context.Context, id string) func (store *IamGrpcStore) GetServiceAccount(ctx context.Context, id string) (*iam_pb.ServiceAccount, error) { var sa *iam_pb.ServiceAccount - err := store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + err := store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { resp, err := client.GetServiceAccount(ctx, &iam_pb.GetServiceAccountRequest{ Id: id, }) @@ -51,7 +51,7 @@ func (store *IamGrpcStore) GetServiceAccount(ctx context.Context, id string) (*i func (store *IamGrpcStore) ListServiceAccounts(ctx context.Context) ([]*iam_pb.ServiceAccount, error) { var accounts []*iam_pb.ServiceAccount - err := store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + err := store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { resp, err := client.ListServiceAccounts(ctx, &iam_pb.ListServiceAccountsRequest{}) if err != nil { return err @@ -64,7 +64,7 @@ func (store *IamGrpcStore) ListServiceAccounts(ctx context.Context) ([]*iam_pb.S func (store *IamGrpcStore) GetServiceAccountByAccessKey(ctx context.Context, accessKey string) (*iam_pb.ServiceAccount, error) { var sa *iam_pb.ServiceAccount - err := store.withIamClient(func(client iam_pb.SeaweedIdentityAccessManagementClient) error { + err := store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error { resp, err := client.GetServiceAccountByAccessKey(ctx, &iam_pb.GetServiceAccountByAccessKeyRequest{ AccessKey: accessKey, }) diff --git a/weed/credential/grpc/grpc_store.go b/weed/credential/grpc/grpc_store.go index aa86cc915..9b59ba0a4 100644 --- a/weed/credential/grpc/grpc_store.go +++ b/weed/credential/grpc/grpc_store.go @@ -1,25 +1,37 @@ package grpc import ( + "context" "fmt" "sync" "github.com/seaweedfs/seaweedfs/weed/credential" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" + "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util" "google.golang.org/grpc" + "google.golang.org/grpc/metadata" ) func init() { credential.Stores = append(credential.Stores, &IamGrpcStore{}) } -// IamGrpcStore implements CredentialStore using SeaweedFS IAM gRPC service +// IamGrpcStore implements CredentialStore by calling the filer's IAM gRPC +// service. The filer requires an admin-signed Bearer token on every RPC +// (see weed/server/filer_server_handlers_iam_grpc.go); SetAdminSigning must +// be called with the same jwt.filer_signing.key value that the filer reads +// from security.toml, or every call will fail with Unauthenticated. type IamGrpcStore struct { filerAddressFunc func() pb.ServerAddress // Function to get current active filer grpcDialOption grpc.DialOption - mu sync.RWMutex // Protects filerAddressFunc and grpcDialOption + // adminSigningKey is the HS256 secret used to mint Bearer tokens that the + // filer's IAM gRPC service validates. Must match jwt.filer_signing.key on + // the filer side. Empty means no token is sent (the filer will reject). + adminSigningKey security.SigningKey + adminSigningExpiresAfterSec int + mu sync.RWMutex // Protects filerAddressFunc, grpcDialOption, adminSigningKey, and adminSigningExpiresAfterSec } func (store *IamGrpcStore) GetName() credential.CredentialStoreTypeName { @@ -47,7 +59,22 @@ func (store *IamGrpcStore) SetFilerAddressFunc(getFiler func() pb.ServerAddress, store.grpcDialOption = grpcDialOption } -func (store *IamGrpcStore) withIamClient(fn func(client iam_pb.SeaweedIdentityAccessManagementClient) error) error { +// SetAdminSigning configures the HS256 secret used to mint Bearer tokens for +// the filer's IAM gRPC service. The key must match jwt.filer_signing.key in +// the filer's security.toml. If expiresAfterSec is 0, tokens are minted +// without an exp claim. +func (store *IamGrpcStore) SetAdminSigning(key security.SigningKey, expiresAfterSec int) { + store.mu.Lock() + defer store.mu.Unlock() + store.adminSigningKey = key + store.adminSigningExpiresAfterSec = expiresAfterSec +} + +// withIamClient invokes fn against a (possibly cached) gRPC client to the +// filer's IAM service. If an admin signing key is configured the call attaches +// a freshly minted Bearer token via outgoing metadata; otherwise no auth +// header is sent and the filer will return Unauthenticated. +func (store *IamGrpcStore) withIamClient(ctx context.Context, fn func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error) error { store.mu.RLock() if store.filerAddressFunc == nil { store.mu.RUnlock() @@ -56,15 +83,24 @@ func (store *IamGrpcStore) withIamClient(fn func(client iam_pb.SeaweedIdentityAc filerAddress := store.filerAddressFunc() dialOption := store.grpcDialOption + signingKey := store.adminSigningKey + expiresAfterSec := store.adminSigningExpiresAfterSec store.mu.RUnlock() if filerAddress == "" { return fmt.Errorf("iam_grpc: no filer discovered yet") } + if len(signingKey) > 0 { + token := security.GenJwtForFilerAdmin(signingKey, expiresAfterSec) + if token != "" { + ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+string(token)) + } + } + return pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error { client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn) - return fn(client) + return fn(ctx, client) }, filerAddress.ToGrpcAddress(), false, dialOption) } diff --git a/weed/credential/grpc/grpc_store_test.go b/weed/credential/grpc/grpc_store_test.go new file mode 100644 index 000000000..c157cc143 --- /dev/null +++ b/weed/credential/grpc/grpc_store_test.go @@ -0,0 +1,108 @@ +package grpc + +import ( + "context" + "fmt" + "net" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/credential" + _ "github.com/seaweedfs/seaweedfs/weed/credential/memory" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" + "github.com/seaweedfs/seaweedfs/weed/security" + weed_server "github.com/seaweedfs/seaweedfs/weed/server" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" +) + +// TestIamGrpcStore_AdminBearerToken pins the contract that broke in 4.24: +// after PR #9442 the filer's IAM gRPC service requires an admin-signed +// Bearer token on every RPC, and IamGrpcStore is the only production +// client that talks to it (from the admin server). If the store doesn't +// mint and attach a token the admin UI Users/Groups pages fail with +// Unauthenticated; that's what issues #9495/#9496 reported. +func TestIamGrpcStore_AdminBearerToken(t *testing.T) { + const goodKey = "iam-admin-itest-key" + const wrongKey = "iam-admin-itest-key-different" + + // Real IamGrpcServer backed by an in-memory credential manager so we + // exercise the full handler (auth check + business logic), not a stub. + cm, err := credential.NewCredentialManager(credential.StoreTypeMemory, nil, "") + if err != nil { + t.Fatalf("NewCredentialManager: %v", err) + } + defer cm.Shutdown() + + iamSrv := weed_server.NewIamGrpcServer(cm, security.SigningKey(goodKey)) + + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + grpcSrv := grpc.NewServer() + iam_pb.RegisterSeaweedIdentityAccessManagementServer(grpcSrv, iamSrv) + go func() { _ = grpcSrv.Serve(lis) }() + t.Cleanup(grpcSrv.Stop) + + // pb.ServerAddress encodes the grpc port in the trailing ".N" segment. + // host:0 is the unused HTTP port; the listener's actual port is the + // gRPC one that ToGrpcAddress() unpacks for dialing. + _, portStr, err := net.SplitHostPort(lis.Addr().String()) + if err != nil { + t.Fatalf("split listener addr: %v", err) + } + serverAddr := pb.ServerAddress(fmt.Sprintf("127.0.0.1:0.%s", portStr)) + + newStore := func(key string) *IamGrpcStore { + s := &IamGrpcStore{} + s.SetFilerAddressFunc(func() pb.ServerAddress { return serverAddr }, grpc.WithTransportCredentials(insecure.NewCredentials())) + if key != "" { + s.SetAdminSigning(security.SigningKey(key), 0) + } + return s + } + + ctx := context.Background() + + t.Run("matching_key_succeeds", func(t *testing.T) { + store := newStore(goodKey) + if _, err := store.ListUsers(ctx); err != nil { + t.Fatalf("ListUsers with matching key: %v", err) + } + if err := store.CreateUser(ctx, &iam_pb.Identity{Name: "alice"}); err != nil { + t.Fatalf("CreateUser with matching key: %v", err) + } + got, err := store.GetUser(ctx, "alice") + if err != nil { + t.Fatalf("GetUser with matching key: %v", err) + } + if got == nil || got.Name != "alice" { + t.Fatalf("GetUser returned %+v, want name=alice", got) + } + }) + + t.Run("wrong_key_unauthenticated", func(t *testing.T) { + store := newStore(wrongKey) + _, err := store.ListUsers(ctx) + if got := status.Code(err); got != codes.Unauthenticated { + t.Fatalf("ListUsers with wrong key: got code %v err=%v, want Unauthenticated", got, err) + } + }) + + t.Run("no_key_unauthenticated", func(t *testing.T) { + store := newStore("") + _, err := store.ListUsers(ctx) + // Server reports "missing authorization metadata" when no token is + // attached. Either way the gRPC status code is Unauthenticated. + if got := status.Code(err); got != codes.Unauthenticated { + t.Fatalf("ListUsers with no key: got code %v err=%v, want Unauthenticated", got, err) + } + if err != nil && !strings.Contains(err.Error(), "authorization") { + t.Fatalf("ListUsers with no key: error message %q does not mention authorization", err) + } + }) +}