From 95c65f4e8f889bbf5b000d097cb2a2a6b5196377 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 22 Apr 2024 10:49:30 -0700 Subject: [PATCH] do not panic on rebalance during server restarts (#19563) This PR makes a feasible approach to handle all the scenarios that we must face to avoid returning "panic." Instead, we must return "errServerNotInitialized" when a bucketMetadataSys.Get() is called, allowing the caller to retry their operation and wait. Bonus fix the way data-usage-cache stores the object. Instead of storing usage-cache.bin with the bucket as `.minio.sys/buckets`, the `buckets` must be relative to the bucket `.minio.sys` as part of the object name. Otherwise, there is no way to decommission entries at `.minio.sys/buckets` and their final erasure set positions. A bucket must never have a `/` in it. Adds code to read() from existing data-usage.bin upon upgrade. --- cmd/api-errors.go | 12 +- cmd/apierrorcode_string.go | 313 ++++++++++++++------------- cmd/bucket-metadata-sys.go | 24 +- cmd/bucket-object-lock.go | 1 - cmd/bucket-replication.go | 27 ++- cmd/bucket-versioning-handler.go | 2 +- cmd/data-scanner.go | 34 ++- cmd/data-usage-cache.go | 33 ++- cmd/erasure-object.go | 15 +- cmd/erasure-server-pool-decom.go | 71 ++++-- cmd/erasure-server-pool-rebalance.go | 46 +++- cmd/global-heal.go | 32 ++- cmd/object-handlers.go | 28 ++- cmd/os-reliable.go | 10 + 14 files changed, 418 insertions(+), 230 deletions(-) diff --git a/cmd/api-errors.go b/cmd/api-errors.go index 677656dde..c4aeb9a6a 100644 --- a/cmd/api-errors.go +++ b/cmd/api-errors.go @@ -263,6 +263,7 @@ const ( ErrInvalidResourceName ErrInvalidLifecycleQueryParameter ErrServerNotInitialized + ErrBucketMetadataNotInitialized ErrRequestTimedout ErrClientDisconnected ErrTooManyRequests @@ -1295,7 +1296,12 @@ var errorCodes = errorCodeMap{ }, ErrServerNotInitialized: { Code: "XMinioServerNotInitialized", - Description: "Server not initialized, please try again.", + Description: "Server not initialized yet, please try again.", + HTTPStatusCode: http.StatusServiceUnavailable, + }, + ErrBucketMetadataNotInitialized: { + Code: "XMinioBucketMetadataNotInitialized", + Description: "Bucket metadata not initialized yet, please try again.", HTTPStatusCode: http.StatusServiceUnavailable, }, ErrMalformedJSON: { @@ -2211,6 +2217,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) { apiErr = ErrInvalidMaxParts case ioutil.ErrOverread: apiErr = ErrExcessData + case errServerNotInitialized: + apiErr = ErrServerNotInitialized + case errBucketMetadataNotInitialized: + apiErr = ErrBucketMetadataNotInitialized } // Compression errors diff --git a/cmd/apierrorcode_string.go b/cmd/apierrorcode_string.go index b01cf3d52..defe739f3 100644 --- a/cmd/apierrorcode_string.go +++ b/cmd/apierrorcode_string.go @@ -178,165 +178,166 @@ func _() { _ = x[ErrInvalidResourceName-167] _ = x[ErrInvalidLifecycleQueryParameter-168] _ = x[ErrServerNotInitialized-169] - _ = x[ErrRequestTimedout-170] - _ = x[ErrClientDisconnected-171] - _ = x[ErrTooManyRequests-172] - _ = x[ErrInvalidRequest-173] - _ = x[ErrTransitionStorageClassNotFoundError-174] - _ = x[ErrInvalidStorageClass-175] - _ = x[ErrBackendDown-176] - _ = x[ErrMalformedJSON-177] - _ = x[ErrAdminNoSuchUser-178] - _ = x[ErrAdminNoSuchUserLDAPWarn-179] - _ = x[ErrAdminLDAPExpectedLoginName-180] - _ = x[ErrAdminNoSuchGroup-181] - _ = x[ErrAdminGroupNotEmpty-182] - _ = x[ErrAdminGroupDisabled-183] - _ = x[ErrAdminNoSuchJob-184] - _ = x[ErrAdminNoSuchPolicy-185] - _ = x[ErrAdminPolicyChangeAlreadyApplied-186] - _ = x[ErrAdminInvalidArgument-187] - _ = x[ErrAdminInvalidAccessKey-188] - _ = x[ErrAdminInvalidSecretKey-189] - _ = x[ErrAdminConfigNoQuorum-190] - _ = x[ErrAdminConfigTooLarge-191] - _ = x[ErrAdminConfigBadJSON-192] - _ = x[ErrAdminNoSuchConfigTarget-193] - _ = x[ErrAdminConfigEnvOverridden-194] - _ = x[ErrAdminConfigDuplicateKeys-195] - _ = x[ErrAdminConfigInvalidIDPType-196] - _ = x[ErrAdminConfigLDAPNonDefaultConfigName-197] - _ = x[ErrAdminConfigLDAPValidation-198] - _ = x[ErrAdminConfigIDPCfgNameAlreadyExists-199] - _ = x[ErrAdminConfigIDPCfgNameDoesNotExist-200] - _ = x[ErrInsecureClientRequest-201] - _ = x[ErrObjectTampered-202] - _ = x[ErrAdminLDAPNotEnabled-203] - _ = x[ErrSiteReplicationInvalidRequest-204] - _ = x[ErrSiteReplicationPeerResp-205] - _ = x[ErrSiteReplicationBackendIssue-206] - _ = x[ErrSiteReplicationServiceAccountError-207] - _ = x[ErrSiteReplicationBucketConfigError-208] - _ = x[ErrSiteReplicationBucketMetaError-209] - _ = x[ErrSiteReplicationIAMError-210] - _ = x[ErrSiteReplicationConfigMissing-211] - _ = x[ErrSiteReplicationIAMConfigMismatch-212] - _ = x[ErrAdminRebalanceAlreadyStarted-213] - _ = x[ErrAdminRebalanceNotStarted-214] - _ = x[ErrAdminBucketQuotaExceeded-215] - _ = x[ErrAdminNoSuchQuotaConfiguration-216] - _ = x[ErrHealNotImplemented-217] - _ = x[ErrHealNoSuchProcess-218] - _ = x[ErrHealInvalidClientToken-219] - _ = x[ErrHealMissingBucket-220] - _ = x[ErrHealAlreadyRunning-221] - _ = x[ErrHealOverlappingPaths-222] - _ = x[ErrIncorrectContinuationToken-223] - _ = x[ErrEmptyRequestBody-224] - _ = x[ErrUnsupportedFunction-225] - _ = x[ErrInvalidExpressionType-226] - _ = x[ErrBusy-227] - _ = x[ErrUnauthorizedAccess-228] - _ = x[ErrExpressionTooLong-229] - _ = x[ErrIllegalSQLFunctionArgument-230] - _ = x[ErrInvalidKeyPath-231] - _ = x[ErrInvalidCompressionFormat-232] - _ = x[ErrInvalidFileHeaderInfo-233] - _ = x[ErrInvalidJSONType-234] - _ = x[ErrInvalidQuoteFields-235] - _ = x[ErrInvalidRequestParameter-236] - _ = x[ErrInvalidDataType-237] - _ = x[ErrInvalidTextEncoding-238] - _ = x[ErrInvalidDataSource-239] - _ = x[ErrInvalidTableAlias-240] - _ = x[ErrMissingRequiredParameter-241] - _ = x[ErrObjectSerializationConflict-242] - _ = x[ErrUnsupportedSQLOperation-243] - _ = x[ErrUnsupportedSQLStructure-244] - _ = x[ErrUnsupportedSyntax-245] - _ = x[ErrUnsupportedRangeHeader-246] - _ = x[ErrLexerInvalidChar-247] - _ = x[ErrLexerInvalidOperator-248] - _ = x[ErrLexerInvalidLiteral-249] - _ = x[ErrLexerInvalidIONLiteral-250] - _ = x[ErrParseExpectedDatePart-251] - _ = x[ErrParseExpectedKeyword-252] - _ = x[ErrParseExpectedTokenType-253] - _ = x[ErrParseExpected2TokenTypes-254] - _ = x[ErrParseExpectedNumber-255] - _ = x[ErrParseExpectedRightParenBuiltinFunctionCall-256] - _ = x[ErrParseExpectedTypeName-257] - _ = x[ErrParseExpectedWhenClause-258] - _ = x[ErrParseUnsupportedToken-259] - _ = x[ErrParseUnsupportedLiteralsGroupBy-260] - _ = x[ErrParseExpectedMember-261] - _ = x[ErrParseUnsupportedSelect-262] - _ = x[ErrParseUnsupportedCase-263] - _ = x[ErrParseUnsupportedCaseClause-264] - _ = x[ErrParseUnsupportedAlias-265] - _ = x[ErrParseUnsupportedSyntax-266] - _ = x[ErrParseUnknownOperator-267] - _ = x[ErrParseMissingIdentAfterAt-268] - _ = x[ErrParseUnexpectedOperator-269] - _ = x[ErrParseUnexpectedTerm-270] - _ = x[ErrParseUnexpectedToken-271] - _ = x[ErrParseUnexpectedKeyword-272] - _ = x[ErrParseExpectedExpression-273] - _ = x[ErrParseExpectedLeftParenAfterCast-274] - _ = x[ErrParseExpectedLeftParenValueConstructor-275] - _ = x[ErrParseExpectedLeftParenBuiltinFunctionCall-276] - _ = x[ErrParseExpectedArgumentDelimiter-277] - _ = x[ErrParseCastArity-278] - _ = x[ErrParseInvalidTypeParam-279] - _ = x[ErrParseEmptySelect-280] - _ = x[ErrParseSelectMissingFrom-281] - _ = x[ErrParseExpectedIdentForGroupName-282] - _ = x[ErrParseExpectedIdentForAlias-283] - _ = x[ErrParseUnsupportedCallWithStar-284] - _ = x[ErrParseNonUnaryAggregateFunctionCall-285] - _ = x[ErrParseMalformedJoin-286] - _ = x[ErrParseExpectedIdentForAt-287] - _ = x[ErrParseAsteriskIsNotAloneInSelectList-288] - _ = x[ErrParseCannotMixSqbAndWildcardInSelectList-289] - _ = x[ErrParseInvalidContextForWildcardInSelectList-290] - _ = x[ErrIncorrectSQLFunctionArgumentType-291] - _ = x[ErrValueParseFailure-292] - _ = x[ErrEvaluatorInvalidArguments-293] - _ = x[ErrIntegerOverflow-294] - _ = x[ErrLikeInvalidInputs-295] - _ = x[ErrCastFailed-296] - _ = x[ErrInvalidCast-297] - _ = x[ErrEvaluatorInvalidTimestampFormatPattern-298] - _ = x[ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing-299] - _ = x[ErrEvaluatorTimestampFormatPatternDuplicateFields-300] - _ = x[ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch-301] - _ = x[ErrEvaluatorUnterminatedTimestampFormatPatternToken-302] - _ = x[ErrEvaluatorInvalidTimestampFormatPatternToken-303] - _ = x[ErrEvaluatorInvalidTimestampFormatPatternSymbol-304] - _ = x[ErrEvaluatorBindingDoesNotExist-305] - _ = x[ErrMissingHeaders-306] - _ = x[ErrInvalidColumnIndex-307] - _ = x[ErrAdminConfigNotificationTargetsFailed-308] - _ = x[ErrAdminProfilerNotEnabled-309] - _ = x[ErrInvalidDecompressedSize-310] - _ = x[ErrAddUserInvalidArgument-311] - _ = x[ErrAdminResourceInvalidArgument-312] - _ = x[ErrAdminAccountNotEligible-313] - _ = x[ErrAccountNotEligible-314] - _ = x[ErrAdminServiceAccountNotFound-315] - _ = x[ErrPostPolicyConditionInvalidFormat-316] - _ = x[ErrInvalidChecksum-317] - _ = x[ErrLambdaARNInvalid-318] - _ = x[ErrLambdaARNNotFound-319] - _ = x[ErrInvalidAttributeName-320] - _ = x[ErrAdminNoAccessKey-321] - _ = x[ErrAdminNoSecretKey-322] - _ = x[apiErrCodeEnd-323] + _ = x[ErrBucketMetadataNotInitialized-170] + _ = x[ErrRequestTimedout-171] + _ = x[ErrClientDisconnected-172] + _ = x[ErrTooManyRequests-173] + _ = x[ErrInvalidRequest-174] + _ = x[ErrTransitionStorageClassNotFoundError-175] + _ = x[ErrInvalidStorageClass-176] + _ = x[ErrBackendDown-177] + _ = x[ErrMalformedJSON-178] + _ = x[ErrAdminNoSuchUser-179] + _ = x[ErrAdminNoSuchUserLDAPWarn-180] + _ = x[ErrAdminLDAPExpectedLoginName-181] + _ = x[ErrAdminNoSuchGroup-182] + _ = x[ErrAdminGroupNotEmpty-183] + _ = x[ErrAdminGroupDisabled-184] + _ = x[ErrAdminNoSuchJob-185] + _ = x[ErrAdminNoSuchPolicy-186] + _ = x[ErrAdminPolicyChangeAlreadyApplied-187] + _ = x[ErrAdminInvalidArgument-188] + _ = x[ErrAdminInvalidAccessKey-189] + _ = x[ErrAdminInvalidSecretKey-190] + _ = x[ErrAdminConfigNoQuorum-191] + _ = x[ErrAdminConfigTooLarge-192] + _ = x[ErrAdminConfigBadJSON-193] + _ = x[ErrAdminNoSuchConfigTarget-194] + _ = x[ErrAdminConfigEnvOverridden-195] + _ = x[ErrAdminConfigDuplicateKeys-196] + _ = x[ErrAdminConfigInvalidIDPType-197] + _ = x[ErrAdminConfigLDAPNonDefaultConfigName-198] + _ = x[ErrAdminConfigLDAPValidation-199] + _ = x[ErrAdminConfigIDPCfgNameAlreadyExists-200] + _ = x[ErrAdminConfigIDPCfgNameDoesNotExist-201] + _ = x[ErrInsecureClientRequest-202] + _ = x[ErrObjectTampered-203] + _ = x[ErrAdminLDAPNotEnabled-204] + _ = x[ErrSiteReplicationInvalidRequest-205] + _ = x[ErrSiteReplicationPeerResp-206] + _ = x[ErrSiteReplicationBackendIssue-207] + _ = x[ErrSiteReplicationServiceAccountError-208] + _ = x[ErrSiteReplicationBucketConfigError-209] + _ = x[ErrSiteReplicationBucketMetaError-210] + _ = x[ErrSiteReplicationIAMError-211] + _ = x[ErrSiteReplicationConfigMissing-212] + _ = x[ErrSiteReplicationIAMConfigMismatch-213] + _ = x[ErrAdminRebalanceAlreadyStarted-214] + _ = x[ErrAdminRebalanceNotStarted-215] + _ = x[ErrAdminBucketQuotaExceeded-216] + _ = x[ErrAdminNoSuchQuotaConfiguration-217] + _ = x[ErrHealNotImplemented-218] + _ = x[ErrHealNoSuchProcess-219] + _ = x[ErrHealInvalidClientToken-220] + _ = x[ErrHealMissingBucket-221] + _ = x[ErrHealAlreadyRunning-222] + _ = x[ErrHealOverlappingPaths-223] + _ = x[ErrIncorrectContinuationToken-224] + _ = x[ErrEmptyRequestBody-225] + _ = x[ErrUnsupportedFunction-226] + _ = x[ErrInvalidExpressionType-227] + _ = x[ErrBusy-228] + _ = x[ErrUnauthorizedAccess-229] + _ = x[ErrExpressionTooLong-230] + _ = x[ErrIllegalSQLFunctionArgument-231] + _ = x[ErrInvalidKeyPath-232] + _ = x[ErrInvalidCompressionFormat-233] + _ = x[ErrInvalidFileHeaderInfo-234] + _ = x[ErrInvalidJSONType-235] + _ = x[ErrInvalidQuoteFields-236] + _ = x[ErrInvalidRequestParameter-237] + _ = x[ErrInvalidDataType-238] + _ = x[ErrInvalidTextEncoding-239] + _ = x[ErrInvalidDataSource-240] + _ = x[ErrInvalidTableAlias-241] + _ = x[ErrMissingRequiredParameter-242] + _ = x[ErrObjectSerializationConflict-243] + _ = x[ErrUnsupportedSQLOperation-244] + _ = x[ErrUnsupportedSQLStructure-245] + _ = x[ErrUnsupportedSyntax-246] + _ = x[ErrUnsupportedRangeHeader-247] + _ = x[ErrLexerInvalidChar-248] + _ = x[ErrLexerInvalidOperator-249] + _ = x[ErrLexerInvalidLiteral-250] + _ = x[ErrLexerInvalidIONLiteral-251] + _ = x[ErrParseExpectedDatePart-252] + _ = x[ErrParseExpectedKeyword-253] + _ = x[ErrParseExpectedTokenType-254] + _ = x[ErrParseExpected2TokenTypes-255] + _ = x[ErrParseExpectedNumber-256] + _ = x[ErrParseExpectedRightParenBuiltinFunctionCall-257] + _ = x[ErrParseExpectedTypeName-258] + _ = x[ErrParseExpectedWhenClause-259] + _ = x[ErrParseUnsupportedToken-260] + _ = x[ErrParseUnsupportedLiteralsGroupBy-261] + _ = x[ErrParseExpectedMember-262] + _ = x[ErrParseUnsupportedSelect-263] + _ = x[ErrParseUnsupportedCase-264] + _ = x[ErrParseUnsupportedCaseClause-265] + _ = x[ErrParseUnsupportedAlias-266] + _ = x[ErrParseUnsupportedSyntax-267] + _ = x[ErrParseUnknownOperator-268] + _ = x[ErrParseMissingIdentAfterAt-269] + _ = x[ErrParseUnexpectedOperator-270] + _ = x[ErrParseUnexpectedTerm-271] + _ = x[ErrParseUnexpectedToken-272] + _ = x[ErrParseUnexpectedKeyword-273] + _ = x[ErrParseExpectedExpression-274] + _ = x[ErrParseExpectedLeftParenAfterCast-275] + _ = x[ErrParseExpectedLeftParenValueConstructor-276] + _ = x[ErrParseExpectedLeftParenBuiltinFunctionCall-277] + _ = x[ErrParseExpectedArgumentDelimiter-278] + _ = x[ErrParseCastArity-279] + _ = x[ErrParseInvalidTypeParam-280] + _ = x[ErrParseEmptySelect-281] + _ = x[ErrParseSelectMissingFrom-282] + _ = x[ErrParseExpectedIdentForGroupName-283] + _ = x[ErrParseExpectedIdentForAlias-284] + _ = x[ErrParseUnsupportedCallWithStar-285] + _ = x[ErrParseNonUnaryAggregateFunctionCall-286] + _ = x[ErrParseMalformedJoin-287] + _ = x[ErrParseExpectedIdentForAt-288] + _ = x[ErrParseAsteriskIsNotAloneInSelectList-289] + _ = x[ErrParseCannotMixSqbAndWildcardInSelectList-290] + _ = x[ErrParseInvalidContextForWildcardInSelectList-291] + _ = x[ErrIncorrectSQLFunctionArgumentType-292] + _ = x[ErrValueParseFailure-293] + _ = x[ErrEvaluatorInvalidArguments-294] + _ = x[ErrIntegerOverflow-295] + _ = x[ErrLikeInvalidInputs-296] + _ = x[ErrCastFailed-297] + _ = x[ErrInvalidCast-298] + _ = x[ErrEvaluatorInvalidTimestampFormatPattern-299] + _ = x[ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing-300] + _ = x[ErrEvaluatorTimestampFormatPatternDuplicateFields-301] + _ = x[ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch-302] + _ = x[ErrEvaluatorUnterminatedTimestampFormatPatternToken-303] + _ = x[ErrEvaluatorInvalidTimestampFormatPatternToken-304] + _ = x[ErrEvaluatorInvalidTimestampFormatPatternSymbol-305] + _ = x[ErrEvaluatorBindingDoesNotExist-306] + _ = x[ErrMissingHeaders-307] + _ = x[ErrInvalidColumnIndex-308] + _ = x[ErrAdminConfigNotificationTargetsFailed-309] + _ = x[ErrAdminProfilerNotEnabled-310] + _ = x[ErrInvalidDecompressedSize-311] + _ = x[ErrAddUserInvalidArgument-312] + _ = x[ErrAdminResourceInvalidArgument-313] + _ = x[ErrAdminAccountNotEligible-314] + _ = x[ErrAccountNotEligible-315] + _ = x[ErrAdminServiceAccountNotFound-316] + _ = x[ErrPostPolicyConditionInvalidFormat-317] + _ = x[ErrInvalidChecksum-318] + _ = x[ErrLambdaARNInvalid-319] + _ = x[ErrLambdaARNNotFound-320] + _ = x[ErrInvalidAttributeName-321] + _ = x[ErrAdminNoAccessKey-322] + _ = x[ErrAdminNoSecretKey-323] + _ = x[apiErrCodeEnd-324] } -const _APIErrorCode_name = "NoneAccessDeniedBadDigestEntityTooSmallEntityTooLargePolicyTooLargeIncompleteBodyInternalErrorInvalidAccessKeyIDAccessKeyDisabledInvalidArgumentInvalidBucketNameInvalidDigestInvalidRangeInvalidRangePartNumberInvalidCopyPartRangeInvalidCopyPartRangeSourceInvalidMaxKeysInvalidEncodingMethodInvalidMaxUploadsInvalidMaxPartsInvalidPartNumberMarkerInvalidPartNumberInvalidRequestBodyInvalidCopySourceInvalidMetadataDirectiveInvalidCopyDestInvalidPolicyDocumentInvalidObjectStateMalformedXMLMissingContentLengthMissingContentMD5MissingRequestBodyErrorMissingSecurityHeaderNoSuchBucketNoSuchBucketPolicyNoSuchBucketLifecycleNoSuchLifecycleConfigurationInvalidLifecycleWithObjectLockNoSuchBucketSSEConfigNoSuchCORSConfigurationNoSuchWebsiteConfigurationReplicationConfigurationNotFoundErrorRemoteDestinationNotFoundErrorReplicationDestinationMissingLockRemoteTargetNotFoundErrorReplicationRemoteConnectionErrorReplicationBandwidthLimitErrorBucketRemoteIdenticalToSourceBucketRemoteAlreadyExistsBucketRemoteLabelInUseBucketRemoteArnTypeInvalidBucketRemoteArnInvalidBucketRemoteRemoveDisallowedRemoteTargetNotVersionedErrorReplicationSourceNotVersionedErrorReplicationNeedsVersioningErrorReplicationBucketNeedsVersioningErrorReplicationDenyEditErrorRemoteTargetDenyAddErrorReplicationNoExistingObjectsReplicationValidationErrorReplicationPermissionCheckErrorObjectRestoreAlreadyInProgressNoSuchKeyNoSuchUploadInvalidVersionIDNoSuchVersionNotImplementedPreconditionFailedRequestTimeTooSkewedSignatureDoesNotMatchMethodNotAllowedInvalidPartInvalidPartOrderMissingPartAuthorizationHeaderMalformedMalformedPOSTRequestPOSTFileRequiredSignatureVersionNotSupportedBucketNotEmptyAllAccessDisabledPolicyInvalidVersionMissingFieldsMissingCredTagCredMalformedInvalidRegionInvalidServiceS3InvalidServiceSTSInvalidRequestVersionMissingSignTagMissingSignHeadersTagMalformedDateMalformedPresignedDateMalformedCredentialDateMalformedExpiresNegativeExpiresAuthHeaderEmptyExpiredPresignRequestRequestNotReadyYetUnsignedHeadersMissingDateHeaderInvalidQuerySignatureAlgoInvalidQueryParamsBucketAlreadyOwnedByYouInvalidDurationBucketAlreadyExistsMetadataTooLargeUnsupportedMetadataUnsupportedHostHeaderMaximumExpiresSlowDownReadSlowDownWriteMaxVersionsExceededInvalidPrefixMarkerBadRequestKeyTooLongErrorInvalidBucketObjectLockConfigurationObjectLockConfigurationNotFoundObjectLockConfigurationNotAllowedNoSuchObjectLockConfigurationObjectLockedInvalidRetentionDatePastObjectLockRetainDateUnknownWORMModeDirectiveBucketTaggingNotFoundObjectLockInvalidHeadersInvalidTagDirectivePolicyAlreadyAttachedPolicyNotAttachedExcessDataInvalidEncryptionMethodInvalidEncryptionKeyIDInsecureSSECustomerRequestSSEMultipartEncryptedSSEEncryptedObjectInvalidEncryptionParametersInvalidEncryptionParametersSSECInvalidSSECustomerAlgorithmInvalidSSECustomerKeyMissingSSECustomerKeyMissingSSECustomerKeyMD5SSECustomerKeyMD5MismatchInvalidSSECustomerParametersIncompatibleEncryptionMethodKMSNotConfiguredKMSKeyNotFoundExceptionKMSDefaultKeyAlreadyConfiguredNoAccessKeyInvalidTokenEventNotificationARNNotificationRegionNotificationOverlappingFilterNotificationFilterNameInvalidFilterNamePrefixFilterNameSuffixFilterValueInvalidOverlappingConfigsUnsupportedNotificationContentSHA256MismatchContentChecksumMismatchStorageFullRequestBodyParseObjectExistsAsDirectoryInvalidObjectNameInvalidObjectNamePrefixSlashInvalidResourceNameInvalidLifecycleQueryParameterServerNotInitializedRequestTimedoutClientDisconnectedTooManyRequestsInvalidRequestTransitionStorageClassNotFoundErrorInvalidStorageClassBackendDownMalformedJSONAdminNoSuchUserAdminNoSuchUserLDAPWarnAdminLDAPExpectedLoginNameAdminNoSuchGroupAdminGroupNotEmptyAdminGroupDisabledAdminNoSuchJobAdminNoSuchPolicyAdminPolicyChangeAlreadyAppliedAdminInvalidArgumentAdminInvalidAccessKeyAdminInvalidSecretKeyAdminConfigNoQuorumAdminConfigTooLargeAdminConfigBadJSONAdminNoSuchConfigTargetAdminConfigEnvOverriddenAdminConfigDuplicateKeysAdminConfigInvalidIDPTypeAdminConfigLDAPNonDefaultConfigNameAdminConfigLDAPValidationAdminConfigIDPCfgNameAlreadyExistsAdminConfigIDPCfgNameDoesNotExistInsecureClientRequestObjectTamperedAdminLDAPNotEnabledSiteReplicationInvalidRequestSiteReplicationPeerRespSiteReplicationBackendIssueSiteReplicationServiceAccountErrorSiteReplicationBucketConfigErrorSiteReplicationBucketMetaErrorSiteReplicationIAMErrorSiteReplicationConfigMissingSiteReplicationIAMConfigMismatchAdminRebalanceAlreadyStartedAdminRebalanceNotStartedAdminBucketQuotaExceededAdminNoSuchQuotaConfigurationHealNotImplementedHealNoSuchProcessHealInvalidClientTokenHealMissingBucketHealAlreadyRunningHealOverlappingPathsIncorrectContinuationTokenEmptyRequestBodyUnsupportedFunctionInvalidExpressionTypeBusyUnauthorizedAccessExpressionTooLongIllegalSQLFunctionArgumentInvalidKeyPathInvalidCompressionFormatInvalidFileHeaderInfoInvalidJSONTypeInvalidQuoteFieldsInvalidRequestParameterInvalidDataTypeInvalidTextEncodingInvalidDataSourceInvalidTableAliasMissingRequiredParameterObjectSerializationConflictUnsupportedSQLOperationUnsupportedSQLStructureUnsupportedSyntaxUnsupportedRangeHeaderLexerInvalidCharLexerInvalidOperatorLexerInvalidLiteralLexerInvalidIONLiteralParseExpectedDatePartParseExpectedKeywordParseExpectedTokenTypeParseExpected2TokenTypesParseExpectedNumberParseExpectedRightParenBuiltinFunctionCallParseExpectedTypeNameParseExpectedWhenClauseParseUnsupportedTokenParseUnsupportedLiteralsGroupByParseExpectedMemberParseUnsupportedSelectParseUnsupportedCaseParseUnsupportedCaseClauseParseUnsupportedAliasParseUnsupportedSyntaxParseUnknownOperatorParseMissingIdentAfterAtParseUnexpectedOperatorParseUnexpectedTermParseUnexpectedTokenParseUnexpectedKeywordParseExpectedExpressionParseExpectedLeftParenAfterCastParseExpectedLeftParenValueConstructorParseExpectedLeftParenBuiltinFunctionCallParseExpectedArgumentDelimiterParseCastArityParseInvalidTypeParamParseEmptySelectParseSelectMissingFromParseExpectedIdentForGroupNameParseExpectedIdentForAliasParseUnsupportedCallWithStarParseNonUnaryAggregateFunctionCallParseMalformedJoinParseExpectedIdentForAtParseAsteriskIsNotAloneInSelectListParseCannotMixSqbAndWildcardInSelectListParseInvalidContextForWildcardInSelectListIncorrectSQLFunctionArgumentTypeValueParseFailureEvaluatorInvalidArgumentsIntegerOverflowLikeInvalidInputsCastFailedInvalidCastEvaluatorInvalidTimestampFormatPatternEvaluatorInvalidTimestampFormatPatternSymbolForParsingEvaluatorTimestampFormatPatternDuplicateFieldsEvaluatorTimestampFormatPatternHourClockAmPmMismatchEvaluatorUnterminatedTimestampFormatPatternTokenEvaluatorInvalidTimestampFormatPatternTokenEvaluatorInvalidTimestampFormatPatternSymbolEvaluatorBindingDoesNotExistMissingHeadersInvalidColumnIndexAdminConfigNotificationTargetsFailedAdminProfilerNotEnabledInvalidDecompressedSizeAddUserInvalidArgumentAdminResourceInvalidArgumentAdminAccountNotEligibleAccountNotEligibleAdminServiceAccountNotFoundPostPolicyConditionInvalidFormatInvalidChecksumLambdaARNInvalidLambdaARNNotFoundInvalidAttributeNameAdminNoAccessKeyAdminNoSecretKeyapiErrCodeEnd" +const _APIErrorCode_name = "NoneAccessDeniedBadDigestEntityTooSmallEntityTooLargePolicyTooLargeIncompleteBodyInternalErrorInvalidAccessKeyIDAccessKeyDisabledInvalidArgumentInvalidBucketNameInvalidDigestInvalidRangeInvalidRangePartNumberInvalidCopyPartRangeInvalidCopyPartRangeSourceInvalidMaxKeysInvalidEncodingMethodInvalidMaxUploadsInvalidMaxPartsInvalidPartNumberMarkerInvalidPartNumberInvalidRequestBodyInvalidCopySourceInvalidMetadataDirectiveInvalidCopyDestInvalidPolicyDocumentInvalidObjectStateMalformedXMLMissingContentLengthMissingContentMD5MissingRequestBodyErrorMissingSecurityHeaderNoSuchBucketNoSuchBucketPolicyNoSuchBucketLifecycleNoSuchLifecycleConfigurationInvalidLifecycleWithObjectLockNoSuchBucketSSEConfigNoSuchCORSConfigurationNoSuchWebsiteConfigurationReplicationConfigurationNotFoundErrorRemoteDestinationNotFoundErrorReplicationDestinationMissingLockRemoteTargetNotFoundErrorReplicationRemoteConnectionErrorReplicationBandwidthLimitErrorBucketRemoteIdenticalToSourceBucketRemoteAlreadyExistsBucketRemoteLabelInUseBucketRemoteArnTypeInvalidBucketRemoteArnInvalidBucketRemoteRemoveDisallowedRemoteTargetNotVersionedErrorReplicationSourceNotVersionedErrorReplicationNeedsVersioningErrorReplicationBucketNeedsVersioningErrorReplicationDenyEditErrorRemoteTargetDenyAddErrorReplicationNoExistingObjectsReplicationValidationErrorReplicationPermissionCheckErrorObjectRestoreAlreadyInProgressNoSuchKeyNoSuchUploadInvalidVersionIDNoSuchVersionNotImplementedPreconditionFailedRequestTimeTooSkewedSignatureDoesNotMatchMethodNotAllowedInvalidPartInvalidPartOrderMissingPartAuthorizationHeaderMalformedMalformedPOSTRequestPOSTFileRequiredSignatureVersionNotSupportedBucketNotEmptyAllAccessDisabledPolicyInvalidVersionMissingFieldsMissingCredTagCredMalformedInvalidRegionInvalidServiceS3InvalidServiceSTSInvalidRequestVersionMissingSignTagMissingSignHeadersTagMalformedDateMalformedPresignedDateMalformedCredentialDateMalformedExpiresNegativeExpiresAuthHeaderEmptyExpiredPresignRequestRequestNotReadyYetUnsignedHeadersMissingDateHeaderInvalidQuerySignatureAlgoInvalidQueryParamsBucketAlreadyOwnedByYouInvalidDurationBucketAlreadyExistsMetadataTooLargeUnsupportedMetadataUnsupportedHostHeaderMaximumExpiresSlowDownReadSlowDownWriteMaxVersionsExceededInvalidPrefixMarkerBadRequestKeyTooLongErrorInvalidBucketObjectLockConfigurationObjectLockConfigurationNotFoundObjectLockConfigurationNotAllowedNoSuchObjectLockConfigurationObjectLockedInvalidRetentionDatePastObjectLockRetainDateUnknownWORMModeDirectiveBucketTaggingNotFoundObjectLockInvalidHeadersInvalidTagDirectivePolicyAlreadyAttachedPolicyNotAttachedExcessDataInvalidEncryptionMethodInvalidEncryptionKeyIDInsecureSSECustomerRequestSSEMultipartEncryptedSSEEncryptedObjectInvalidEncryptionParametersInvalidEncryptionParametersSSECInvalidSSECustomerAlgorithmInvalidSSECustomerKeyMissingSSECustomerKeyMissingSSECustomerKeyMD5SSECustomerKeyMD5MismatchInvalidSSECustomerParametersIncompatibleEncryptionMethodKMSNotConfiguredKMSKeyNotFoundExceptionKMSDefaultKeyAlreadyConfiguredNoAccessKeyInvalidTokenEventNotificationARNNotificationRegionNotificationOverlappingFilterNotificationFilterNameInvalidFilterNamePrefixFilterNameSuffixFilterValueInvalidOverlappingConfigsUnsupportedNotificationContentSHA256MismatchContentChecksumMismatchStorageFullRequestBodyParseObjectExistsAsDirectoryInvalidObjectNameInvalidObjectNamePrefixSlashInvalidResourceNameInvalidLifecycleQueryParameterServerNotInitializedBucketMetadataNotInitializedRequestTimedoutClientDisconnectedTooManyRequestsInvalidRequestTransitionStorageClassNotFoundErrorInvalidStorageClassBackendDownMalformedJSONAdminNoSuchUserAdminNoSuchUserLDAPWarnAdminLDAPExpectedLoginNameAdminNoSuchGroupAdminGroupNotEmptyAdminGroupDisabledAdminNoSuchJobAdminNoSuchPolicyAdminPolicyChangeAlreadyAppliedAdminInvalidArgumentAdminInvalidAccessKeyAdminInvalidSecretKeyAdminConfigNoQuorumAdminConfigTooLargeAdminConfigBadJSONAdminNoSuchConfigTargetAdminConfigEnvOverriddenAdminConfigDuplicateKeysAdminConfigInvalidIDPTypeAdminConfigLDAPNonDefaultConfigNameAdminConfigLDAPValidationAdminConfigIDPCfgNameAlreadyExistsAdminConfigIDPCfgNameDoesNotExistInsecureClientRequestObjectTamperedAdminLDAPNotEnabledSiteReplicationInvalidRequestSiteReplicationPeerRespSiteReplicationBackendIssueSiteReplicationServiceAccountErrorSiteReplicationBucketConfigErrorSiteReplicationBucketMetaErrorSiteReplicationIAMErrorSiteReplicationConfigMissingSiteReplicationIAMConfigMismatchAdminRebalanceAlreadyStartedAdminRebalanceNotStartedAdminBucketQuotaExceededAdminNoSuchQuotaConfigurationHealNotImplementedHealNoSuchProcessHealInvalidClientTokenHealMissingBucketHealAlreadyRunningHealOverlappingPathsIncorrectContinuationTokenEmptyRequestBodyUnsupportedFunctionInvalidExpressionTypeBusyUnauthorizedAccessExpressionTooLongIllegalSQLFunctionArgumentInvalidKeyPathInvalidCompressionFormatInvalidFileHeaderInfoInvalidJSONTypeInvalidQuoteFieldsInvalidRequestParameterInvalidDataTypeInvalidTextEncodingInvalidDataSourceInvalidTableAliasMissingRequiredParameterObjectSerializationConflictUnsupportedSQLOperationUnsupportedSQLStructureUnsupportedSyntaxUnsupportedRangeHeaderLexerInvalidCharLexerInvalidOperatorLexerInvalidLiteralLexerInvalidIONLiteralParseExpectedDatePartParseExpectedKeywordParseExpectedTokenTypeParseExpected2TokenTypesParseExpectedNumberParseExpectedRightParenBuiltinFunctionCallParseExpectedTypeNameParseExpectedWhenClauseParseUnsupportedTokenParseUnsupportedLiteralsGroupByParseExpectedMemberParseUnsupportedSelectParseUnsupportedCaseParseUnsupportedCaseClauseParseUnsupportedAliasParseUnsupportedSyntaxParseUnknownOperatorParseMissingIdentAfterAtParseUnexpectedOperatorParseUnexpectedTermParseUnexpectedTokenParseUnexpectedKeywordParseExpectedExpressionParseExpectedLeftParenAfterCastParseExpectedLeftParenValueConstructorParseExpectedLeftParenBuiltinFunctionCallParseExpectedArgumentDelimiterParseCastArityParseInvalidTypeParamParseEmptySelectParseSelectMissingFromParseExpectedIdentForGroupNameParseExpectedIdentForAliasParseUnsupportedCallWithStarParseNonUnaryAggregateFunctionCallParseMalformedJoinParseExpectedIdentForAtParseAsteriskIsNotAloneInSelectListParseCannotMixSqbAndWildcardInSelectListParseInvalidContextForWildcardInSelectListIncorrectSQLFunctionArgumentTypeValueParseFailureEvaluatorInvalidArgumentsIntegerOverflowLikeInvalidInputsCastFailedInvalidCastEvaluatorInvalidTimestampFormatPatternEvaluatorInvalidTimestampFormatPatternSymbolForParsingEvaluatorTimestampFormatPatternDuplicateFieldsEvaluatorTimestampFormatPatternHourClockAmPmMismatchEvaluatorUnterminatedTimestampFormatPatternTokenEvaluatorInvalidTimestampFormatPatternTokenEvaluatorInvalidTimestampFormatPatternSymbolEvaluatorBindingDoesNotExistMissingHeadersInvalidColumnIndexAdminConfigNotificationTargetsFailedAdminProfilerNotEnabledInvalidDecompressedSizeAddUserInvalidArgumentAdminResourceInvalidArgumentAdminAccountNotEligibleAccountNotEligibleAdminServiceAccountNotFoundPostPolicyConditionInvalidFormatInvalidChecksumLambdaARNInvalidLambdaARNNotFoundInvalidAttributeNameAdminNoAccessKeyAdminNoSecretKeyapiErrCodeEnd" -var _APIErrorCode_index = [...]uint16{0, 4, 16, 25, 39, 53, 67, 81, 94, 112, 129, 144, 161, 174, 186, 208, 228, 254, 268, 289, 306, 321, 344, 361, 379, 396, 420, 435, 456, 474, 486, 506, 523, 546, 567, 579, 597, 618, 646, 676, 697, 720, 746, 783, 813, 846, 871, 903, 933, 962, 987, 1009, 1035, 1057, 1085, 1114, 1148, 1179, 1216, 1240, 1264, 1292, 1318, 1349, 1379, 1388, 1400, 1416, 1429, 1443, 1461, 1481, 1502, 1518, 1529, 1545, 1556, 1584, 1604, 1620, 1648, 1662, 1679, 1699, 1712, 1726, 1739, 1752, 1768, 1785, 1806, 1820, 1841, 1854, 1876, 1899, 1915, 1930, 1945, 1966, 1984, 1999, 2016, 2041, 2059, 2082, 2097, 2116, 2132, 2151, 2172, 2186, 2198, 2211, 2230, 2249, 2259, 2274, 2310, 2341, 2374, 2403, 2415, 2435, 2459, 2483, 2504, 2528, 2547, 2568, 2585, 2595, 2618, 2640, 2666, 2687, 2705, 2732, 2763, 2790, 2811, 2832, 2856, 2881, 2909, 2937, 2953, 2976, 3006, 3017, 3029, 3046, 3061, 3079, 3108, 3125, 3141, 3157, 3175, 3193, 3216, 3237, 3260, 3271, 3287, 3310, 3327, 3355, 3374, 3404, 3424, 3439, 3457, 3472, 3486, 3521, 3540, 3551, 3564, 3579, 3602, 3628, 3644, 3662, 3680, 3694, 3711, 3742, 3762, 3783, 3804, 3823, 3842, 3860, 3883, 3907, 3931, 3956, 3991, 4016, 4050, 4083, 4104, 4118, 4137, 4166, 4189, 4216, 4250, 4282, 4312, 4335, 4363, 4395, 4423, 4447, 4471, 4500, 4518, 4535, 4557, 4574, 4592, 4612, 4638, 4654, 4673, 4694, 4698, 4716, 4733, 4759, 4773, 4797, 4818, 4833, 4851, 4874, 4889, 4908, 4925, 4942, 4966, 4993, 5016, 5039, 5056, 5078, 5094, 5114, 5133, 5155, 5176, 5196, 5218, 5242, 5261, 5303, 5324, 5347, 5368, 5399, 5418, 5440, 5460, 5486, 5507, 5529, 5549, 5573, 5596, 5615, 5635, 5657, 5680, 5711, 5749, 5790, 5820, 5834, 5855, 5871, 5893, 5923, 5949, 5977, 6011, 6029, 6052, 6087, 6127, 6169, 6201, 6218, 6243, 6258, 6275, 6285, 6296, 6334, 6388, 6434, 6486, 6534, 6577, 6621, 6649, 6663, 6681, 6717, 6740, 6763, 6785, 6813, 6836, 6854, 6881, 6913, 6928, 6944, 6961, 6981, 6997, 7013, 7026} +var _APIErrorCode_index = [...]uint16{0, 4, 16, 25, 39, 53, 67, 81, 94, 112, 129, 144, 161, 174, 186, 208, 228, 254, 268, 289, 306, 321, 344, 361, 379, 396, 420, 435, 456, 474, 486, 506, 523, 546, 567, 579, 597, 618, 646, 676, 697, 720, 746, 783, 813, 846, 871, 903, 933, 962, 987, 1009, 1035, 1057, 1085, 1114, 1148, 1179, 1216, 1240, 1264, 1292, 1318, 1349, 1379, 1388, 1400, 1416, 1429, 1443, 1461, 1481, 1502, 1518, 1529, 1545, 1556, 1584, 1604, 1620, 1648, 1662, 1679, 1699, 1712, 1726, 1739, 1752, 1768, 1785, 1806, 1820, 1841, 1854, 1876, 1899, 1915, 1930, 1945, 1966, 1984, 1999, 2016, 2041, 2059, 2082, 2097, 2116, 2132, 2151, 2172, 2186, 2198, 2211, 2230, 2249, 2259, 2274, 2310, 2341, 2374, 2403, 2415, 2435, 2459, 2483, 2504, 2528, 2547, 2568, 2585, 2595, 2618, 2640, 2666, 2687, 2705, 2732, 2763, 2790, 2811, 2832, 2856, 2881, 2909, 2937, 2953, 2976, 3006, 3017, 3029, 3046, 3061, 3079, 3108, 3125, 3141, 3157, 3175, 3193, 3216, 3237, 3260, 3271, 3287, 3310, 3327, 3355, 3374, 3404, 3424, 3452, 3467, 3485, 3500, 3514, 3549, 3568, 3579, 3592, 3607, 3630, 3656, 3672, 3690, 3708, 3722, 3739, 3770, 3790, 3811, 3832, 3851, 3870, 3888, 3911, 3935, 3959, 3984, 4019, 4044, 4078, 4111, 4132, 4146, 4165, 4194, 4217, 4244, 4278, 4310, 4340, 4363, 4391, 4423, 4451, 4475, 4499, 4528, 4546, 4563, 4585, 4602, 4620, 4640, 4666, 4682, 4701, 4722, 4726, 4744, 4761, 4787, 4801, 4825, 4846, 4861, 4879, 4902, 4917, 4936, 4953, 4970, 4994, 5021, 5044, 5067, 5084, 5106, 5122, 5142, 5161, 5183, 5204, 5224, 5246, 5270, 5289, 5331, 5352, 5375, 5396, 5427, 5446, 5468, 5488, 5514, 5535, 5557, 5577, 5601, 5624, 5643, 5663, 5685, 5708, 5739, 5777, 5818, 5848, 5862, 5883, 5899, 5921, 5951, 5977, 6005, 6039, 6057, 6080, 6115, 6155, 6197, 6229, 6246, 6271, 6286, 6303, 6313, 6324, 6362, 6416, 6462, 6514, 6562, 6605, 6649, 6677, 6691, 6709, 6745, 6768, 6791, 6813, 6841, 6864, 6882, 6909, 6941, 6956, 6972, 6989, 7009, 7025, 7041, 7054} func (i APIErrorCode) String() string { if i < 0 || i >= APIErrorCode(len(_APIErrorCode_index)-1) { diff --git a/cmd/bucket-metadata-sys.go b/cmd/bucket-metadata-sys.go index d6c9a9786..d36bd1b18 100644 --- a/cmd/bucket-metadata-sys.go +++ b/cmd/bucket-metadata-sys.go @@ -46,6 +46,7 @@ type BucketMetadataSys struct { objAPI ObjectLayer sync.RWMutex + initialized bool metadataMap map[string]BucketMetadata } @@ -433,6 +434,8 @@ func (sys *BucketMetadataSys) GetConfigFromDisk(ctx context.Context, bucket stri return loadBucketMetadata(ctx, objAPI, bucket) } +var errBucketMetadataNotInitialized = errors.New("bucket metadata not initialized yet") + // GetConfig returns a specific configuration from the bucket metadata. // The returned object may not be modified. // reloaded will be true if metadata refreshed from disk @@ -454,6 +457,10 @@ func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (met } meta, err = loadBucketMetadata(ctx, objAPI, bucket) if err != nil { + if !sys.Initialized() { + // bucket metadata not yet initialized + return newBucketMetadata(bucket), reloaded, errBucketMetadataNotInitialized + } return meta, reloaded, err } sys.Lock() @@ -498,9 +505,10 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck } errs := g.Wait() - for _, err := range errs { + for index, err := range errs { if err != nil { - internalLogIf(ctx, err, logger.WarningKind) + internalLogOnceIf(ctx, fmt.Errorf("Unable to load bucket metadata, will be retried: %w", err), + "load-bucket-metadata-"+buckets[index].Name, logger.WarningKind) } } @@ -583,6 +591,14 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context, fa } } +// Initialized indicates if bucket metadata sys is initialized atleast once. +func (sys *BucketMetadataSys) Initialized() bool { + sys.RLock() + defer sys.RUnlock() + + return sys.initialized +} + // Loads bucket metadata for all buckets into BucketMetadataSys. func (sys *BucketMetadataSys) init(ctx context.Context, buckets []BucketInfo) { count := 100 // load 100 bucket metadata at a time. @@ -596,6 +612,10 @@ func (sys *BucketMetadataSys) init(ctx context.Context, buckets []BucketInfo) { buckets = buckets[count:] } + sys.Lock() + sys.initialized = true + sys.Unlock() + if globalIsDistErasure { go sys.refreshBucketsMetadataLoop(ctx, failedBuckets) } diff --git a/cmd/bucket-object-lock.go b/cmd/bucket-object-lock.go index a4a042cf0..ada088b5b 100644 --- a/cmd/bucket-object-lock.go +++ b/cmd/bucket-object-lock.go @@ -44,7 +44,6 @@ func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention, if errors.Is(err, errInvalidArgument) { return r, err } - logger.CriticalIf(context.Background(), err) return r, err } return config.ToRetention(), nil diff --git a/cmd/bucket-replication.go b/cmd/bucket-replication.go index 171d7e90d..d9c6d6011 100644 --- a/cmd/bucket-replication.go +++ b/cmd/bucket-replication.go @@ -81,13 +81,10 @@ const ( // gets replication config associated to a given bucket name. func getReplicationConfig(ctx context.Context, bucketName string) (rc *replication.Config, err error) { rCfg, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucketName) - if err != nil { - if errors.Is(err, BucketReplicationConfigNotFound{Bucket: bucketName}) || errors.Is(err, errInvalidArgument) { - return rCfg, err - } - logger.CriticalIf(ctx, err) + if err != nil && !errors.Is(err, BucketReplicationConfigNotFound{Bucket: bucketName}) { + return rCfg, err } - return rCfg, err + return rCfg, nil } // validateReplicationDestination returns error if replication destination bucket missing or not configured @@ -261,10 +258,16 @@ func mustReplicate(ctx context.Context, bucket, object string, mopts mustReplica if mopts.replicationRequest { // incoming replication request on target cluster return } + cfg, err := getReplicationConfig(ctx, bucket) if err != nil { + replLogOnceIf(ctx, err, bucket) return } + if cfg == nil { + return + } + opts := replication.ObjectOpts{ Name: object, SSEC: crypto.SSEC.IsEncrypted(mopts.meta), @@ -312,6 +315,7 @@ var standardHeaders = []string{ func hasReplicationRules(ctx context.Context, bucket string, objects []ObjectToDelete) bool { c, err := getReplicationConfig(ctx, bucket) if err != nil || c == nil { + replLogOnceIf(ctx, err, bucket) return false } for _, obj := range objects { @@ -331,6 +335,7 @@ func isStandardHeader(matchHeaderKey string) bool { func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelete, oi ObjectInfo, delOpts ObjectOptions, gerr error) (dsc ReplicateDecision) { rcfg, err := getReplicationConfig(ctx, bucket) if err != nil || rcfg == nil { + replLogOnceIf(ctx, err, bucket) return } // If incoming request is a replication request, it does not need to be re-replicated. @@ -2231,6 +2236,8 @@ func getProxyTargets(ctx context.Context, bucket, object string, opts ObjectOpti } cfg, err := getReplicationConfig(ctx, bucket) if err != nil || cfg == nil { + replLogOnceIf(ctx, err, bucket) + return &madmin.BucketTargets{} } topts := replication.ObjectOpts{Name: object} @@ -3124,7 +3131,7 @@ func saveResyncStatus(ctx context.Context, bucket string, brs BucketReplicationR func getReplicationDiff(ctx context.Context, objAPI ObjectLayer, bucket string, opts madmin.ReplDiffOpts) (chan madmin.DiffInfo, error) { cfg, err := getReplicationConfig(ctx, bucket) if err != nil { - replLogIf(ctx, err) + replLogOnceIf(ctx, err, bucket) return nil, err } tgts, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket) @@ -3217,7 +3224,11 @@ func QueueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, ret if oi.ModTime.IsZero() { return } - rcfg, _ := getReplicationConfig(ctx, bucket) + rcfg, err := getReplicationConfig(ctx, bucket) + if err != nil { + replLogOnceIf(ctx, err, bucket) + return + } tgts, _ := globalBucketTargetSys.ListBucketTargets(ctx, bucket) queueReplicationHeal(ctx, bucket, oi, replicationConfig{ Config: rcfg, diff --git a/cmd/bucket-versioning-handler.go b/cmd/bucket-versioning-handler.go index 5480748c9..31dc6fd4c 100644 --- a/cmd/bucket-versioning-handler.go +++ b/cmd/bucket-versioning-handler.go @@ -82,7 +82,7 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r }, r.URL) return } - if _, err := getReplicationConfig(ctx, bucket); err == nil && v.Suspended() { + if rc, _ := getReplicationConfig(ctx, bucket); rc != nil && v.Suspended() { writeErrorResponse(ctx, w, APIError{ Code: "InvalidBucketState", Description: "A replication configuration is present on this bucket, bucket wide versioning cannot be suspended.", diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go index 035065711..4e3c2cadb 100644 --- a/cmd/data-scanner.go +++ b/cmd/data-scanner.go @@ -36,7 +36,9 @@ import ( "github.com/minio/madmin-go/v3" "github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/bucket/object/lock" + objectlock "github.com/minio/minio/internal/bucket/object/lock" "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/bucket/versioning" "github.com/minio/minio/internal/color" "github.com/minio/minio/internal/config/heal" "github.com/minio/minio/internal/event" @@ -952,10 +954,32 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje } versionID := oi.VersionID - vcfg, _ := globalBucketVersioningSys.Get(i.bucket) - rCfg, _ := globalBucketObjectLockSys.Get(i.bucket) - replcfg, _ := getReplicationConfig(ctx, i.bucket) - lcEvt := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, replcfg, oi) + + var vc *versioning.Versioning + var lr objectlock.Retention + var rcfg *replication.Config + if i.bucket != minioMetaBucket { + vc, err = globalBucketVersioningSys.Get(i.bucket) + if err != nil { + scannerLogOnceIf(ctx, err, i.bucket) + return + } + + // Check if bucket is object locked. + lr, err = globalBucketObjectLockSys.Get(i.bucket) + if err != nil { + scannerLogOnceIf(ctx, err, i.bucket) + return + } + + rcfg, err = getReplicationConfig(ctx, i.bucket) + if err != nil { + scannerLogOnceIf(ctx, err, i.bucket) + return + } + } + + lcEvt := evalActionFromLifecycle(ctx, *i.lifeCycle, lr, rcfg, oi) if i.debug { if versionID != "" { console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, lcEvt.Action) @@ -973,7 +997,7 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje size = 0 case lifecycle.DeleteAction: // On a non-versioned bucket, DeleteObject removes the only version permanently. - if !vcfg.PrefixEnabled(oi.Name) { + if !vc.PrefixEnabled(oi.Name) { size = 0 } } diff --git a/cmd/data-usage-cache.go b/cmd/data-usage-cache.go index 262948b28..acfa505e9 100644 --- a/cmd/data-usage-cache.go +++ b/cmd/data-usage-cache.go @@ -18,7 +18,6 @@ package cmd import ( - "bytes" "context" "errors" "fmt" @@ -36,7 +35,6 @@ import ( "github.com/klauspost/compress/zstd" "github.com/minio/madmin-go/v3" "github.com/minio/minio/internal/bucket/lifecycle" - "github.com/minio/minio/internal/hash" "github.com/tinylib/msgp/msgp" "github.com/valyala/bytebufferpool" ) @@ -1005,11 +1003,23 @@ func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string) ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, ObjectOptions{NoLock: true}) + r, err := store.GetObjectNInfo(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, name), nil, http.Header{}, ObjectOptions{NoLock: true}) if err != nil { switch err.(type) { case ObjectNotFound, BucketNotFound: - return false, nil + r, err = store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, ObjectOptions{NoLock: true}) + if err != nil { + switch err.(type) { + case ObjectNotFound, BucketNotFound: + return false, nil + case InsufficientReadQuorum, StorageErr: + return true, nil + } + return false, err + } + err = d.deserialize(r) + r.Close() + return err != nil, nil case InsufficientReadQuorum, StorageErr: return true, nil } @@ -1070,24 +1080,11 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string) } save := func(name string, timeout time.Duration) error { - hr, err := hash.NewReader(ctx, bytes.NewReader(buf.Bytes()), int64(buf.Len()), "", "", int64(buf.Len())) - if err != nil { - return err - } - // Abandon if more than a minute, so we don't hold up scanner. ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - _, err = store.PutObject(ctx, - dataUsageBucket, - name, - NewPutObjReader(hr), - ObjectOptions{NoLock: true}) - if isErrBucketNotFound(err) { - return nil - } - return err + return saveConfig(ctx, store, pathJoin(bucketMetaPrefix, name), buf.Bytes()) } defer save(name+".bkp", 5*time.Second) // Keep a backup as well diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 66d3bc924..7f7eead84 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -1833,9 +1833,18 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string var replcfg *replication.Config if opts.Expiration.Expire { // Check if the current bucket has a configured lifecycle policy - lc, _ = globalLifecycleSys.Get(bucket) - rcfg, _ = globalBucketObjectLockSys.Get(bucket) - replcfg, _ = getReplicationConfig(ctx, bucket) + lc, err = globalLifecycleSys.Get(bucket) + if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) { + return objInfo, err + } + rcfg, err = globalBucketObjectLockSys.Get(bucket) + if err != nil { + return objInfo, err + } + replcfg, err = getReplicationConfig(ctx, bucket) + if err != nil { + return objInfo, err + } } // expiration attempted on a bucket with no lifecycle diff --git a/cmd/erasure-server-pool-decom.go b/cmd/erasure-server-pool-decom.go index 8ff9ddc3c..3f66dfc2e 100644 --- a/cmd/erasure-server-pool-decom.go +++ b/cmd/erasure-server-pool-decom.go @@ -31,6 +31,10 @@ import ( "github.com/dustin/go-humanize" "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/lifecycle" + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/bucket/versioning" "github.com/minio/minio/internal/hash" "github.com/minio/minio/internal/logger" "github.com/minio/pkg/v2/console" @@ -754,14 +758,33 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool return err } - vc, _ := globalBucketVersioningSys.Get(bi.Name) + var vc *versioning.Versioning + var lc *lifecycle.Lifecycle + var lr objectlock.Retention + var rcfg *replication.Config + if bi.Name != minioMetaBucket { + vc, err = globalBucketVersioningSys.Get(bi.Name) + if err != nil { + return err + } - // Check if the current bucket has a configured lifecycle policy - lc, _ := globalLifecycleSys.Get(bi.Name) + // Check if the current bucket has a configured lifecycle policy + lc, err = globalLifecycleSys.Get(bi.Name) + if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bi.Name}) { + return err + } - // Check if bucket is object locked. - lr, _ := globalBucketObjectLockSys.Get(bi.Name) - rcfg, _ := getReplicationConfig(ctx, bi.Name) + // Check if bucket is object locked. + lr, err = globalBucketObjectLockSys.Get(bi.Name) + if err != nil { + return err + } + + rcfg, err = getReplicationConfig(ctx, bi.Name) + if err != nil { + return err + } + } for setIdx, set := range pool.sets { set := set @@ -1088,14 +1111,33 @@ func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error pool := z.serverPools[idx] for _, set := range pool.sets { for _, bi := range buckets { - vc, _ := globalBucketVersioningSys.Get(bi.Name) + var vc *versioning.Versioning + var lc *lifecycle.Lifecycle + var lr objectlock.Retention + var rcfg *replication.Config + if bi.Name != minioMetaBucket { + vc, err = globalBucketVersioningSys.Get(bi.Name) + if err != nil { + return err + } - // Check if the current bucket has a configured lifecycle policy - lc, _ := globalLifecycleSys.Get(bi.Name) + // Check if the current bucket has a configured lifecycle policy + lc, err = globalLifecycleSys.Get(bi.Name) + if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bi.Name}) { + return err + } - // Check if bucket is object locked. - lr, _ := globalBucketObjectLockSys.Get(bi.Name) - rcfg, _ := getReplicationConfig(ctx, bi.Name) + // Check if bucket is object locked. + lr, err = globalBucketObjectLockSys.Get(bi.Name) + if err != nil { + return err + } + + rcfg, err = getReplicationConfig(ctx, bi.Name) + if err != nil { + return err + } + } filterLifecycle := func(bucket, object string, fi FileInfo) bool { if lc == nil { @@ -1118,7 +1160,7 @@ func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error } var versionsFound int - err := set.listObjectsToDecommission(ctx, bi, func(entry metaCacheEntry) { + if err = set.listObjectsToDecommission(ctx, bi, func(entry metaCacheEntry) { if !entry.isObject() { return } @@ -1146,8 +1188,7 @@ func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error versionsFound++ } - }) - if err != nil { + }); err != nil { return err } diff --git a/cmd/erasure-server-pool-rebalance.go b/cmd/erasure-server-pool-rebalance.go index ed9eddc27..89eb6bed4 100644 --- a/cmd/erasure-server-pool-rebalance.go +++ b/cmd/erasure-server-pool-rebalance.go @@ -32,6 +32,10 @@ import ( "github.com/dustin/go-humanize" "github.com/lithammer/shortuuid/v4" "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/lifecycle" + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/bucket/versioning" "github.com/minio/minio/internal/hash" xioutil "github.com/minio/minio/internal/ioutil" "github.com/minio/minio/internal/logger" @@ -448,9 +452,11 @@ func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int) } stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceBucket, poolIdx, bucket) - err = z.rebalanceBucket(ctx, bucket, poolIdx) - if err != nil { + if err = z.rebalanceBucket(ctx, bucket, poolIdx); err != nil { stopFn(err) + if errors.Is(err, errServerNotInitialized) || errors.Is(err, errBucketMetadataNotInitialized) { + continue + } rebalanceLogIf(ctx, err) return } @@ -521,14 +527,36 @@ func (set *erasureObjects) listObjectsToRebalance(ctx context.Context, bucketNam } // rebalanceBucket rebalances objects under bucket in poolIdx pool -func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string, poolIdx int) error { +func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string, poolIdx int) (err error) { ctx = logger.SetReqInfo(ctx, &logger.ReqInfo{}) - vc, _ := globalBucketVersioningSys.Get(bucket) - // Check if the current bucket has a configured lifecycle policy - lc, _ := globalLifecycleSys.Get(bucket) - // Check if bucket is object locked. - lr, _ := globalBucketObjectLockSys.Get(bucket) - rcfg, _ := getReplicationConfig(ctx, bucket) + + var vc *versioning.Versioning + var lc *lifecycle.Lifecycle + var lr objectlock.Retention + var rcfg *replication.Config + if bucket != minioMetaBucket { + vc, err = globalBucketVersioningSys.Get(bucket) + if err != nil { + return err + } + + // Check if the current bucket has a configured lifecycle policy + lc, err = globalLifecycleSys.Get(bucket) + if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) { + return err + } + + // Check if bucket is object locked. + lr, err = globalBucketObjectLockSys.Get(bucket) + if err != nil { + return err + } + + rcfg, err = getReplicationConfig(ctx, bucket) + if err != nil { + return err + } + } pool := z.serverPools[poolIdx] diff --git a/cmd/global-heal.go b/cmd/global-heal.go index ad5a67788..ea1d1d97f 100644 --- a/cmd/global-heal.go +++ b/cmd/global-heal.go @@ -19,6 +19,7 @@ package cmd import ( "context" + "errors" "fmt" "math/rand" "runtime" @@ -213,14 +214,35 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string, continue } - vc, _ := globalBucketVersioningSys.Get(bucket) + vc, err := globalBucketVersioningSys.Get(bucket) + if err != nil { + retErr = err + healingLogIf(ctx, err) + continue + } // Check if the current bucket has a configured lifecycle policy - lc, _ := globalLifecycleSys.Get(bucket) + lc, err := globalLifecycleSys.Get(bucket) + if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) { + retErr = err + healingLogIf(ctx, err) + continue + } // Check if bucket is object locked. - lr, _ := globalBucketObjectLockSys.Get(bucket) - rcfg, _ := getReplicationConfig(ctx, bucket) + lr, err := globalBucketObjectLockSys.Get(bucket) + if err != nil { + retErr = err + healingLogIf(ctx, err) + continue + } + + rcfg, err := getReplicationConfig(ctx, bucket) + if err != nil { + retErr = err + healingLogIf(ctx, err) + continue + } if serverDebugLog { console.Debugf(color.Green("healDrive:")+" healing bucket %s content on %s erasure set\n", @@ -442,7 +464,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string, bucket: actualBucket, } - err := listPathRaw(ctx, listPathRawOptions{ + err = listPathRaw(ctx, listPathRawOptions{ disks: disks, fallbackDisks: fallbackDisks, bucket: actualBucket, diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index af2ffa04d..9cd9cb004 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -556,15 +556,23 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj if !proxy.Proxy { // apply lifecycle rules only for local requests // Automatically remove the object/version if an expiry lifecycle rule can be applied if lc, err := globalLifecycleSys.Get(bucket); err == nil { - rcfg, _ := globalBucketObjectLockSys.Get(bucket) - replcfg, _ := getReplicationConfig(ctx, bucket) + rcfg, err := globalBucketObjectLockSys.Get(bucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + replcfg, err := getReplicationConfig(ctx, bucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } event := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, objInfo) if event.Action.Delete() { // apply whatever the expiry rule is. applyExpiryRule(event, lcEventSrc_s3GetObject, objInfo) if !event.Action.DeleteRestored() { // If the ILM action is not on restored object return error. - writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey)) + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNoSuchKey), r.URL) return } } @@ -728,7 +736,7 @@ func (api objectAPIHandlers) getObjectAttributesHandler(ctx context.Context, obj } if _, err = DecryptObjectInfo(&objInfo, r); err != nil { - writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } @@ -1077,8 +1085,16 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob if !proxy.Proxy { // apply lifecycle rules only locally not for proxied requests // Automatically remove the object/version if an expiry lifecycle rule can be applied if lc, err := globalLifecycleSys.Get(bucket); err == nil { - rcfg, _ := globalBucketObjectLockSys.Get(bucket) - replcfg, _ := getReplicationConfig(ctx, bucket) + rcfg, err := globalBucketObjectLockSys.Get(bucket) + if err != nil { + writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) + return + } + replcfg, err := getReplicationConfig(ctx, bucket) + if err != nil { + writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) + return + } event := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, objInfo) if event.Action.Delete() { // apply whatever the expiry rule is. diff --git a/cmd/os-reliable.go b/cmd/os-reliable.go index 9ab77939a..3561cbd33 100644 --- a/cmd/os-reliable.go +++ b/cmd/os-reliable.go @@ -108,6 +108,16 @@ func reliableMkdirAll(dirPath string, mode os.FileMode, baseDir string) (err err // Retry only for the first retryable error. if osIsNotExist(err) && i == 0 { i++ + // Determine if os.NotExist error is because of + // baseDir's parent being present, retry it once such + // that the MkdirAll is retried once for the parent + // of dirPath. + // Because it is worth a retry to skip a different + // baseDir which is slightly higher up the depth. + nbaseDir := path.Dir(baseDir) + if baseDir != "" && nbaseDir != "" && nbaseDir != SlashSeparator { + baseDir = nbaseDir + } continue } }