From 0c068b15c70cd060f855f1f79ea82375166c4a81 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 13 Feb 2024 06:55:54 -0800 Subject: [PATCH] add missing handler for reloading site replication config on peers (#19042) --- cmd/peer-rest-client.go | 19 ++-- cmd/peer-rest-common.go | 92 ++++++++----------- .../run-multi-site-minio-idp.sh | 43 +++++++-- 3 files changed, 82 insertions(+), 72 deletions(-) diff --git a/cmd/peer-rest-client.go b/cmd/peer-rest-client.go index 2f7ba74b7..50ed8759c 100644 --- a/cmd/peer-rest-client.go +++ b/cmd/peer-rest-client.go @@ -540,6 +540,16 @@ func (client *peerRESTClient) LoadGroup(group string) error { return err } +func (client *peerRESTClient) ReloadSiteReplicationConfig(ctx context.Context) error { + conn := client.gridConn() + if conn == nil { + return nil + } + + _, err := reloadSiteReplicationConfigHandler.Call(ctx, conn, grid.NewMSSWith(map[string]string{})) + return err +} + // VerifyBinary - sends verify binary message to remote peers. func (client *peerRESTClient) VerifyBinary(ctx context.Context, u *url.URL, sha256Sum []byte, releaseInfo string, reader io.Reader) error { values := make(url.Values) @@ -970,15 +980,6 @@ func (client *peerRESTClient) DriveSpeedTest(ctx context.Context, opts madmin.Dr return result, nil } -func (client *peerRESTClient) ReloadSiteReplicationConfig(ctx context.Context) error { - respBody, err := client.callWithContext(context.Background(), peerRESTMethodReloadSiteReplicationConfig, nil, nil, -1) - if err != nil { - return err - } - defer xhttp.DrainBody(respBody) - return nil -} - func (client *peerRESTClient) GetLastDayTierStats(ctx context.Context) (DailyAllTierStats, error) { var result map[string]lastDayTierStats respBody, err := client.callWithContext(context.Background(), peerRESTMethodGetLastDayTierStats, nil, nil, -1) diff --git a/cmd/peer-rest-common.go b/cmd/peer-rest-common.go index b9fedfc19..672b3b785 100644 --- a/cmd/peer-rest-common.go +++ b/cmd/peer-rest-common.go @@ -25,61 +25,43 @@ const ( ) const ( - peerRESTMethodHealth = "/health" - peerRESTMethodServerInfo = "/serverinfo" - peerRESTMethodLocalStorageInfo = "/localstorageinfo" - peerRESTMethodCPUInfo = "/cpuinfo" - peerRESTMethodDiskHwInfo = "/diskhwinfo" - peerRESTMethodNetHwInfo = "/nethwinfo" - peerRESTMethodOsInfo = "/osinfo" - peerRESTMethodMemInfo = "/meminfo" - peerRESTMethodProcInfo = "/procinfo" - peerRESTMethodSysErrors = "/syserrors" - peerRESTMethodSysServices = "/sysservices" - peerRESTMethodSysConfig = "/sysconfig" - peerRESTMethodDeleteBucketMetadata = "/deletebucketmetadata" - peerRESTMethodLoadBucketMetadata = "/loadbucketmetadata" - peerRESTMethodGetBucketStats = "/getbucketstats" - peerRESTMethodGetAllBucketStats = "/getallbucketstats" - peerRESTMethodVerifyBinary = "/verifybinary" - peerRESTMethodCommitBinary = "/commitbinary" - peerRESTMethodSignalService = "/signalservice" - peerRESTMethodBackgroundHealStatus = "/backgroundhealstatus" - peerRESTMethodGetLocks = "/getlocks" - peerRESTMethodLoadUser = "/loaduser" - peerRESTMethodLoadServiceAccount = "/loadserviceaccount" - peerRESTMethodDeleteUser = "/deleteuser" - peerRESTMethodDeleteServiceAccount = "/deleteserviceaccount" - peerRESTMethodLoadPolicy = "/loadpolicy" - peerRESTMethodLoadPolicyMapping = "/loadpolicymapping" - peerRESTMethodDeletePolicy = "/deletepolicy" - peerRESTMethodLoadGroup = "/loadgroup" - peerRESTMethodStartProfiling = "/startprofiling" - peerRESTMethodDownloadProfilingData = "/downloadprofilingdata" - peerRESTMethodCycleBloom = "/cyclebloom" - peerRESTMethodTrace = "/trace" - peerRESTMethodListen = "/listen" - peerRESTMethodLog = "/log" - peerRESTMethodGetLocalDiskIDs = "/getlocaldiskids" - peerRESTMethodGetBandwidth = "/bandwidth" - peerRESTMethodGetMetacacheListing = "/getmetacache" - peerRESTMethodUpdateMetacacheListing = "/updatemetacache" - peerRESTMethodGetPeerMetrics = "/peermetrics" - peerRESTMethodGetPeerBucketMetrics = "/peerbucketmetrics" - peerRESTMethodLoadTransitionTierConfig = "/loadtransitiontierconfig" - peerRESTMethodSpeedTest = "/speedtest" - peerRESTMethodDriveSpeedTest = "/drivespeedtest" - peerRESTMethodReloadSiteReplicationConfig = "/reloadsitereplicationconfig" - peerRESTMethodReloadPoolMeta = "/reloadpoolmeta" - peerRESTMethodLoadRebalanceMeta = "/loadrebalancemeta" - peerRESTMethodStopRebalance = "/stoprebalance" - peerRESTMethodGetLastDayTierStats = "/getlastdaytierstats" - peerRESTMethodDevNull = "/devnull" - peerRESTMethodNetperf = "/netperf" - peerRESTMethodMetrics = "/metrics" - peerRESTMethodResourceMetrics = "/resourcemetrics" - peerRESTMethodGetReplicationMRF = "/getreplicationmrf" - peerRESTMethodGetSRMetrics = "/getsrmetrics" + peerRESTMethodHealth = "/health" + peerRESTMethodServerInfo = "/serverinfo" + peerRESTMethodLocalStorageInfo = "/localstorageinfo" + peerRESTMethodCPUInfo = "/cpuinfo" + peerRESTMethodDiskHwInfo = "/diskhwinfo" + peerRESTMethodNetHwInfo = "/nethwinfo" + peerRESTMethodOsInfo = "/osinfo" + peerRESTMethodMemInfo = "/meminfo" + peerRESTMethodProcInfo = "/procinfo" + peerRESTMethodSysErrors = "/syserrors" + peerRESTMethodSysServices = "/sysservices" + peerRESTMethodSysConfig = "/sysconfig" + peerRESTMethodGetBucketStats = "/getbucketstats" + peerRESTMethodGetAllBucketStats = "/getallbucketstats" + peerRESTMethodVerifyBinary = "/verifybinary" + peerRESTMethodCommitBinary = "/commitbinary" + peerRESTMethodSignalService = "/signalservice" + peerRESTMethodBackgroundHealStatus = "/backgroundhealstatus" + peerRESTMethodGetLocks = "/getlocks" + peerRESTMethodStartProfiling = "/startprofiling" + peerRESTMethodDownloadProfilingData = "/downloadprofilingdata" + peerRESTMethodLog = "/log" + peerRESTMethodGetBandwidth = "/bandwidth" + peerRESTMethodGetMetacacheListing = "/getmetacache" + peerRESTMethodUpdateMetacacheListing = "/updatemetacache" + peerRESTMethodGetPeerMetrics = "/peermetrics" + peerRESTMethodGetPeerBucketMetrics = "/peerbucketmetrics" + peerRESTMethodSpeedTest = "/speedtest" + peerRESTMethodDriveSpeedTest = "/drivespeedtest" + peerRESTMethodStopRebalance = "/stoprebalance" + peerRESTMethodGetLastDayTierStats = "/getlastdaytierstats" + peerRESTMethodDevNull = "/devnull" + peerRESTMethodNetperf = "/netperf" + peerRESTMethodMetrics = "/metrics" + peerRESTMethodResourceMetrics = "/resourcemetrics" + peerRESTMethodGetReplicationMRF = "/getreplicationmrf" + peerRESTMethodGetSRMetrics = "/getsrmetrics" ) const ( diff --git a/docs/site-replication/run-multi-site-minio-idp.sh b/docs/site-replication/run-multi-site-minio-idp.sh index 3cf0202f6..a7d04df3c 100755 --- a/docs/site-replication/run-multi-site-minio-idp.sh +++ b/docs/site-replication/run-multi-site-minio-idp.sh @@ -41,12 +41,20 @@ if [ ! -f ./mc ]; then chmod +x mc fi -minio server --config-dir /tmp/minio-internal --address ":9001" /tmp/minio-internal-idp1/{1...4} >/tmp/minio1_1.log 2>&1 & -site1_pid=$! -minio server --config-dir /tmp/minio-internal --address ":9002" /tmp/minio-internal-idp2/{1...4} >/tmp/minio2_1.log 2>&1 & -site2_pid=$! -minio server --config-dir /tmp/minio-internal --address ":9003" /tmp/minio-internal-idp3/{1...4} >/tmp/minio3_1.log 2>&1 & -site3_pid=$! +minio server --config-dir /tmp/minio-internal --address ":9001" http://localhost:9001/tmp/minio-internal-idp1/{1...4} http://localhost:9010/tmp/minio-internal-idp1/{5...8} >/tmp/minio1_1.log 2>&1 & +site1_pid1=$! +minio server --config-dir /tmp/minio-internal --address ":9010" http://localhost:9001/tmp/minio-internal-idp1/{1...4} http://localhost:9010/tmp/minio-internal-idp1/{5...8} >/tmp/minio1_2.log 2>&1 & +site1_pid2=$! + +minio server --config-dir /tmp/minio-internal --address ":9002" http://localhost:9002/tmp/minio-internal-idp2/{1...4} http://localhost:9020/tmp/minio-internal-idp2/{5...8} >/tmp/minio2_1.log 2>&1 & +site2_pid1=$! +minio server --config-dir /tmp/minio-internal --address ":9020" http://localhost:9002/tmp/minio-internal-idp2/{1...4} http://localhost:9020/tmp/minio-internal-idp2/{5...8} >/tmp/minio2_2.log 2>&1 & +site2_pid2=$! + +minio server --config-dir /tmp/minio-internal --address ":9003" http://localhost:9003/tmp/minio-internal-idp3/{1...4} http://localhost:9030/tmp/minio-internal-idp3/{5...8} >/tmp/minio3_1.log 2>&1 & +site3_pid1=$! +minio server --config-dir /tmp/minio-internal --address ":9030" http://localhost:9003/tmp/minio-internal-idp3/{1...4} http://localhost:9030/tmp/minio-internal-idp3/{5...8} >/tmp/minio3_2.log 2>&1 & +site3_pid2=$! sleep 10 @@ -54,8 +62,25 @@ export MC_HOST_minio1=http://minio:minio123@localhost:9001 export MC_HOST_minio2=http://minio:minio123@localhost:9002 export MC_HOST_minio3=http://minio:minio123@localhost:9003 +export MC_HOST_minio10=http://minio:minio123@localhost:9010 +export MC_HOST_minio20=http://minio:minio123@localhost:9020 +export MC_HOST_minio30=http://minio:minio123@localhost:9030 + ./mc admin replicate add minio1 minio2 +site_enabled=$(./mc admin replicate info minio1) +site_enabled_peer=$(./mc admin replicate info minio10) + +[[ $site_enabled =~ "is not enabled" ]] && { + echo "expected both peers to have same information" + exit_1 +} + +[[ $site_enabled_peer =~ "is not enabled" ]] && { + echo "expected both peers to have same information" + exit_1 +} + ./mc admin user add minio1 foobar foo12345 ## add foobar-g group with foobar @@ -361,7 +386,8 @@ if [ "${policy}" != "null" ]; then exit_1 fi -kill -9 ${site1_pid} +kill -9 ${site1_pid1} ${site1_pid2} + # Update tag on minio2/newbucket when minio1 is down ./mc tag set minio2/newbucket "key=val2" # create a new bucket on minio2. This should replicate to minio1 after it comes online. @@ -371,7 +397,8 @@ kill -9 ${site1_pid} ./mc rb minio2/bucket2 # Restart minio1 instance -minio server --config-dir /tmp/minio-internal --address ":9001" /tmp/minio-internal-idp1/{1...4} >/tmp/minio1_1.log 2>&1 & +minio server --config-dir /tmp/minio-internal --address ":9001" http://localhost:9001/tmp/minio-internal-idp1/{1...4} http://localhost:9010/tmp/minio-internal-idp1/{5...8} >/tmp/minio1_1.log 2>&1 & +minio server --config-dir /tmp/minio-internal --address ":9010" http://localhost:9001/tmp/minio-internal-idp1/{1...4} http://localhost:9010/tmp/minio-internal-idp1/{5...8} >/tmp/minio1_2.log 2>&1 & sleep 200 # Test whether most recent tag update on minio2 is replicated to minio1