Add bucket replication test (#2082)

.github/workflows/jobs.yaml

@@ -1368,7 +1368,22 @@ jobs:
           echo "We are going to use the built image on test-integration";
           VERSION="minio/minio:$VERSION";
           echo $VERSION;
-          make test-integration MINIO_VERSION=$VERSION;
+          
+          echo "Create bucket for replication with versioning"
+          echo "Download mc for Ubuntu"
+          wget -q https://dl.min.io/client/mc/release/linux-amd64/mc
+          echo "Change the permissions to execute mc command"
+          chmod +x mc
+          echo "Create the folder to put the all.out file"
+          TARGET_BUCKET=`echo $RANDOM | md5sum | head -c 20; echo;`
+          echo "TARGET_BUCKET: ${TARGET_BUCKET}"
+          ./mc mb --ignore-existing play/${TARGET_BUCKET}/
+          ./mc version enable play/${TARGET_BUCKET}
+          # Via API we are going to test:
+          # mc admin bucket remote add myminio/source https://minioadmin:minioadmin@play.min.io/target --service "replication"
+          # expected output is: Remote ARN = `arn:minio:replication::f5bdb8d7-541d-415e-aaf4-592979484ba9:target`.
+          
+          make test-integration MINIO_VERSION=$VERSION TARGET_BUCKET=$TARGET_BUCKET;
 
       - uses: actions/cache@v2
         id: coverage-cache
@@ -1521,7 +1536,7 @@ jobs:
           go tool cover -func=all.out | grep total > tmp2
           result=`cat tmp2 | awk 'END {print $3}'`
           result=${result%\%}
-          threshold=51.00
+          threshold=51.40
           echo "Result:"
           echo "$result%"
           if (( $(echo "$result >= $threshold" |bc -l) )); then

Makefile

@@ -5,6 +5,7 @@ BUILD_VERSION:=$(shell git describe --exact-match --tags $(git log -n1 --pretty=
 BUILD_TIME:=$(shell date 2>/dev/null)
 TAG ?= "minio/console:$(BUILD_VERSION)-dev"
 MINIO_VERSION ?= "quay.io/minio/minio:latest"
+TARGET_BUCKET ?= "target"
 
 default: console
 
@@ -74,8 +75,8 @@ test-integration:
 	@echo $(MINIO_VERSION)
 	@(docker run -v /data1 -v /data2 -v /data3 -v /data4 --net=mynet123 -d --name minio --rm -p 9000:9000 -p 9001:9001 -e MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= $(MINIO_VERSION) server /data{1...4} --console-address ':9001' && sleep 5)
 	@(docker run --net=mynet123 --ip=173.18.0.3 --name pgsqlcontainer --rm -p 5432:5432 -e POSTGRES_PASSWORD=password -d postgres && sleep 5)
-	@echo "execute test and get coverage"
+	@echo "execute test and get coverage for test-integration:"
-	@(cd integration && go test -coverpkg=../restapi -c -tags testrunmain . && mkdir -p coverage && ./integration.test -test.v -test.run "^Test*" -test.coverprofile=coverage/system.out)
+	@(cd integration && go test -coverpkg=../restapi -c -tags testrunmain . && mkdir -p coverage && export THETARGET=$(TARGET_BUCKET) && echo "THETARGET: ${THETARGET}" && ./integration.test -test.v -test.run "^Test*" -test.coverprofile=coverage/system.out)
 	@(docker stop pgsqlcontainer)
 	@(docker stop minio)
 	@(docker network rm mynet123)

integration/user_api_bucket_test.go

@@ -3638,3 +3638,120 @@ func TestGetBucketRewind(t *testing.T) {
 			200, resp.StatusCode, inspectHTTPResponse(resp))
 	}
 }
+
+func GetRemoteBucket() (*http.Response, error) {
+	request, err := http.NewRequest(
+		"GET",
+		"http://localhost:9090/api/v1/remote-buckets",
+		nil,
+	)
+	if err != nil {
+		log.Println(err)
+	}
+	request.Header.Add("Cookie", fmt.Sprintf("token=%s", token))
+	request.Header.Add("Content-Type", "application/json")
+	client := &http.Client{
+		Timeout: 2 * time.Second,
+	}
+	response, err := client.Do(request)
+	return response, err
+}
+
+func AddRemoteBucket(accessKey, secretKey, targetURL, sourceBucket, targetBucket string) (*http.Response, error) {
+	// Needed Parameters for API Call
+	bucketsRelationArray := make([]map[string]interface{}, 1)
+	bucketsRelationIndex0 := map[string]interface{}{
+		"originBucket":      sourceBucket,
+		"destinationBucket": targetBucket,
+	}
+	bucketsRelationArray[0] = bucketsRelationIndex0
+	requestDataAdd := map[string]interface{}{
+		"accessKey":              accessKey,
+		"secretKey":              secretKey,
+		"targetURL":              targetURL,
+		"sourceBucket":           sourceBucket,
+		"targetBucket":           targetBucket,
+		"region":                 "",
+		"bucketsRelation":        bucketsRelationArray,
+		"syncMode":               "async",
+		"bandwidth":              107374182400,
+		"healthCheckPeriod":      60,
+		"prefix":                 "",
+		"tags":                   "",
+		"replicateDeleteMarkers": true,
+		"replicateDeletes":       true,
+		"priority":               1,
+		"storageClass":           "",
+		"replicateMetadata":      true,
+	}
+
+	// Creating the Call by adding the URL and Headers
+	requestDataJSON, _ := json.Marshal(requestDataAdd)
+	requestDataBody := bytes.NewReader(requestDataJSON)
+	request, err := http.NewRequest(
+		"POST",
+		"http://localhost:9090/api/v1/remote-buckets",
+		requestDataBody,
+	)
+	if err != nil {
+		log.Println(err)
+	}
+	request.Header.Add("Cookie", fmt.Sprintf("token=%s", token))
+	request.Header.Add("Content-Type", "application/json")
+
+	// Performing the call
+	client := &http.Client{
+		Timeout: 2 * time.Second,
+	}
+	response, err := client.Do(request)
+	return response, err
+}
+
+func TestAddRemoteBucket(t *testing.T) {
+	// Variables
+	assert := assert.New(t)
+	accessKey := "minioadmin"
+	secretKey := "minioadmin"
+	targetURL := "https://play.min.io"
+	sourceBucket := "source"
+	targetBucket := os.Getenv("THETARGET")
+	fmt.Println("targetBucket: ", targetBucket)
+
+	// 1. Create bucket
+	if !BucketGotAdded("source", true, true, nil, nil, assert, 201) {
+		return
+	}
+
+	// 2. Add Remote Bucket
+	resp, err := AddRemoteBucket(
+		accessKey,
+		secretKey,
+		targetURL,
+		sourceBucket,
+		targetBucket,
+	)
+	assert.Nil(err)
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	if resp != nil {
+		assert.Equal(
+			201, resp.StatusCode, inspectHTTPResponse(resp))
+	}
+
+	// 3. Verify Remote Bucket was created
+	resp, err = GetRemoteBucket()
+	assert.Nil(err)
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	finalResponse := inspectHTTPResponse(resp)
+	if resp != nil {
+		assert.Equal(
+			200, resp.StatusCode, finalResponse)
+	}
+	fmt.Println("finalResponse: ", finalResponse)
+	assert.Equal(strings.Contains(finalResponse, targetBucket), true)
+}