Improve TestServer* tests reliability (#508)

* explicitly ignore error from test connection close

* add client with timeout to places which used default http client

* move random port creation and waiting for server in separate function for reuse

* improve TestServer tests robustness

* move all server waiting code in tests to separate functions

* change chooseRandomUnusedPort to try to listen to port before return

* fix waitForHTTPSServerStart

backend/_example/memory_store/server/rpc_test.go

@@ -379,6 +379,29 @@ func TestRPC_admEventHndl(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func chooseRandomUnusedPort() (port int) {
+	for i := 0; i < 10; i++ {
+		port = 40000 + int(rand.Int31n(10000))
+		if ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)); err == nil {
+			_ = ln.Close()
+			break
+		}
+	}
+	return port
+}
+
+func waitForHTTPServerStart(port int) {
+	// wait for up to 3 seconds for server to start before returning it
+	client := http.Client{Timeout: time.Second}
+	for i := 0; i < 300; i++ {
+		time.Sleep(time.Millisecond * 10)
+		if resp, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
+			_ = resp.Body.Close()
+			return
+		}
+	}
+}
+
 func prepTestStore(t *testing.T) (s *RPC, port int, teardown func()) {
 	mg := accessor.NewMemData()
 	adm := accessor.NewMemAdminStore("secret")
@@ -396,28 +419,12 @@ func prepTestStore(t *testing.T) (s *RPC, port int, teardown func()) {
 	admRecDisabled.Enabled = false
 	adm.Set("test-site-disabled", admRecDisabled)
 
-	// check if port is in use before trying to start a new server on it
-	for i := 0; i < 10; i++ {
-		port = 40000 + int(rand.Int31n(10000))
-		conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), time.Millisecond*10)
-		if err != nil {
-			break
-		}
-		if conn != nil {
-			conn.Close()
-		}
-	}
+	port = chooseRandomUnusedPort()
 	go func() {
 		log.Printf("%v", s.Run(port))
 	}()
 
-	// wait for up to 3 seconds for server to start before returning it
-	for i := 0; i < 300; i++ {
-		time.Sleep(time.Millisecond * 10)
-		if _, err := http.Get(fmt.Sprintf("http://localhost:%d", port)); err == nil {
-			break
-		}
-	}
+	waitForHTTPServerStart(port)
 
 	return s, port, func() {
 		require.NoError(t, s.Shutdown())