allow disabling O_DIRECT in certain environments for reads (#14115)

repeated reads on single large objects in HPC like
workloads, need the following option to disable
O_DIRECT for a more effective usage of the kernel
page-cache.

However this optional should be used in very specific
situations only, and shouldn't be enabled on all
servers.

NVMe servers benefit always from keeping O_DIRECT on.

cmd/handler-api.go

@@ -49,6 +49,7 @@ type apiConfig struct {
 	staleUploadsExpiry          time.Duration
 	staleUploadsCleanupInterval time.Duration
 	deleteCleanupInterval       time.Duration
+	disableODirect              bool
 }
 
 const cgroupLimitFile = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
@@ -150,6 +151,14 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 	t.staleUploadsExpiry = cfg.StaleUploadsExpiry
 	t.staleUploadsCleanupInterval = cfg.StaleUploadsCleanupInterval
 	t.deleteCleanupInterval = cfg.DeleteCleanupInterval
+	t.disableODirect = cfg.DisableODirect
+}
+
+func (t *apiConfig) isDisableODirect() bool {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	return t.disableODirect
 }
 
 func (t *apiConfig) getListQuorum() int {