diff --git a/cmd/config/constants.go b/cmd/config/constants.go index 0551519eb..0173f6df4 100644 --- a/cmd/config/constants.go +++ b/cmd/config/constants.go @@ -23,21 +23,23 @@ const ( // Top level common ENVs const ( - EnvAccessKey = "MINIO_ACCESS_KEY" - EnvSecretKey = "MINIO_SECRET_KEY" - EnvRootUser = "MINIO_ROOT_USER" - EnvRootPassword = "MINIO_ROOT_PASSWORD" - EnvAccessKeyOld = "MINIO_ACCESS_KEY_OLD" - EnvSecretKeyOld = "MINIO_SECRET_KEY_OLD" - EnvRootUserOld = "MINIO_ROOT_USER_OLD" - EnvRootPasswordOld = "MINIO_ROOT_PASSWORD_OLD" - EnvBrowser = "MINIO_BROWSER" - EnvDomain = "MINIO_DOMAIN" - EnvRegionName = "MINIO_REGION_NAME" - EnvPublicIPs = "MINIO_PUBLIC_IPS" - EnvFSOSync = "MINIO_FS_OSYNC" - EnvArgs = "MINIO_ARGS" - EnvDNSWebhook = "MINIO_DNS_WEBHOOK_ENDPOINT" + EnvAccessKey = "MINIO_ACCESS_KEY" + EnvSecretKey = "MINIO_SECRET_KEY" + EnvRootUser = "MINIO_ROOT_USER" + EnvRootPassword = "MINIO_ROOT_PASSWORD" + EnvAccessKeyOld = "MINIO_ACCESS_KEY_OLD" + EnvSecretKeyOld = "MINIO_SECRET_KEY_OLD" + EnvRootUserOld = "MINIO_ROOT_USER_OLD" + EnvRootPasswordOld = "MINIO_ROOT_PASSWORD_OLD" + EnvBrowser = "MINIO_BROWSER" + EnvDomain = "MINIO_DOMAIN" + EnvRegionName = "MINIO_REGION_NAME" + EnvPublicIPs = "MINIO_PUBLIC_IPS" + EnvFSOSync = "MINIO_FS_OSYNC" + EnvArgs = "MINIO_ARGS" + EnvDNSWebhook = "MINIO_DNS_WEBHOOK_ENDPOINT" + EnvLogPosixTimes = "MINIO_LOG_POSIX_TIMES" + EnvLogPosixThresholdInMS = "MINIO_LOG_POSIX_THRESHOLD_MS" EnvUpdate = "MINIO_UPDATE" diff --git a/cmd/os-instrumented.go b/cmd/os-instrumented.go new file mode 100644 index 000000000..6114317a5 --- /dev/null +++ b/cmd/os-instrumented.go @@ -0,0 +1,169 @@ +/* + * MinIO Cloud Storage, (C) 2021 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/minio/minio/cmd/config" + "github.com/minio/minio/pkg/disk" + "github.com/minio/minio/pkg/env" +) + +var ( + logTime bool = false + threshold time.Duration +) + +func init() { + logTime = env.IsSet(config.EnvLogPosixTimes) + t, _ := env.GetInt( + config.EnvLogPosixThresholdInMS, + 100, + ) + threshold = time.Duration(t) * time.Millisecond +} + +func getThreshold() time.Duration { + return threshold +} + +func collectLogTime() bool { + return logTime +} + +func reportTime(name *strings.Builder, startTime time.Time) { + delta := time.Since(startTime) + if delta > getThreshold() { + name.WriteString(" ") + name.WriteString(delta.String()) + fmt.Println(name.String()) + } +} + +// RemoveAll captures time taken to call the underlying os.RemoveAll +func RemoveAll(dirPath string) error { + if collectLogTime() { + startTime := time.Now() + var s strings.Builder + s.WriteString("os.RemoveAll: ") + s.WriteString(dirPath) + defer reportTime(&s, startTime) + } + return os.RemoveAll(dirPath) +} + +// MkdirAll captures time taken to call os.MkdirAll +func MkdirAll(dirPath string, mode os.FileMode) error { + if collectLogTime() { + startTime := time.Now() + var s strings.Builder + s.WriteString("os.MkdirAll: ") + s.WriteString(dirPath) + defer reportTime(&s, startTime) + } + return os.MkdirAll(dirPath, mode) +} + +// Rename captures time taken to call os.Rename +func Rename(src, dst string) error { + if collectLogTime() { + startTime := time.Now() + var s strings.Builder + s.WriteString("os.Rename: ") + s.WriteString(src) + s.WriteString(" to ") + s.WriteString(dst) + defer reportTime(&s, startTime) + } + return os.Rename(src, dst) +} + +// OpenFile captures time taken to call os.OpenFile +func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) { + if collectLogTime() { + startTime := time.Now() + var s strings.Builder + s.WriteString("os.OpenFile: ") + s.WriteString(name) + defer reportTime(&s, startTime) + } + return os.OpenFile(name, flag, perm) +} + +// Open captures time taken to call os.Open +func Open(name string) (*os.File, error) { + if collectLogTime() { + startTime := time.Now() + var s strings.Builder + s.WriteString("os.Open: ") + s.WriteString(name) + defer reportTime(&s, startTime) + } + return os.Open(name) +} + +// OpenFileDirectIO captures time taken to call disk.OpenFileDirectIO +func OpenFileDirectIO(name string, flag int, perm os.FileMode) (*os.File, error) { + if collectLogTime() { + startTime := time.Now() + var s strings.Builder + s.WriteString("disk.OpenFileDirectIO: ") + s.WriteString(name) + defer reportTime(&s, startTime) + } + return disk.OpenFileDirectIO(name, flag, perm) +} + +// Lstat captures time taken to call os.Lstat +func Lstat(name string) (os.FileInfo, error) { + if collectLogTime() { + startTime := time.Now() + var s strings.Builder + s.WriteString("os.Lstat: ") + s.WriteString(name) + defer reportTime(&s, startTime) + } + return os.Lstat(name) +} + +// Remove captures time taken to call os.Remove +func Remove(deletePath string) error { + if collectLogTime() { + startTime := time.Now() + var s strings.Builder + s.WriteString("os.Remove: ") + s.WriteString(deletePath) + defer reportTime(&s, startTime) + } + return os.Remove(deletePath) +} + +// Stat captures time taken to call os.Stat +func Stat(name string) (os.FileInfo, error) { + if collectLogTime() { + startTime := time.Now() + var s strings.Builder + s.WriteString("os.Stat: ") + s.WriteString(name) + defer reportTime(&s, startTime) + } + return os.Stat(name) +} diff --git a/cmd/os-readdir_other.go b/cmd/os-readdir_other.go index 256cbeea9..83dfef485 100644 --- a/cmd/os-readdir_other.go +++ b/cmd/os-readdir_other.go @@ -33,7 +33,7 @@ func readDir(dirPath string) (entries []string, err error) { // the directory itself, if the dirPath doesn't exist this function doesn't return // an error. func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error) error { - d, err := os.Open(dirPath) + d, err := Open(dirPath) if err != nil { if osErrToFileErr(err) == errFileNotFound { return nil @@ -58,7 +58,7 @@ func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error) } for _, fi := range fis { if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - fi, err = os.Stat(pathJoin(dirPath, fi.Name())) + fi, err = Stat(pathJoin(dirPath, fi.Name())) if err != nil { // It got deleted in the meantime, not found // or returns too many symlinks ignore this @@ -86,7 +86,7 @@ func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error) // Return N entries at the directory dirPath. If count is -1, return all entries func readDirN(dirPath string, count int) (entries []string, err error) { - d, err := os.Open(dirPath) + d, err := Open(dirPath) if err != nil { return nil, osErrToFileErr(err) } @@ -117,7 +117,7 @@ func readDirN(dirPath string, count int) (entries []string, err error) { } for _, fi := range fis { if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - fi, err = os.Stat(pathJoin(dirPath, fi.Name())) + fi, err = Stat(pathJoin(dirPath, fi.Name())) if err != nil { // It got deleted in the meantime, not found // or returns too many symlinks ignore this diff --git a/cmd/os-reliable.go b/cmd/os-reliable.go index 0edecb5e2..3c9be855e 100644 --- a/cmd/os-reliable.go +++ b/cmd/os-reliable.go @@ -57,7 +57,7 @@ func reliableRemoveAll(dirPath string) (err error) { i := 0 for { // Removes all the directories and files. - if err = os.RemoveAll(dirPath); err != nil { + if err = RemoveAll(dirPath); err != nil { // Retry only for the first retryable error. if isSysErrNotEmpty(err) && i == 0 { i++ @@ -101,7 +101,7 @@ func reliableMkdirAll(dirPath string, mode os.FileMode) (err error) { i := 0 for { // Creates all the parent directories, with mode 0777 mkdir honors system umask. - if err = os.MkdirAll(dirPath, mode); err != nil { + if err = MkdirAll(dirPath, mode); err != nil { // Retry only for the first retryable error. if osIsNotExist(err) && i == 0 { i++ @@ -166,7 +166,7 @@ func reliableRename(srcFilePath, dstFilePath string) (err error) { i := 0 for { // After a successful parent directory create attempt a renameAll. - if err = os.Rename(srcFilePath, dstFilePath); err != nil { + if err = Rename(srcFilePath, dstFilePath); err != nil { // Retry only for the first retryable error. if osIsNotExist(err) && i == 0 { i++ diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index a13e46ff7..c9b84b598 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -172,7 +172,7 @@ func getValidPath(path string) (string, error) { return path, err } - fi, err := os.Lstat(path) + fi, err := Lstat(path) if err != nil && !osIsNotExist(err) { return path, err } @@ -296,7 +296,7 @@ func newXLStorage(ep Endpoint) (*xlStorage, error) { return p, err } w.Close() - defer os.Remove(filePath) + defer Remove(filePath) // Success. return p, nil @@ -525,11 +525,11 @@ func (s *xlStorage) GetDiskID() (string, error) { s.Unlock() formatFile := pathJoin(s.diskPath, minioMetaBucket, formatConfigFile) - fi, err := os.Lstat(formatFile) + fi, err := Lstat(formatFile) if err != nil { // If the disk is still not initialized. if osIsNotExist(err) { - _, err = os.Lstat(s.diskPath) + _, err = Lstat(s.diskPath) if err == nil { // Disk is present but missing `format.json` return "", errUnformattedDisk @@ -560,7 +560,7 @@ func (s *xlStorage) GetDiskID() (string, error) { if err != nil { // If the disk is still not initialized. if osIsNotExist(err) { - _, err = os.Lstat(s.diskPath) + _, err = Lstat(s.diskPath) if err == nil { // Disk is present but missing `format.json` return "", errUnformattedDisk @@ -623,7 +623,7 @@ func (s *xlStorage) MakeVol(ctx context.Context, volume string) error { return err } - if _, err := os.Lstat(volumeDir); err != nil { + if _, err := Lstat(volumeDir); err != nil { // Volume does not exist we proceed to create. if osIsNotExist(err) { // Make a volume entry, with mode 0777 mkdir honors system umask. @@ -677,7 +677,7 @@ func (s *xlStorage) StatVol(ctx context.Context, volume string) (vol VolInfo, er } // Stat a volume entry. var st os.FileInfo - st, err = os.Lstat(volumeDir) + st, err = Lstat(volumeDir) if err != nil { switch { case osIsNotExist(err): @@ -708,9 +708,9 @@ func (s *xlStorage) DeleteVol(ctx context.Context, volume string, forceDelete bo } if forceDelete { - err = os.RemoveAll(volumeDir) + err = RemoveAll(volumeDir) } else { - err = os.Remove(volumeDir) + err = Remove(volumeDir) } if err != nil { @@ -736,7 +736,7 @@ func (s *xlStorage) isLeaf(volume string, leafPath string) bool { return false } - _, err = os.Lstat(pathJoin(volumeDir, leafPath, xlStorageFormatFile)) + _, err = Lstat(pathJoin(volumeDir, leafPath, xlStorageFormatFile)) if err == nil { return true } @@ -767,7 +767,7 @@ func (s *xlStorage) ListDir(ctx context.Context, volume, dirPath string, count i } if err != nil { if err == errFileNotFound { - if _, verr := os.Lstat(volumeDir); verr != nil { + if _, verr := Lstat(volumeDir); verr != nil { if osIsNotExist(verr) { return nil, errVolumeNotFound } else if isSysErrIO(verr) { @@ -942,7 +942,7 @@ func (s *xlStorage) renameLegacyMetadata(volumeDir, path string) (err error) { } }() - if err = os.Rename(srcFilePath, dstFilePath); err != nil { + if err = Rename(srcFilePath, dstFilePath); err != nil { switch { case isSysErrNotDir(err): return errFileNotFound @@ -1035,13 +1035,13 @@ func (s *xlStorage) readAllData(volumeDir string, filePath string, requireDirect if requireDirectIO { f, err = disk.OpenFileDirectIO(filePath, readMode, 0666) } else { - f, err = os.OpenFile(filePath, readMode, 0) + f, err = OpenFile(filePath, readMode, 0) } if err != nil { if osIsNotExist(err) { // Check if the object doesn't exist because its bucket // is missing in order to return the correct error. - _, err = os.Lstat(volumeDir) + _, err = Lstat(volumeDir) if err != nil && osIsNotExist(err) { return nil, errVolumeNotFound } @@ -1058,7 +1058,7 @@ func (s *xlStorage) readAllData(volumeDir string, filePath string, requireDirect } else if isSysErrTooManyFiles(err) { return nil, errTooManyOpenFiles } else if isSysErrInvalidArg(err) { - st, _ := os.Lstat(filePath) + st, _ := Lstat(filePath) if st != nil && st.IsDir() { // Linux returns InvalidArg for directory O_DIRECT // we need to keep this fallback code to return correct @@ -1128,7 +1128,7 @@ func (s *xlStorage) ReadFile(ctx context.Context, volume string, path string, of var n int // Stat a volume entry. - _, err = os.Lstat(volumeDir) + _, err = Lstat(volumeDir) if err != nil { if osIsNotExist(err) { return 0, errVolumeNotFound @@ -1145,7 +1145,7 @@ func (s *xlStorage) ReadFile(ctx context.Context, volume string, path string, of } // Open the file for reading. - file, err := os.Open(filePath) + file, err := Open(filePath) if err != nil { switch { case osIsNotExist(err): @@ -1223,7 +1223,7 @@ func (s *xlStorage) openFile(volume, path string, mode int) (f *os.File, err err return nil, osErrToFileErr(err) } - w, err := os.OpenFile(filePath, mode|writeMode, 0666) + w, err := OpenFile(filePath, mode|writeMode, 0666) if err != nil { // File path cannot be verified since one of the parents is a file. switch { @@ -1333,12 +1333,12 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off file, err = disk.OpenFileDirectIO(filePath, readMode, 0666) } else { // Open the file for reading. - file, err = os.OpenFile(filePath, readMode, 0666) + file, err = OpenFile(filePath, readMode, 0666) } if err != nil { switch { case osIsNotExist(err): - _, err = os.Lstat(volumeDir) + _, err = Lstat(volumeDir) if err != nil && osIsNotExist(err) { return nil, errVolumeNotFound } @@ -1461,7 +1461,7 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz return osErrToFileErr(err) } - w, err := disk.OpenFileDirectIO(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0666) + w, err := OpenFileDirectIO(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0666) if err != nil { return osErrToFileErr(err) } @@ -1516,7 +1516,7 @@ func (s *xlStorage) AppendFile(ctx context.Context, volume string, path string, } // Stat a volume entry. - if _, err = os.Lstat(volumeDir); err != nil { + if _, err = Lstat(volumeDir); err != nil { if osIsNotExist(err) { return errVolumeNotFound } @@ -1552,7 +1552,7 @@ func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string, } // Stat a volume entry. - if _, err = os.Lstat(volumeDir); err != nil { + if _, err = Lstat(volumeDir); err != nil { if osIsNotExist(err) { return errVolumeNotFound } @@ -1568,7 +1568,7 @@ func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string, if err = checkPathLength(filePath); err != nil { return err } - st, err := os.Lstat(filePath) + st, err := Lstat(filePath) if err != nil { return osErrToFileErr(err) } @@ -1608,7 +1608,7 @@ func (s *xlStorage) CheckFile(ctx context.Context, volume string, path string) e return err } - st, _ := os.Lstat(filePath) + st, _ := Lstat(filePath) if st == nil { if !s.formatLegacy { return errPathNotFound @@ -1619,7 +1619,7 @@ func (s *xlStorage) CheckFile(ctx context.Context, volume string, path string) e return err } - st, _ = os.Lstat(filePathOld) + st, _ = Lstat(filePathOld) if st == nil { return errPathNotFound } @@ -1661,7 +1661,7 @@ func (s *xlStorage) deleteFile(basePath, deletePath string, recursive bool) erro tmpuuid := mustGetUUID() err = renameAll(deletePath, pathutil.Join(s.diskPath, minioMetaTmpDeletedBucket, tmpuuid)) } else { - err = os.Remove(deletePath) + err = Remove(deletePath) } if err != nil { switch { @@ -1703,7 +1703,7 @@ func (s *xlStorage) Delete(ctx context.Context, volume string, path string, recu } // Stat a volume entry. - _, err = os.Lstat(volumeDir) + _, err = Lstat(volumeDir) if err != nil { if osIsNotExist(err) { return errVolumeNotFound @@ -1739,7 +1739,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, } // Stat a volume entry. - _, err = os.Lstat(srcVolumeDir) + _, err = Lstat(srcVolumeDir) if err != nil { if osIsNotExist(err) { return errVolumeNotFound @@ -1749,7 +1749,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, return err } - if _, err = os.Lstat(dstVolumeDir); err != nil { + if _, err = Lstat(dstVolumeDir); err != nil { if osIsNotExist(err) { return errVolumeNotFound } else if isSysErrIO(err) { @@ -1912,7 +1912,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir, continue } - if err = os.Rename(pathJoin(currentDataPath, entry), pathJoin(legacyDataPath, entry)); err != nil { + if err = Rename(pathJoin(currentDataPath, entry), pathJoin(legacyDataPath, entry)); err != nil { return osErrToFileErr(err) } } @@ -1980,7 +1980,7 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum return err } // Stat a volume entry. - _, err = os.Lstat(srcVolumeDir) + _, err = Lstat(srcVolumeDir) if err != nil { if osIsNotExist(err) { return errVolumeNotFound @@ -1989,7 +1989,7 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum } return err } - _, err = os.Lstat(dstVolumeDir) + _, err = Lstat(dstVolumeDir) if err != nil { if osIsNotExist(err) { return errVolumeNotFound @@ -2017,7 +2017,7 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum // If source is a directory, we expect the destination to be non-existent but we // we still need to allow overwriting an empty directory since it represents // an object empty directory. - dirInfo, err := os.Lstat(dstFilePath) + dirInfo, err := Lstat(dstFilePath) if isSysErrIO(err) { return errFaultyDisk } @@ -2029,7 +2029,7 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum if !dirInfo.IsDir() { return errFileAccessDenied } - if err = os.Remove(dstFilePath); err != nil { + if err = Remove(dstFilePath); err != nil { if isSysErrNotEmpty(err) { return errFileAccessDenied } @@ -2051,7 +2051,7 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum func (s *xlStorage) bitrotVerify(partPath string, partSize int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error { // Open the file for reading. - file, err := os.Open(partPath) + file, err := Open(partPath) if err != nil { return osErrToFileErr(err) } @@ -2124,7 +2124,7 @@ func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi File } // Stat a volume entry. - _, err = os.Lstat(volumeDir) + _, err = Lstat(volumeDir) if err != nil { if osIsNotExist(err) { return errVolumeNotFound