diff --git a/cmd/endpoint.go b/cmd/endpoint.go index 45fbe229f..78fe487b8 100644 --- a/cmd/endpoint.go +++ b/cmd/endpoint.go @@ -257,18 +257,17 @@ func (l EndpointServerPools) Localhost() string { return "" } -// FirstLocalDiskPath returns the disk path of first (in cmdline args order) -// local endpoint. -func (l EndpointServerPools) FirstLocalDiskPath() string { - var diskPath string +// LocalDisksPaths returns the disk paths of the local disks +func (l EndpointServerPools) LocalDisksPaths() []string { + var disks []string for _, ep := range l { for _, endpoint := range ep.Endpoints { if endpoint.IsLocal { - return endpoint.Path + disks = append(disks, endpoint.Path) } } } - return diskPath + return disks } // FirstLocal returns true if the first endpoint is local. diff --git a/cmd/prepare-storage.go b/cmd/prepare-storage.go index 95d7597ec..9c00b8b5b 100644 --- a/cmd/prepare-storage.go +++ b/cmd/prepare-storage.go @@ -20,18 +20,15 @@ package cmd import ( "context" "crypto/tls" - "errors" "fmt" "net/http" "net/url" - "os" "sync" "time" "github.com/dustin/go-humanize" xhttp "github.com/minio/minio/internal/http" "github.com/minio/minio/internal/logger" - "github.com/minio/minio/internal/sync/errgroup" ) var printEndpointError = func() func(Endpoint, error, bool) { @@ -72,78 +69,37 @@ var printEndpointError = func() func(Endpoint, error, bool) { } }() -// Migrates backend format of local disks. -func formatErasureMigrateLocalEndpoints(endpoints Endpoints) error { - g := errgroup.WithNErrs(len(endpoints)) - for index, endpoint := range endpoints { - if !endpoint.IsLocal { - continue - } - index := index - g.Go(func() error { - epPath := endpoints[index].Path - err := formatErasureMigrate(epPath) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } - return nil - }, index) +// Cleans up tmp directory of the local disk. +func formatErasureCleanupTmp(diskPath string) error { + // Need to move temporary objects left behind from previous run of minio + // server to a unique directory under `minioMetaTmpBucket-old` to clean + // up `minioMetaTmpBucket` for the current run. + // + // /disk1/.minio.sys/tmp-old/ + // |__ 33a58b40-aecc-4c9f-a22f-ff17bfa33b62 + // |__ e870a2c1-d09c-450c-a69c-6eaa54a89b3e + // + // In this example, `33a58b40-aecc-4c9f-a22f-ff17bfa33b62` directory contains + // temporary objects from one of the previous runs of minio server. + tmpOld := pathJoin(diskPath, minioMetaTmpBucket+"-old", mustGetUUID()) + if err := renameAll(pathJoin(diskPath, minioMetaTmpBucket), + tmpOld); err != nil && err != errFileNotFound { + logger.LogIf(GlobalContext, fmt.Errorf("unable to rename (%s -> %s) %w, drive may be faulty please investigate", + pathJoin(diskPath, minioMetaTmpBucket), + tmpOld, + osErrToFileErr(err))) } - for _, err := range g.Wait() { - if err != nil { - return err - } - } - return nil -} -// Cleans up tmp directory of local disks. -func formatErasureCleanupTmpLocalEndpoints(endpoints Endpoints) error { - g := errgroup.WithNErrs(len(endpoints)) - for index, endpoint := range endpoints { - if !endpoint.IsLocal { - continue - } - index := index - g.Go(func() error { - epPath := endpoints[index].Path - // Need to move temporary objects left behind from previous run of minio - // server to a unique directory under `minioMetaTmpBucket-old` to clean - // up `minioMetaTmpBucket` for the current run. - // - // /disk1/.minio.sys/tmp-old/ - // |__ 33a58b40-aecc-4c9f-a22f-ff17bfa33b62 - // |__ e870a2c1-d09c-450c-a69c-6eaa54a89b3e - // - // In this example, `33a58b40-aecc-4c9f-a22f-ff17bfa33b62` directory contains - // temporary objects from one of the previous runs of minio server. - tmpOld := pathJoin(epPath, minioMetaTmpBucket+"-old", mustGetUUID()) - if err := renameAll(pathJoin(epPath, minioMetaTmpBucket), - tmpOld); err != nil && err != errFileNotFound { - logger.LogIf(GlobalContext, fmt.Errorf("unable to rename (%s -> %s) %w, drive may be faulty please investigate", - pathJoin(epPath, minioMetaTmpBucket), - tmpOld, - osErrToFileErr(err))) - } + // Renames and schedules for purging all bucket metacache. + renameAllBucketMetacache(diskPath) - // Renames and schedules for puring all bucket metacache. - renameAllBucketMetacache(epPath) + // Removal of tmp-old folder is backgrounded completely. + go removeAll(pathJoin(diskPath, minioMetaTmpBucket+"-old")) - // Removal of tmp-old folder is backgrounded completely. - go removeAll(pathJoin(epPath, minioMetaTmpBucket+"-old")) - - if err := mkdirAll(pathJoin(epPath, minioMetaTmpBucket), 0777); err != nil { - logger.LogIf(GlobalContext, fmt.Errorf("unable to create (%s) %w, drive may be faulty please investigate", - pathJoin(epPath, minioMetaTmpBucket), - err)) - } - return nil - }, index) - } - for _, err := range g.Wait() { - if err != nil { - return err - } + if err := mkdirAll(pathJoin(diskPath, minioMetaTmpBucket), 0777); err != nil { + logger.LogIf(GlobalContext, fmt.Errorf("unable to create (%s) %w, drive may be faulty please investigate", + pathJoin(diskPath, minioMetaTmpBucket), + err)) } return nil } @@ -337,14 +293,6 @@ func waitForFormatErasure(firstDisk bool, endpoints Endpoints, poolCount, setCou return nil, nil, errInvalidArgument } - if err := formatErasureMigrateLocalEndpoints(endpoints); err != nil { - return nil, nil, err - } - - if err := formatErasureCleanupTmpLocalEndpoints(endpoints); err != nil { - return nil, nil, err - } - // prepare getElapsedTime() to calculate elapsed time since we started trying formatting disks. // All times are rounded to avoid showing milli, micro and nano seconds formatStartTime := time.Now().Round(time.Second) diff --git a/cmd/server-main.go b/cmd/server-main.go index adec098c6..e93fbfd68 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -555,7 +555,7 @@ func serverMain(ctx *cli.Context) { if globalIsErasure { // to be done after config init initBackgroundReplication(GlobalContext, newObject) - globalTierJournal, err = initTierDeletionJournal(GlobalContext.Done()) + globalTierJournal, err = initTierDeletionJournal(GlobalContext) if err != nil { logger.FatalIf(err, "Unable to initialize remote tier pending deletes journal") } diff --git a/cmd/tier-journal.go b/cmd/tier-journal.go index 97479f2c5..984d0781a 100644 --- a/cmd/tier-journal.go +++ b/cmd/tier-journal.go @@ -56,23 +56,28 @@ var ( errUnsupportedJournalVersion = errors.New("unsupported pending deletes journal version") ) -func initTierDeletionJournal(done <-chan struct{}) (*tierJournal, error) { - diskPath := globalEndpoints.FirstLocalDiskPath() - j := &tierJournal{ - diskPath: diskPath, +func initTierDeletionJournal(ctx context.Context) (*tierJournal, error) { + for _, diskPath := range globalEndpoints.LocalDisksPaths() { + j := &tierJournal{ + diskPath: diskPath, + } + + if err := os.MkdirAll(filepath.Dir(j.JournalPath()), os.FileMode(0700)); err != nil { + logger.LogIf(ctx, err) + continue + } + + err := j.Open() + if err != nil { + logger.LogIf(ctx, err) + continue + } + + go j.deletePending(ctx.Done()) + return j, nil } - if err := os.MkdirAll(filepath.Dir(j.JournalPath()), os.FileMode(0700)); err != nil { - return nil, err - } - - err := j.Open() - if err != nil { - return nil, err - } - - go j.deletePending(done) - return j, nil + return nil, errors.New("no local disk found") } // rotate rotates the journal. If a read-only journal already exists it does diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index 3c55b4d45..786698f3c 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -300,7 +300,15 @@ func newXLStorage(ep Endpoint) (*xlStorage, error) { return p, err } w.Close() - defer Remove(filePath) + Remove(filePath) + + if err := formatErasureMigrate(p.diskPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return p, err + } + + if err := formatErasureCleanupTmp(p.diskPath); err != nil { + return p, err + } // Success. return p, nil