From 689959e29f68b00884d73f5c9d0c71ad9308fbb3 Mon Sep 17 00:00:00 2001 From: Samuel N Cui Date: Wed, 11 Mar 2026 00:55:46 +0800 Subject: [PATCH] feat: add acp-rewrite --- attrs.go | 24 ++ cleanup.go | 6 +- cmd/acp-rewrite/file_other.go | 16 + cmd/acp-rewrite/file_unix.go | 50 +++ cmd/acp-rewrite/file_windows.go | 16 + cmd/acp-rewrite/main.go | 546 ++++++++++++++++++++++++++++++++ cmd/acp-rewrite/main_test.go | 137 ++++++++ cmd/acp-rewrite/task.md | 13 + copy.go | 2 +- index.go | 50 ++- job.go | 12 +- stat.go | 28 ++ syscall_other.go | 2 +- syscall_unix.go | 2 +- syscall_windows.go | 2 +- 15 files changed, 864 insertions(+), 42 deletions(-) create mode 100644 attrs.go create mode 100644 cmd/acp-rewrite/file_other.go create mode 100644 cmd/acp-rewrite/file_unix.go create mode 100644 cmd/acp-rewrite/file_windows.go create mode 100644 cmd/acp-rewrite/main.go create mode 100644 cmd/acp-rewrite/main_test.go create mode 100644 cmd/acp-rewrite/task.md create mode 100644 stat.go diff --git a/attrs.go b/attrs.go new file mode 100644 index 0000000..066066c --- /dev/null +++ b/attrs.go @@ -0,0 +1,24 @@ +package acp + +import ( + "fmt" + "os" +) + +func CopyAttrs(dst, src string) error { + fi, err := os.Stat(src) + if err != nil { + return fmt.Errorf("get src stat failed, path= %q, %w", src, err) + } + + stat, err := newStat(src, fi) + if err != nil { + return fmt.Errorf("new stat failed, path= %q, %w", src, err) + } + + if err := writeSysStat(dst, stat); err != nil { + return fmt.Errorf("write sys stat failed, path= %q, %w", dst, err) + } + + return nil +} diff --git a/cleanup.go b/cleanup.go index 1228966..49b7484 100644 --- a/cleanup.go +++ b/cleanup.go @@ -13,9 +13,9 @@ func (c *Copyer) cleanupJob(ctx context.Context, copyed <-chan *baseJob) { return } - for _, name := range job.successTargets { - if err := writeSysStat(name, job); err != nil { - c.reportError(job.path, name, fmt.Errorf("change info, write sys stat fail, %w", err)) + for _, dst := range job.successTargets { + if err := writeSysStat(dst, job.stat); err != nil { + c.reportError(job.path, dst, fmt.Errorf("change info, write sys stat fail, %w", err)) } } diff --git a/cmd/acp-rewrite/file_other.go b/cmd/acp-rewrite/file_other.go new file mode 100644 index 0000000..4d61fe0 --- /dev/null +++ b/cmd/acp-rewrite/file_other.go @@ -0,0 +1,16 @@ +//go:build !darwin && !linux && !windows +// +build !darwin,!linux,!windows + +package main + +import "io/fs" + +func isFileBusy(path string) (bool, error) { + return false, nil +} + +type fileIdentity struct{} + +func checkFileLinked(info fs.FileInfo) (fileIdentity, bool) { + return fileIdentity{}, false +} diff --git a/cmd/acp-rewrite/file_unix.go b/cmd/acp-rewrite/file_unix.go new file mode 100644 index 0000000..0cacf20 --- /dev/null +++ b/cmd/acp-rewrite/file_unix.go @@ -0,0 +1,50 @@ +//go:build darwin || linux +// +build darwin linux + +package main + +import ( + "io/fs" + "os" + "syscall" +) + +func isFileBusy(path string) (bool, error) { + f, err := os.OpenFile(path, os.O_RDONLY, 0) + if err != nil { + return false, err + } + defer f.Close() + + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + if err == syscall.EWOULDBLOCK || err == syscall.EAGAIN { + return true, nil + } + return false, err + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_UN); err != nil { + return false, err + } + return false, nil +} + +type fileIdentity struct { + dev uint64 + ino uint64 + nlink uint64 +} + +func checkFileLinked(info fs.FileInfo) (fileIdentity, bool) { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || stat == nil { + return fileIdentity{}, false + } + if stat.Nlink <= 1 { + return fileIdentity{}, false + } + return fileIdentity{ + dev: uint64(stat.Dev), + ino: stat.Ino, + nlink: uint64(stat.Nlink), + }, true +} diff --git a/cmd/acp-rewrite/file_windows.go b/cmd/acp-rewrite/file_windows.go new file mode 100644 index 0000000..58f2e31 --- /dev/null +++ b/cmd/acp-rewrite/file_windows.go @@ -0,0 +1,16 @@ +//go:build windows +// +build windows + +package main + +import "io/fs" + +func isFileBusy(path string) (bool, error) { + return false, nil +} + +type fileIdentity struct{} + +func checkFileLinked(info fs.FileInfo) (fileIdentity, bool) { + return fileIdentity{}, false +} diff --git a/cmd/acp-rewrite/main.go b/cmd/acp-rewrite/main.go new file mode 100644 index 0000000..1dce3be --- /dev/null +++ b/cmd/acp-rewrite/main.go @@ -0,0 +1,546 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "io/fs" + "math/rand" + "os" + "os/signal" + "path" + "path/filepath" + "sort" + "time" + + "github.com/samuelncui/acp" + "github.com/schollz/progressbar/v3" + "github.com/sirupsen/logrus" +) + +var ( + randSource = rand.New(rand.NewSource(time.Now().UnixNano())) +) + +type rewriteEntry struct { + Path string `json:"path"` + Links []string `json:"links,omitempty"` +} + +type rewriteState struct { + Root string `json:"root"` + Pending []rewriteEntry `json:"pending,omitempty"` + Busy []rewriteEntry `json:"busy,omitempty"` + TmpFiles []string `json:"tmp_files,omitempty"` +} + +func main() { + withProgressBar := flag.Bool("p", true, "display progress bar") + dryRun := flag.Bool("dryrun", false, "only generate task list without rewriting") + statePath := flag.String("state", ".acp-rewrite-state.json", "state storage path") + reportPath := flag.String("report", "", "json report storage path") + reportIndent := flag.Bool("report-indent", false, "json report with indent") + + flag.Parse() + if flag.NArg() == 0 { + logrus.Fatalf("path required") + } + + root := flag.Arg(0) + rootAbs, err := filepath.Abs(root) + if err != nil { + logrus.Fatalf("get abs root fail, %s", err) + } + stateAbs, err := filepath.Abs(*statePath) + if err != nil { + logrus.Fatalf("get abs state path fail, %s", err) + } + var reportAbs string + if *reportPath != "" { + reportAbs, err = filepath.Abs(*reportPath) + if err != nil { + logrus.Fatalf("get abs report path fail, %s", err) + } + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + + state, err := loadState(stateAbs) + if err != nil { + logrus.Fatalf("load state fail, %s", err) + } + if state == nil || state.Root != rootAbs { + state = &rewriteState{Root: rootAbs} + } + + reportJobs, reportErrors := loadReport(*reportPath) + + if len(state.TmpFiles) > 0 { + if err := cleanupTmpFiles(state); err != nil { + logrus.Warnf("cleanup tmp files fail, %s", err) + } + if err := saveState(stateAbs, state); err != nil { + logrus.Fatalf("save state fail, %s", err) + } + } + + if len(state.Pending) == 0 && len(state.Busy) == 0 { + entries, err := scanEntries(rootAbs, stateAbs, reportAbs) + if err != nil { + logrus.Fatalf("scan path fail, %s", err) + } + state.Pending = entries + if err := saveState(stateAbs, state); err != nil { + logrus.Fatalf("save state fail, %s", err) + } + } + if *dryRun { + logrus.Infof("dryrun tasks= %d", len(state.Pending)+len(state.Busy)) + return + } + + queue := append([]rewriteEntry{}, state.Pending...) + queue = append(queue, state.Busy...) + state.Pending = nil + state.Busy = nil + if err := saveState(stateAbs, state); err != nil { + logrus.Fatalf("save state fail, %s", err) + } + + var bar *progressbar.ProgressBar + if *withProgressBar { + bar = progressbar.NewOptions(len(queue)) + } + + var currentTmp string + for idx, entry := range queue { + if ctx.Err() != nil { + if currentTmp != "" { + _ = os.Remove(currentTmp) + currentTmp = "" + } + state.Pending = append(state.Pending, queue[idx:]...) + if err := saveState(stateAbs, state); err != nil { + logrus.Fatalf("save state fail, %s", err) + } + return + } + + busy, err := isFileBusy(entry.Path) + if err != nil { + logrus.Warnf("check busy fail, path= '%s', err= %s", entry.Path, err) + state.Pending = append(state.Pending, entry) + if bar != nil { + _ = bar.Add(1) + } + continue + } + if busy { + state.Busy = append(state.Busy, entry) + if bar != nil { + _ = bar.Add(1) + } + continue + } + + tmpPath, err := newTmpPath(entry.Path, randSource) + if err != nil { + logrus.Warnf("generate tmp fail, path= '%s', err= %s", entry.Path, err) + state.Pending = append(state.Pending, entry) + if bar != nil { + _ = bar.Add(1) + } + continue + } + + state.TmpFiles = append(state.TmpFiles, tmpPath) + if err := saveState(stateAbs, state); err != nil { + logrus.Fatalf("save state fail, %s", err) + } + + currentTmp = tmpPath + report, err := rewriteFile(ctx, entry, tmpPath) + currentTmp = "" + + state.TmpFiles = removeTmp(state.TmpFiles, tmpPath) + if err := saveState(stateAbs, state); err != nil { + logrus.Fatalf("save state fail, %s", err) + } + + if err != nil { + logrus.Warnf("rewrite fail, path= '%s', err= %s", entry.Path, err) + state.Pending = append(state.Pending, entry) + if bar != nil { + _ = bar.Add(1) + } + continue + } + + if err := relink(entry); err != nil { + logrus.Warnf("relink fail, path= '%s', err= %s", entry.Path, err) + } + + mergeReport(reportJobs, &reportErrors, report) + if err := saveReport(*reportPath, *reportIndent, reportJobs, reportErrors); err != nil { + logrus.Warnf("save report fail, %s", err) + } + if err := saveState(stateAbs, state); err != nil { + logrus.Fatalf("save state fail, %s", err) + } + if bar != nil { + _ = bar.Add(1) + } + } + + if err := saveState(stateAbs, state); err != nil { + logrus.Fatalf("save state fail, %s", err) + } + if err := saveReport(*reportPath, *reportIndent, reportJobs, reportErrors); err != nil { + logrus.Warnf("save report fail, %s", err) + } + + printDuplicates(reportJobs) +} + +func scanEntries(root, statePath, reportPath string) ([]rewriteEntry, error) { + groups := make(map[fileIdentity][]string) + entries := make([]rewriteEntry, 0, 128) + + err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if p == statePath || (reportPath != "" && p == reportPath) { + return nil + } + if d.IsDir() { + return nil + } + + info, err := d.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return nil + } + + id, linked := checkFileLinked(info) + if linked { + groups[id] = append(groups[id], p) + return nil + } + + entries = append(entries, rewriteEntry{Path: p}) + return nil + }) + if err != nil { + return nil, err + } + + for _, paths := range groups { + sort.Strings(paths) + if len(paths) == 0 { + continue + } + entry := rewriteEntry{Path: paths[0]} + if len(paths) > 1 { + entry.Links = append(entry.Links, paths[1:]...) + } + entries = append(entries, entry) + } + + sort.Slice(entries, func(i, j int) bool { return entries[i].Path < entries[j].Path }) + return entries, nil +} + +func newTmpPath(path string, rnd *rand.Rand) (string, error) { + suffix := func() string { + const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + buf := make([]byte, 4) + for i := range buf { + buf[i] = letters[rnd.Intn(len(letters))] + } + return string(buf) + } + for i := 0; i < 10; i++ { + tmp := fmt.Sprintf("%s.tmp%s", path, suffix()) + if _, err := os.Stat(tmp); err != nil { + if os.IsNotExist(err) { + return tmp, nil + } + return "", err + } + } + return "", fmt.Errorf("tmp path collide") +} + +func rewriteFile(ctx context.Context, entry rewriteEntry, tmpPath string) (*acp.Report, error) { + stat, err := os.Stat(entry.Path) + if err != nil { + return nil, err + } + if !stat.Mode().IsRegular() { + return nil, nil + } + + opts := []acp.Option{ + acp.AccurateJob(entry.Path, []string{tmpPath}), + acp.WithHash(true), + acp.Overwrite(true), + } + + handler, getter := acp.NewReportGetter() + opts = append(opts, acp.WithEventHandler(handler)) + + c, err := acp.New(ctx, opts...) + if err != nil { + return nil, err + } + c.Wait() + + report := getter() + if report == nil { + return nil, fmt.Errorf("report nil") + } + if len(report.Errors) > 0 { + return report, report.Errors[0] + } + + job, ok := findJob(report, entry.Path) + if !ok { + return report, fmt.Errorf("job not found") + } + if len(job.FailTargets) > 0 { + return report, fmt.Errorf("copy fail") + } + if len(job.SuccessTargets) == 0 { + return report, fmt.Errorf("copy not finished") + } + + if err := os.Rename(tmpPath, entry.Path); err != nil { + if remErr := os.Remove(entry.Path); remErr != nil { + return report, err + } + if err2 := os.Rename(tmpPath, entry.Path); err2 != nil { + return report, err2 + } + } + + return report, nil +} + +func relink(entry rewriteEntry) error { + if len(entry.Links) == 0 { + return nil + } + for _, link := range entry.Links { + if link == entry.Path { + continue + } + if err := relinkOne(entry.Path, link); err != nil { + return err + } + } + return nil +} + +func relinkOne(src, link string) error { + tmpPath, err := newTmpPath(link, randSource) + if err != nil { + return fmt.Errorf("new tmp path fail, path= %q, %w", link, err) + } + if err := os.Link(src, tmpPath); err != nil { + return fmt.Errorf("create link fail, src= %q dst= %q, %w", src, tmpPath, err) + } + + if err := acp.CopyAttrs(tmpPath, link); err != nil { + _ = os.Remove(tmpPath) + return fmt.Errorf("copy attrs fail, dst= %q src= %q, %w", tmpPath, link, err) + } + + if err := os.Rename(tmpPath, link); err != nil { + _ = os.Remove(tmpPath) + return fmt.Errorf("rename tmp fail, from= %q to= %q, %w", tmpPath, link, err) + } + + return nil +} + +func loadState(path string) (*rewriteState, error) { + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + var state rewriteState + if err := json.NewDecoder(f).Decode(&state); err != nil { + return nil, err + } + return &state, nil +} + +func saveState(path string, state *rewriteState) error { + if state == nil { + return nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + + tmp := path + ".tmp" + f, err := os.Create(tmp) + if err != nil { + return err + } + enc := json.NewEncoder(f) + enc.SetIndent("", "\t") + if err := enc.Encode(state); err != nil { + _ = f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + return os.Rename(tmp, path) +} + +func cleanupTmpFiles(state *rewriteState) error { + for _, tmp := range state.TmpFiles { + _ = os.Remove(tmp) + } + state.TmpFiles = nil + return nil +} + +func removeTmp(tmpFiles []string, tmp string) []string { + next := make([]string, 0, len(tmpFiles)) + for _, t := range tmpFiles { + if t == tmp { + continue + } + next = append(next, t) + } + return next +} + +func findJob(report *acp.Report, filePath string) (*acp.Job, bool) { + for _, job := range report.Jobs { + full := job.Base + path.Join(job.Path...) + if full == filePath { + return job, true + } + } + return nil, false +} + +func loadReport(path string) (map[string]*acp.Job, []*acp.Error) { + jobs := make(map[string]*acp.Job, 128) + errors := make([]*acp.Error, 0) + if path == "" { + return jobs, errors + } + if _, err := os.Stat(path); err != nil { + return jobs, errors + } + + f, err := os.Open(path) + if err != nil { + return jobs, errors + } + defer f.Close() + + var report acp.Report + if err := json.NewDecoder(f).Decode(&report); err != nil { + return jobs, errors + } + + mergeReport(jobs, &errors, &report) + return jobs, errors +} + +func mergeReport(jobs map[string]*acp.Job, errors *[]*acp.Error, report *acp.Report) { + if report == nil { + return + } + for _, job := range report.Jobs { + full := job.Base + path.Join(job.Path...) + jobs[full] = job + } + if len(report.Errors) > 0 { + *errors = append(*errors, report.Errors...) + } +} + +func saveReport(path string, indent bool, jobs map[string]*acp.Job, errors []*acp.Error) error { + if path == "" { + return nil + } + report := &acp.Report{ + Jobs: make([]*acp.Job, 0, len(jobs)), + Errors: errors, + } + for _, job := range jobs { + report.Jobs = append(report.Jobs, job) + } + + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + enc := json.NewEncoder(f) + if indent { + enc.SetIndent("", "\t") + } + return enc.Encode(report) +} + +func printDuplicates(jobs map[string]*acp.Job) { + type dupKey struct { + size int64 + hash string + } + dups := make(map[dupKey][]string) + for _, job := range jobs { + if job == nil || job.SHA256 == "" || job.Size == 0 { + continue + } + full := job.Base + path.Join(job.Path...) + key := dupKey{size: job.Size, hash: job.SHA256} + dups[key] = append(dups[key], full) + } + + keys := make([]dupKey, 0, len(dups)) + for key := range dups { + if len(dups[key]) > 1 { + keys = append(keys, key) + } + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].size != keys[j].size { + return keys[i].size < keys[j].size + } + return keys[i].hash < keys[j].hash + }) + + for _, key := range keys { + paths := dups[key] + sort.Strings(paths) + logrus.Infof("duplicate size= %d sha256= %s files= %v", key.size, key.hash, paths) + } +} diff --git a/cmd/acp-rewrite/main_test.go b/cmd/acp-rewrite/main_test.go new file mode 100644 index 0000000..a76a6ab --- /dev/null +++ b/cmd/acp-rewrite/main_test.go @@ -0,0 +1,137 @@ +package main + +import ( + "os" + "path/filepath" + "runtime" + "testing" + "time" +) + +func TestScanEntriesHardlinks(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("hardlink scan unsupported on windows") + } + + root := t.TempDir() + statePath := filepath.Join(root, ".acp-rewrite-state.json") + reportPath := filepath.Join(root, "report.json") + + if err := os.WriteFile(statePath, []byte("{}"), 0o644); err != nil { + t.Fatalf("write state: %v", err) + } + if err := os.WriteFile(reportPath, []byte("{}"), 0o644); err != nil { + t.Fatalf("write report: %v", err) + } + + a := filepath.Join(root, "a.txt") + b := filepath.Join(root, "b.txt") + c := filepath.Join(root, "c.txt") + + if err := os.WriteFile(a, []byte("a"), 0o644); err != nil { + t.Fatalf("write a: %v", err) + } + if err := os.Link(a, b); err != nil { + t.Fatalf("link b: %v", err) + } + if err := os.WriteFile(c, []byte("c"), 0o644); err != nil { + t.Fatalf("write c: %v", err) + } + + entries, err := scanEntries(root, statePath, reportPath) + if err != nil { + t.Fatalf("scan entries: %v", err) + } + if len(entries) != 2 { + t.Fatalf("entries count = %d", len(entries)) + } + + var withLinks *rewriteEntry + var single *rewriteEntry + for i := range entries { + if len(entries[i].Links) > 0 { + withLinks = &entries[i] + } else { + single = &entries[i] + } + } + if withLinks == nil || single == nil { + t.Fatalf("unexpected entries: %+v", entries) + } + if withLinks.Path != a { + t.Fatalf("hardlink entry path = %q", withLinks.Path) + } + if len(withLinks.Links) != 1 || withLinks.Links[0] != b { + t.Fatalf("hardlink entry links = %+v", withLinks.Links) + } + if single.Path != c { + t.Fatalf("single entry path = %q", single.Path) + } +} + +func TestRelinkOnePreserveLinkAttrs(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("hardlink attrs unsupported on windows") + } + + root := t.TempDir() + src := filepath.Join(root, "src.txt") + link := filepath.Join(root, "link.txt") + tmp := filepath.Join(root, "tmp.txt") + + if err := os.WriteFile(src, []byte("old"), 0o644); err != nil { + t.Fatalf("write src: %v", err) + } + if err := os.Link(src, link); err != nil { + t.Fatalf("link: %v", err) + } + + oldTime := time.Unix(1700000000, 0) + newTime := time.Unix(1700001000, 0) + + if err := os.Chmod(link, 0o600); err != nil { + t.Fatalf("chmod link: %v", err) + } + if err := os.Chtimes(link, oldTime, oldTime); err != nil { + t.Fatalf("chtimes link: %v", err) + } + + if err := os.WriteFile(tmp, []byte("new"), 0o644); err != nil { + t.Fatalf("write tmp: %v", err) + } + if err := os.Chtimes(tmp, newTime, newTime); err != nil { + t.Fatalf("chtimes tmp: %v", err) + } + if err := os.Rename(tmp, src); err != nil { + t.Fatalf("rename tmp to src: %v", err) + } + + if err := relinkOne(src, link); err != nil { + t.Fatalf("relinkOne: %v", err) + } + + srcInfo, err := os.Stat(src) + if err != nil { + t.Fatalf("stat src: %v", err) + } + linkInfo, err := os.Stat(link) + if err != nil { + t.Fatalf("stat link: %v", err) + } + if !os.SameFile(srcInfo, linkInfo) { + t.Fatalf("src and link not same file") + } + if srcInfo.Mode().Perm() != 0o600 || linkInfo.Mode().Perm() != 0o600 { + t.Fatalf("mode mismatch src=%v link=%v", srcInfo.Mode().Perm(), linkInfo.Mode().Perm()) + } + if srcInfo.ModTime().Unix() != oldTime.Unix() || linkInfo.ModTime().Unix() != oldTime.Unix() { + t.Fatalf("modtime mismatch src=%v link=%v", srcInfo.ModTime(), linkInfo.ModTime()) + } + data, err := os.ReadFile(link) + if err != nil { + t.Fatalf("read link: %v", err) + } + if string(data) != "new" { + t.Fatalf("link content = %q", string(data)) + } +} diff --git a/cmd/acp-rewrite/task.md b/cmd/acp-rewrite/task.md new file mode 100644 index 0000000..c6abac7 --- /dev/null +++ b/cmd/acp-rewrite/task.md @@ -0,0 +1,13 @@ +add a acp-rewrite command: + +- scan the whole path, gen tasks of copy file to a tmp file and mv back. + - eg aaa.mp4 -cp-> aaa.mp4.tmp{random_number} -mv-> aaa.mp4 + - use acp lib to copy file, with all features of acp. + - keep all file permission, mtime, atime, attrs, xattrs (using the acp's feature). +- only rewrite regular file, skip dir. +- has state storage, can resume from last time. + - if stoped, cleanup unfinished tmp file. + - when resume will cleanup unfinished tmp file from the last time if any. +- skip file if the file is being used by other process. store them to do later. +- if a file is hardlinked, only rewrite one of them, and relink the other links. +- find all same files by sha256&size from the acp report, and list them as final report. diff --git a/copy.go b/copy.go index bf51d92..0861579 100644 --- a/copy.go +++ b/copy.go @@ -130,7 +130,7 @@ func (c *Copyer) write(ctx context.Context, job *writeJob, ch chan<- *baseJob, c continue } - file, err := os.OpenFile(target, c.createFlag, job.mode) + file, err := os.OpenFile(target, c.createFlag, job.stat.mode) if err = mappingError(err); err != nil { if checkErrorAbort(err) { noSpaceDevices.Add(dev) diff --git a/index.go b/index.go index cc1d338..52f9eee 100644 --- a/index.go +++ b/index.go @@ -62,50 +62,51 @@ func (c *Copyer) walk(ctx context.Context) ([]*baseJob, error) { jobs := make([]*baseJob, 0, 64) appendJob := func(job *baseJob) { - if !job.mode.IsRegular() { - c.reportError(job.path, "", fmt.Errorf("unexpected file mode, not regular file, mode= %s", job.mode)) + if !job.stat.mode.IsRegular() { + c.reportError( + job.path, "", + fmt.Errorf( + "unexpected file mode, not regular file, mode= %s", + job.stat.mode, + ), + ) return } c.submit(&EventUpdateJob{job.report()}) jobs = append(jobs, job) atomic.AddInt64(&cntr.files, 1) - atomic.AddInt64(&cntr.bytes, job.size) + atomic.AddInt64(&cntr.bytes, job.stat.size) } var walk func(src *source, dsts []string) walk = func(src *source, dsts []string) { path := src.src() - stat, err := os.Stat(path) + fi, err := os.Stat(path) if err != nil { c.reportError(path, "", fmt.Errorf("walk get stat, %w", err)) return } - mode := stat.Mode() + mode := fi.Mode() if mode.IsRegular() { targets := make([]string, 0, len(dsts)) for _, d := range dsts { targets = append(targets, src.dst(d)) } - sysStat, err := readSysStat(path, stat) + stat, err := newStat(path, fi) if err != nil { c.reportError(path, "", fmt.Errorf("read sys stat, %w", err)) return } appendJob(&baseJob{ - copyer: c, - src: src, - path: path, - - size: stat.Size(), - mode: stat.Mode(), - modTime: stat.ModTime(), - sys: sysStat, - + copyer: c, + src: src, + path: path, + stat: stat, targets: targets, }) return @@ -144,31 +145,26 @@ func (c *Copyer) walk(ctx context.Context) ([]*baseJob, error) { } for _, j := range c.accurateJobs { - stat, err := os.Stat(j.src) + fi, err := os.Stat(j.src) if err != nil { c.reportError(j.src, "", fmt.Errorf("accurate job get stat, %w", err)) continue } - if !stat.Mode().IsRegular() { + if !fi.Mode().IsRegular() { continue } - sysStat, err := readSysStat(j.src, stat) + stat, err := newStat(j.src, fi) if err != nil { c.reportError(j.src, "", fmt.Errorf("read sys stat, %w", err)) continue } appendJob(&baseJob{ - copyer: c, - src: &source{base: "/", path: lo.Filter(strings.Split(j.src, "/"), func(s string, _ int) bool { return s != "" })}, - path: j.src, - - size: stat.Size(), - mode: stat.Mode(), - modTime: stat.ModTime(), - sys: sysStat, - + copyer: c, + src: &source{base: "/", path: lo.Filter(strings.Split(j.src, "/"), func(s string, _ int) bool { return s != "" })}, + path: j.src, + stat: stat, targets: j.dsts, }) } diff --git a/job.go b/job.go index e5841b9..17634e3 100644 --- a/job.go +++ b/job.go @@ -38,11 +38,7 @@ type baseJob struct { copyer *Copyer src *source path string - - size int64 // length in bytes for regular files; system-dependent for others - mode fs.FileMode // file mode bits - modTime time.Time // modification time - sys *sysStat + stat *stat lock sync.Mutex writeTime time.Time @@ -103,9 +99,9 @@ func (j *baseJob) report() *Job { SuccessTargets: j.successTargets, FailTargets: j.failedTargets, - Size: j.size, - Mode: j.mode, - ModTime: j.modTime, + Size: j.stat.size, + Mode: j.stat.mode, + ModTime: j.stat.modTime, WriteTime: j.writeTime, SHA256: hex.EncodeToString(j.hash), } diff --git a/stat.go b/stat.go new file mode 100644 index 0000000..1f64505 --- /dev/null +++ b/stat.go @@ -0,0 +1,28 @@ +package acp + +import ( + "fmt" + "io/fs" + "time" +) + +type stat struct { + size int64 // length in bytes for regular files; system-dependent for others + mode fs.FileMode // file mode bits + modTime time.Time // modification time + sys *sysStat +} + +func newStat(path string, fi fs.FileInfo) (*stat, error) { + sysStat, err := readSysStat(path, fi) + if err != nil { + return nil, fmt.Errorf("read sys stat failed, %w", err) + } + + return &stat{ + size: fi.Size(), + mode: fi.Mode(), + modTime: fi.ModTime(), + sys: sysStat, + }, nil +} diff --git a/syscall_other.go b/syscall_other.go index 151ba06..4f7dd99 100644 --- a/syscall_other.go +++ b/syscall_other.go @@ -22,7 +22,7 @@ func truncate(file *os.File, size int64) error { return nil } -func writeSysStat(name string, j *baseJob) error { +func writeSysStat(name string, j *stat) error { if err := os.Chmod(name, j.mode); err != nil { return fmt.Errorf("chmod fail, %w", err) } diff --git a/syscall_unix.go b/syscall_unix.go index 1f32881..f4e55e5 100644 --- a/syscall_unix.go +++ b/syscall_unix.go @@ -37,7 +37,7 @@ func readSysStat(path string, stat fs.FileInfo) (*sysStat, error) { return &sysStat{Stat_t: sysstat, xattrs: xattrs}, nil } -func writeSysStat(name string, j *baseJob) error { +func writeSysStat(name string, j *stat) error { if err := writeXattrs(name, j.sys.xattrs); err != nil { return fmt.Errorf("write xattr fail, %w", err) } diff --git a/syscall_windows.go b/syscall_windows.go index 684b43a..b83e808 100644 --- a/syscall_windows.go +++ b/syscall_windows.go @@ -22,7 +22,7 @@ func truncate(file *os.File, size int64) error { return nil } -func writeSysStat(name string, j *baseJob) error { +func writeSysStat(name string, j *stat) error { if err := os.Chmod(name, j.mode); err != nil { return fmt.Errorf("chmod fail, %w", err) }