mirror of
https://github.com/samuelncui/acp.git
synced 2025-12-23 13:15:16 +00:00
feat: don't use mmap from linear device
This commit is contained in:
17
copy.go
17
copy.go
@@ -5,6 +5,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"hash"
|
"hash"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -12,7 +13,6 @@ import (
|
|||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/abc950309/acp/mmap"
|
|
||||||
mapset "github.com/deckarep/golang-set/v2"
|
mapset "github.com/deckarep/golang-set/v2"
|
||||||
"github.com/hashicorp/go-multierror"
|
"github.com/hashicorp/go-multierror"
|
||||||
sha256 "github.com/minio/sha256-simd"
|
sha256 "github.com/minio/sha256-simd"
|
||||||
@@ -203,21 +203,18 @@ func (c *Copyer) write(ctx context.Context, job *writeJob, ch chan<- *baseJob, c
|
|||||||
readErr = c.streamCopy(ctx, chans, job.src, &cntr.bytes)
|
readErr = c.streamCopy(ctx, chans, job.src, &cntr.bytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Copyer) streamCopy(ctx context.Context, dsts []chan []byte, src *mmap.ReaderAt, bytes *int64) error {
|
func (c *Copyer) streamCopy(ctx context.Context, dsts []chan []byte, src io.ReadCloser, bytes *int64) error {
|
||||||
if src.Len() == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
for idx := int64(0); ; idx += batchSize {
|
for idx := int64(0); ; idx += batchSize {
|
||||||
buf, err := src.Slice(idx, batchSize)
|
buf := make([]byte, batchSize)
|
||||||
|
|
||||||
|
n, err := io.ReadFull(src, buf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("slice mmap fail, %w", err)
|
return fmt.Errorf("slice mmap fail, %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
copyed := make([]byte, len(buf))
|
buf = buf[:n]
|
||||||
copy(copyed, buf)
|
|
||||||
for _, ch := range dsts {
|
for _, ch := range dsts {
|
||||||
ch <- copyed
|
ch <- buf
|
||||||
}
|
}
|
||||||
|
|
||||||
nr := len(buf)
|
nr := len(buf)
|
||||||
|
|||||||
7
job.go
7
job.go
@@ -2,11 +2,10 @@ package acp
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/abc950309/acp/mmap"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type jobStatus uint8
|
type jobStatus uint8
|
||||||
@@ -106,11 +105,11 @@ func (j *baseJob) report() *Job {
|
|||||||
|
|
||||||
type writeJob struct {
|
type writeJob struct {
|
||||||
*baseJob
|
*baseJob
|
||||||
src *mmap.ReaderAt
|
src io.ReadCloser
|
||||||
ch chan struct{}
|
ch chan struct{}
|
||||||
}
|
}
|
||||||
|
|
||||||
func newWriteJob(job *baseJob, src *mmap.ReaderAt, needWait bool) *writeJob {
|
func newWriteJob(job *baseJob, src io.ReadCloser, needWait bool) *writeJob {
|
||||||
j := &writeJob{
|
j := &writeJob{
|
||||||
baseJob: job,
|
baseJob: job,
|
||||||
src: src,
|
src: src,
|
||||||
|
|||||||
16
mmap/mmap_reader.go
Normal file
16
mmap/mmap_reader.go
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
package mmap
|
||||||
|
|
||||||
|
type Reader struct {
|
||||||
|
*ReaderAt
|
||||||
|
index int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewReader(readerAt *ReaderAt) *Reader {
|
||||||
|
return &Reader{ReaderAt: readerAt}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reader) Read(buf []byte) (n int, err error) {
|
||||||
|
n, err = r.ReadAt(buf, r.index)
|
||||||
|
r.index += int64(n)
|
||||||
|
return
|
||||||
|
}
|
||||||
34
prepare.go
34
prepare.go
@@ -3,6 +3,8 @@ package acp
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/abc950309/acp/mmap"
|
"github.com/abc950309/acp/mmap"
|
||||||
@@ -39,10 +41,36 @@ func (c *Copyer) prepare(ctx context.Context, indexed <-chan *baseJob) <-chan *w
|
|||||||
|
|
||||||
job.setStatus(jobStatusPreparing)
|
job.setStatus(jobStatusPreparing)
|
||||||
|
|
||||||
file, err := mmap.Open(job.path)
|
file, err := func(path string) (io.ReadCloser, error) {
|
||||||
|
if c.fromDevice.linear {
|
||||||
|
file, err := os.Open(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.reportError(job.path, "", fmt.Errorf("open src file fail, %w", err))
|
return nil, fmt.Errorf("open src file fail, %w", err)
|
||||||
return
|
}
|
||||||
|
|
||||||
|
fileInfo, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("get src file stat fail, %w", err)
|
||||||
|
}
|
||||||
|
if fileInfo.Size() == 0 {
|
||||||
|
return nil, fmt.Errorf("get src file, size is zero")
|
||||||
|
}
|
||||||
|
|
||||||
|
return file, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
readerAt, err := mmap.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("open src file by mmap fail, %w", err)
|
||||||
|
}
|
||||||
|
if readerAt.Len() == 0 {
|
||||||
|
return nil, fmt.Errorf("get src file by mmap, size is zero")
|
||||||
|
}
|
||||||
|
|
||||||
|
return mmap.NewReader(readerAt), nil
|
||||||
|
}(job.path)
|
||||||
|
if err != nil {
|
||||||
|
c.reportError(job.path, "", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
wj := newWriteJob(job, file, c.fromDevice.linear)
|
wj := newWriteJob(job, file, c.fromDevice.linear)
|
||||||
|
|||||||
Reference in New Issue
Block a user