mirror of
https://github.com/samuelncui/yatm.git
synced 2025-12-23 06:15:22 +00:00
534 lines
13 KiB
Go
534 lines
13 KiB
Go
package library
|
|
|
|
import (
|
|
"context"
|
|
"encoding/binary"
|
|
"errors"
|
|
"fmt"
|
|
"io/fs"
|
|
"path"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
mapset "github.com/deckarep/golang-set/v2"
|
|
"github.com/samber/lo"
|
|
"github.com/sirupsen/logrus"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
var (
|
|
ModelFile = new(File)
|
|
SignatureV1Header = []byte{0x01}
|
|
|
|
ErrFileNotFound = fmt.Errorf("get file: file not found")
|
|
ErrMkdirNonDirFileExists = fmt.Errorf("mkdir: non dir exists")
|
|
ErrMkdirDirExists = fmt.Errorf("mkdir: dir exists")
|
|
ErrNewFileFileExists = fmt.Errorf("new file: file exists")
|
|
|
|
Root = &File{ID: 0}
|
|
)
|
|
|
|
type File struct {
|
|
ID int64 `gorm:"primaryKey;autoIncrement" json:"id,omitempty"`
|
|
ParentID int64 `gorm:"index:idx_parent_name,unique" json:"parent_id,omitempty"`
|
|
|
|
Name string `gorm:"type:varchar(256);index:idx_parent_name,unique" json:"name,omitempty"`
|
|
Mode uint32 `json:"mode,omitempty"`
|
|
ModTime time.Time `json:"mod_time,omitempty"`
|
|
Hash []byte `gorm:"type:varbinary(32)" json:"hash,omitempty"` // sha256
|
|
Size int64 `json:"size,omitempty"`
|
|
|
|
Signature []byte `gorm:"type:varbinary(256);index:idx_signature" json:"signature,omitempty"` // sha256 + size
|
|
}
|
|
|
|
func (l *Library) MkdirAll(ctx context.Context, parentID int64, name string, perm fs.FileMode) (*File, error) {
|
|
return l.mkdirAll(ctx, l.db.WithContext(ctx), parentID, name, perm)
|
|
}
|
|
|
|
func (l *Library) mkdirAll(ctx context.Context, tx *gorm.DB, parentID int64, name string, perm fs.FileMode) (*File, error) {
|
|
name = path.Clean(strings.TrimSpace(name))
|
|
if strings.ContainsAny(name, "\\") || name == "" {
|
|
return nil, fmt.Errorf("unexpected mkdir path, '%s'", name)
|
|
}
|
|
|
|
current := Root
|
|
if parentID != 0 {
|
|
f, err := l.getFile(ctx, tx, parentID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
current = f
|
|
}
|
|
|
|
parts := strings.Split(name, "/")
|
|
for _, part := range parts {
|
|
part = strings.TrimSpace(part)
|
|
if part == "" {
|
|
continue
|
|
}
|
|
|
|
next, err := l.mkdir(ctx, tx, current.ID, part, perm)
|
|
if err != nil && !errors.Is(err, ErrMkdirDirExists) {
|
|
return nil, fmt.Errorf("mkdir fail, %w", err)
|
|
}
|
|
|
|
current = next
|
|
}
|
|
|
|
return current, nil
|
|
}
|
|
|
|
func (l *Library) mkdir(ctx context.Context, tx *gorm.DB, parentID int64, name string, perm fs.FileMode) (*File, error) {
|
|
perm = fs.ModePerm & perm
|
|
|
|
origin := new(File)
|
|
if r := tx.Where("parent_id = ? AND name = ?", parentID, name).Find(origin); r.Error != nil {
|
|
return nil, fmt.Errorf("mkdir: find origin fail, err= %w", r.Error)
|
|
}
|
|
if origin.ID != 0 {
|
|
if fs.FileMode(origin.Mode).IsDir() {
|
|
return origin, ErrMkdirDirExists
|
|
}
|
|
return nil, ErrMkdirNonDirFileExists
|
|
}
|
|
|
|
dir := &File{
|
|
ParentID: parentID,
|
|
Name: name,
|
|
Mode: uint32(fs.ModeDir | perm),
|
|
ModTime: time.Now(),
|
|
}
|
|
if r := tx.Create(dir); r.Error != nil {
|
|
return nil, fmt.Errorf("create fail, err= %w", r.Error)
|
|
}
|
|
|
|
return dir, nil
|
|
}
|
|
|
|
func (l *Library) GetFile(ctx context.Context, id int64) (*File, error) {
|
|
return l.getFile(ctx, l.db.WithContext(ctx), id)
|
|
}
|
|
|
|
func (l *Library) getFile(ctx context.Context, tx *gorm.DB, id int64) (*File, error) {
|
|
files, err := l.mGetFile(ctx, tx, id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
f, ok := files[id]
|
|
if !ok || f == nil {
|
|
return nil, ErrFileNotFound
|
|
}
|
|
|
|
return f, nil
|
|
}
|
|
|
|
func (l *Library) SaveFile(ctx context.Context, file *File) error {
|
|
return l.db.WithContext(ctx).Save(file).Error
|
|
}
|
|
|
|
func (l *Library) MoveFile(ctx context.Context, file *File) error {
|
|
return l.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
|
return l.moveFile(ctx, tx, file)
|
|
})
|
|
}
|
|
|
|
func (l *Library) moveFile(ctx context.Context, tx *gorm.DB, file *File) error {
|
|
origin, err := l.getByName(ctx, tx, file.ParentID, file.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if origin == nil {
|
|
return tx.Save(file).Error
|
|
}
|
|
|
|
if !fs.FileMode(origin.Mode).IsDir() {
|
|
return fmt.Errorf("same name file exists, name= '%s'", file.Name)
|
|
}
|
|
if !fs.FileMode(file.Mode).IsDir() {
|
|
return fmt.Errorf("same name file is a dir, name= '%s", file.Name)
|
|
}
|
|
|
|
children, err := l.list(ctx, tx, file.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, child := range children {
|
|
child.ParentID = origin.ID
|
|
if err := l.moveFile(ctx, tx, child); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if file.ModTime.After(origin.ModTime) {
|
|
origin.ModTime = file.ModTime
|
|
if err := tx.Save(origin).Error; err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := tx.Delete(file).Error; err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (l *Library) Delete(ctx context.Context, ids []int64) error {
|
|
files, err := l.MGetFile(ctx, ids...)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
moveToTrash := make([]*File, 0, len(files))
|
|
outter:
|
|
for _, file := range files {
|
|
if file.ID == TrashFileID {
|
|
continue
|
|
}
|
|
parents, err := l.ListParents(ctx, file.ID)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
if len(parents) == 0 {
|
|
moveToTrash = append(moveToTrash, file)
|
|
continue
|
|
}
|
|
if parents[0].ID != TrashFileID {
|
|
moveToTrash = append(moveToTrash, file)
|
|
continue
|
|
}
|
|
if !fs.FileMode(file.Mode).IsDir() {
|
|
continue
|
|
}
|
|
|
|
needDelete := make([]*File, 0, 8)
|
|
current := []*File{file}
|
|
for len(current) > 0 {
|
|
next := make([]*File, 0, 8)
|
|
for _, file := range current {
|
|
children, err := l.List(ctx, file.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, child := range children {
|
|
if !fs.FileMode(child.Mode).IsDir() {
|
|
continue outter
|
|
}
|
|
}
|
|
next = append(next, children...)
|
|
}
|
|
|
|
needDelete = append(needDelete, current...)
|
|
current = next
|
|
}
|
|
|
|
if err := l.db.WithContext(ctx).Delete(needDelete).Error; err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if len(moveToTrash) == 0 {
|
|
return nil
|
|
}
|
|
|
|
trash, err := l.newTrash(ctx, l.db.WithContext(ctx))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, file := range moveToTrash {
|
|
file.ParentID = trash.ID
|
|
if err := l.MoveFile(ctx, file); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
const (
|
|
TrashFileID = -1
|
|
)
|
|
|
|
func (l *Library) newTrash(ctx context.Context, tx *gorm.DB) (*File, error) {
|
|
now := time.Now()
|
|
trash := &File{
|
|
ID: TrashFileID,
|
|
Name: ".Trash",
|
|
Mode: uint32(fs.ModePerm | fs.ModeDir),
|
|
ModTime: now,
|
|
}
|
|
if err := tx.Save(trash).Error; err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return l.mkdir(ctx, tx, trash.ID, now.Format(time.RFC3339), fs.ModePerm)
|
|
}
|
|
|
|
func (l *Library) MGetFile(ctx context.Context, ids ...int64) (map[int64]*File, error) {
|
|
return l.mGetFile(ctx, l.db.WithContext(ctx), ids...)
|
|
}
|
|
|
|
func (l *Library) mGetFile(ctx context.Context, tx *gorm.DB, ids ...int64) (map[int64]*File, error) {
|
|
if len(ids) == 0 {
|
|
return map[int64]*File{}, nil
|
|
}
|
|
|
|
files := make([]*File, 0, len(ids))
|
|
if r := tx.Where("id IN (?)", ids).Find(&files); r.Error != nil {
|
|
return nil, fmt.Errorf("find files fail, %w", r.Error)
|
|
}
|
|
|
|
results := make(map[int64]*File, len(files))
|
|
for _, f := range files {
|
|
results[f.ID] = f
|
|
}
|
|
|
|
return results, nil
|
|
}
|
|
|
|
func (l *Library) GetByPath(ctx context.Context, parentID int64, name string) (*File, error) {
|
|
name = path.Clean(strings.TrimSpace(name))
|
|
if strings.ContainsAny(name, "\\") || name == "" {
|
|
return nil, fmt.Errorf("unexpected mkdir path, '%s'", name)
|
|
}
|
|
|
|
current := Root
|
|
if parentID != 0 {
|
|
f, err := l.GetFile(ctx, parentID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
current = f
|
|
}
|
|
|
|
parts := strings.Split(name, "/")
|
|
for _, part := range parts {
|
|
part = strings.TrimSpace(part)
|
|
if part == "" {
|
|
continue
|
|
}
|
|
|
|
next, err := l.GetByName(ctx, current.ID, part)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("get by path fail, %w", err)
|
|
}
|
|
if next == nil {
|
|
return nil, nil
|
|
}
|
|
|
|
current = next
|
|
}
|
|
|
|
return current, nil
|
|
}
|
|
|
|
func (l *Library) GetByName(ctx context.Context, parentID int64, name string) (*File, error) {
|
|
return l.getByName(ctx, l.db.WithContext(ctx), parentID, name)
|
|
}
|
|
|
|
func (l *Library) getByName(ctx context.Context, tx *gorm.DB, parentID int64, name string) (*File, error) {
|
|
file := new(File)
|
|
if r := tx.Where("parent_id = ? AND name = ?", parentID, name).Find(file); r.Error != nil {
|
|
return nil, fmt.Errorf("find files fail, %w", r.Error)
|
|
}
|
|
if file.ID == 0 {
|
|
return nil, nil
|
|
}
|
|
return file, nil
|
|
}
|
|
|
|
func (l *Library) List(ctx context.Context, parentID int64) ([]*File, error) {
|
|
return l.list(ctx, l.db.WithContext(ctx), parentID)
|
|
}
|
|
|
|
func (l *Library) ListWithSize(ctx context.Context, parentID int64) ([]*File, error) {
|
|
all, err := l.listAll(ctx, l.db.WithContext(ctx), parentID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
mapping := lo.GroupBy(all, func(file *File) int64 { return file.ParentID })
|
|
var fetchSize func(file *File)
|
|
fetchSize = func(file *File) {
|
|
for _, child := range mapping[file.ID] {
|
|
fetchSize(child)
|
|
file.Size += child.Size
|
|
}
|
|
}
|
|
|
|
files := mapping[parentID]
|
|
for _, f := range files {
|
|
fetchSize(f)
|
|
}
|
|
|
|
sort.Slice(files, func(i, j int) bool { return files[i].Name < files[j].Name })
|
|
return files, nil
|
|
}
|
|
|
|
func (l *Library) list(ctx context.Context, tx *gorm.DB, parentID int64) ([]*File, error) {
|
|
files := make([]*File, 0, 4)
|
|
if r := tx.Where("parent_id = ?", parentID).Order("name").Find(&files); r.Error != nil {
|
|
return nil, fmt.Errorf("find files fail, %w", r.Error)
|
|
}
|
|
return files, nil
|
|
}
|
|
|
|
func (l *Library) listAll(ctx context.Context, tx *gorm.DB, parentIDs ...int64) ([]*File, error) {
|
|
files := make([]*File, 0, 4)
|
|
|
|
current := parentIDs
|
|
for {
|
|
batch := make([]*File, 0, 4)
|
|
if r := tx.Where("parent_id IN (?)", current).Find(&batch); r.Error != nil {
|
|
return nil, fmt.Errorf("find files fail, %w", r.Error)
|
|
}
|
|
|
|
if len(batch) == 0 {
|
|
break
|
|
}
|
|
|
|
files = append(files, batch...)
|
|
next := make([]int64, 0, 4)
|
|
for _, f := range batch {
|
|
if !fs.FileMode(f.Mode).IsDir() {
|
|
continue
|
|
}
|
|
next = append(next, f.ID)
|
|
}
|
|
|
|
if len(next) == 0 {
|
|
break
|
|
}
|
|
current = next
|
|
}
|
|
|
|
return files, nil
|
|
}
|
|
|
|
func (l *Library) ListParents(ctx context.Context, id int64) ([]*File, error) {
|
|
return l.listParnets(ctx, l.db.WithContext(ctx), id)
|
|
}
|
|
|
|
func (l *Library) listParnets(ctx context.Context, tx *gorm.DB, id int64) ([]*File, error) {
|
|
result := make([]*File, 0, 3)
|
|
|
|
currentID := id
|
|
for i := 0; i < 32 && currentID != 0; i++ {
|
|
file, err := l.getFile(ctx, tx, currentID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
result = append(result, file)
|
|
currentID = file.ParentID
|
|
}
|
|
|
|
num := len(result)
|
|
if num <= 1 {
|
|
return result, nil
|
|
}
|
|
for i := 0; i < num/2; i++ {
|
|
result[i], result[num-i-1] = result[num-i-1], result[i]
|
|
}
|
|
|
|
return result, nil
|
|
}
|
|
|
|
func (l *Library) Search(ctx context.Context, name string) ([]*File, error) {
|
|
files := make([]*File, 0, 4)
|
|
if r := l.db.WithContext(ctx).Where("name LIKE ?", fmt.Sprintf("%"+name+"%")).Order("name").Limit(100).Find(&files); r.Error != nil {
|
|
return nil, fmt.Errorf("find files fail, %w", r.Error)
|
|
}
|
|
return files, nil
|
|
}
|
|
|
|
func (l *Library) TrimFiles(ctx context.Context) error {
|
|
for {
|
|
positions := make([]*Position, 0, batchSize)
|
|
if r := l.db.WithContext(ctx).Where("file_id = ?", 0).Limit(batchSize).Find(&positions); r.Error != nil {
|
|
return fmt.Errorf("list non file position fail, err= %w", r.Error)
|
|
}
|
|
if len(positions) == 0 {
|
|
return nil
|
|
}
|
|
|
|
signatures := make([][]byte, 0, len(positions))
|
|
sign2positions := make(map[string]*Position, len(positions))
|
|
for _, posi := range positions {
|
|
size := make([]byte, 8)
|
|
binary.BigEndian.PutUint64(size, uint64(posi.Size))
|
|
|
|
sign := make([]byte, 0, 64)
|
|
sign = append(sign, SignatureV1Header...)
|
|
sign = append(sign, posi.Hash...)
|
|
sign = append(sign, size...)
|
|
|
|
signatures = append(signatures, sign)
|
|
sign2positions[string(sign)] = posi
|
|
}
|
|
|
|
matched := make([]*File, 0, 4)
|
|
if r := l.db.WithContext(ctx).Where("signature IN (?)", signatures).Find(&matched); r.Error != nil {
|
|
return fmt.Errorf("get matched file fail, err= %w", r.Error)
|
|
}
|
|
|
|
for _, file := range matched {
|
|
posi, has := sign2positions[string(file.Signature)]
|
|
if !has {
|
|
continue
|
|
}
|
|
|
|
posi.FileID = file.ID
|
|
l.db.WithContext(ctx).Save(posi)
|
|
|
|
delete(sign2positions, string(file.Signature))
|
|
}
|
|
|
|
tapeIDs := mapset.NewThreadUnsafeSet[int64]()
|
|
for _, posi := range sign2positions {
|
|
tapeIDs.Add(posi.TapeID)
|
|
}
|
|
|
|
tapes, err := l.MGetTape(ctx, tapeIDs.ToSlice()...)
|
|
if err != nil {
|
|
return fmt.Errorf("mget tape, ids= %v, %w", tapeIDs.ToSlice(), err)
|
|
}
|
|
|
|
for sign, posi := range sign2positions {
|
|
tape := tapes[posi.TapeID]
|
|
if tape == nil {
|
|
logrus.WithContext(ctx).Warnf("trim file, tape not found, tape_id= %d", posi.TapeID)
|
|
continue
|
|
}
|
|
|
|
dirname, filename := path.Split(fmt.Sprintf("Unforged/%s/%s", tape.Barcode, posi.Path))
|
|
dir, err := l.MkdirAll(ctx, Root.ID, dirname, 0x777)
|
|
if err != nil {
|
|
return fmt.Errorf("mkdir, %w", err)
|
|
}
|
|
|
|
origin := new(File)
|
|
if r := l.db.WithContext(ctx).Where("parent_id = ? AND name = ?", dir.ID, filename).Find(origin); r.Error != nil {
|
|
return fmt.Errorf("new file: find origin fail, err= %w", r.Error)
|
|
}
|
|
if origin.ID != 0 {
|
|
return ErrNewFileFileExists
|
|
}
|
|
|
|
file := &File{
|
|
ParentID: dir.ID,
|
|
Name: filename,
|
|
Mode: posi.Mode,
|
|
ModTime: time.Now(),
|
|
Hash: posi.Hash,
|
|
Size: posi.Size,
|
|
Signature: []byte(sign),
|
|
}
|
|
if r := l.db.WithContext(ctx).Create(file); r.Error != nil {
|
|
return fmt.Errorf("new file: create fail, err= %w", r.Error)
|
|
}
|
|
}
|
|
}
|
|
}
|