mirrors/stfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: Use public API structs in persisters

Browse Source
This commit is contained in:
Felicitas Pojtinger
2022-01-02 00:03:01 +01:00
parent cfbb936954
commit 172a6a3f48
20 changed files with 158 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
pkg/config/config.go
View File

@@ -5,8 +5,7 @@ import (
"io"
"io/fs"
"os"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
"time"
)
type DriveReaderConfig struct {
@@ -35,16 +34,40 @@ type BackendConfig struct {
CloseDrive func() error
}
type Header struct {
Record int64
Lastknownrecord int64
Block int64
Lastknownblock int64
Deleted int64
Typeflag int64
Name string
Linkname string
Size int64
Mode int64
UID int64
Gid int64
Uname string
Gname string
Modtime time.Time
Accesstime time.Time
Changetime time.Time
Devmajor int64
Devminor int64
Paxrecords string
Format int64
}
type MetadataPersister interface {
UpsertHeader(ctx context.Context, dbhdr *models.Header) error
UpdateHeaderMetadata(ctx context.Context, dbhdr *models.Header) error
UpsertHeader(ctx context.Context, dbhdr *Header) error
UpdateHeaderMetadata(ctx context.Context, dbhdr *Header) error
MoveHeader(ctx context.Context, oldName string, newName string, lastknownrecord, lastknownblock int64) error
GetHeaders(ctx context.Context) (models.HeaderSlice, error)
GetHeader(ctx context.Context, name string) (*models.Header, error)
GetHeaderChildren(ctx context.Context, name string) (models.HeaderSlice, error)
GetHeaders(ctx context.Context) ([]*Header, error)
GetHeader(ctx context.Context, name string) (*Header, error)
GetHeaderChildren(ctx context.Context, name string) ([]*Header, error)
GetRootPath(ctx context.Context) (string, error)
GetHeaderDirectChildren(ctx context.Context, name string, limit int) (models.HeaderSlice, error)
DeleteHeader(ctx context.Context, name string, lastknownrecord, lastknownblock int64) (*models.Header, error)
GetHeaderDirectChildren(ctx context.Context, name string, limit int) ([]*Header, error)
DeleteHeader(ctx context.Context, name string, lastknownrecord, lastknownblock int64) (*Header, error)
GetLastIndexedRecordAndBlock(ctx context.Context, recordSize int) (int64, int64, error)
PurgeAllHeaders(ctx context.Context) error
}

4
pkg/config/events.go
View File

@@ -1,9 +1,7 @@
package config
import models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
type HeaderEvent struct {
Type string
Indexed bool
Header *models.Header
Header *Header
}

5
pkg/fs/filesystem.go
View File

@@ -12,7 +12,6 @@ import (
"sync"
"time"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
ifs "github.com/pojntfx/stfs/internal/fs"
"github.com/pojntfx/stfs/pkg/cache"
"github.com/pojntfx/stfs/pkg/config"
@@ -34,7 +33,7 @@ type STFS struct {
ioLock sync.Mutex
onHeader func(hdr *models.Header)
onHeader func(hdr *config.Header)
log logging.StructuredLogger
}
@@ -48,7 +47,7 @@ func NewSTFS(
getFileBuffer func() (cache.WriteCache, func() error, error),
ignorePermissionFlags bool,
onHeader func(hdr *models.Header),
onHeader func(hdr *config.Header),
log logging.StructuredLogger,
) *STFS {
return &STFS{

5
pkg/inventory/find.go
View File

@@ -6,7 +6,6 @@ import (
"regexp"
"github.com/pojntfx/stfs/internal/converters"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
"github.com/pojntfx/stfs/pkg/config"
)
@@ -15,7 +14,7 @@ func Find(
expression string,
onHeader func(hdr *models.Header),
onHeader func(hdr *config.Header),
) ([]*tar.Header, error) {
dbHdrs, err := metadata.Metadata.GetHeaders(context.Background())
if err != nil {
@@ -25,7 +24,7 @@ func Find(
headers := []*tar.Header{}
for _, dbhdr := range dbHdrs {
if regexp.MustCompile(expression).Match([]byte(dbhdr.Name)) {
hdr, err := converters.DBHeaderToTarHeader(dbhdr)
hdr, err := converters.DBHeaderToTarHeader(converters.ConfigHeaderToDBHeader(dbhdr))
if err != nil {
return []*tar.Header{}, err
}

5
pkg/inventory/list.go
View File

@@ -5,7 +5,6 @@ import (
"context"
"github.com/pojntfx/stfs/internal/converters"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
"github.com/pojntfx/stfs/pkg/config"
)
@@ -15,7 +14,7 @@ func List(
name string,
limit int,
onHeader func(hdr *models.Header),
onHeader func(hdr *config.Header),
) ([]*tar.Header, error) {
dbHdrs, err := metadata.Metadata.GetHeaderDirectChildren(context.Background(), name, limit)
if err != nil {
@@ -24,7 +23,7 @@ func List(
headers := []*tar.Header{}
for _, dbhdr := range dbHdrs {
hdr, err := converters.DBHeaderToTarHeader(dbhdr)
hdr, err := converters.DBHeaderToTarHeader(converters.ConfigHeaderToDBHeader(dbhdr))
if err != nil {
return []*tar.Header{}, err
}

5
pkg/inventory/stat.go
View File

@@ -8,7 +8,6 @@ import (
"strings"
"github.com/pojntfx/stfs/internal/converters"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
"github.com/pojntfx/stfs/pkg/config"
)
@@ -17,7 +16,7 @@ func Stat(
name string,
onHeader func(hdr *models.Header),
onHeader func(hdr *config.Header),
) (*tar.Header, error) {
name = filepath.ToSlash(name)
@@ -33,7 +32,7 @@ func Stat(
}
}
hdr, err := converters.DBHeaderToTarHeader(dbhdr)
hdr, err := converters.DBHeaderToTarHeader(converters.ConfigHeaderToDBHeader(dbhdr))
if err != nil {
return nil, err
}

5
pkg/operations/archive.go
View File

@@ -10,7 +10,6 @@ import (
"github.com/pojntfx/stfs/internal/compression"
"github.com/pojntfx/stfs/internal/converters"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
"github.com/pojntfx/stfs/internal/encryption"
"github.com/pojntfx/stfs/internal/ioext"
"github.com/pojntfx/stfs/internal/mtio"
@@ -163,7 +162,7 @@ func (o *Operations) Archive(
o.onHeader(&config.HeaderEvent{
Type: config.HeaderEventTypeArchive,
Indexed: false,
Header: dbhdr,
Header: converters.DBHeaderToConfigHeader(dbhdr),
})
}
@@ -294,7 +293,7 @@ func (o *Operations) Archive(
return nil // We sign above, no need to verify
},
func(hdr *models.Header) {
func(hdr *config.Header) {
o.onHeader(&config.HeaderEvent{
Type: config.HeaderEventTypeArchive,
Indexed: true,

9
pkg/operations/delete.go
View File

@@ -6,7 +6,6 @@ import (
"path/filepath"
"github.com/pojntfx/stfs/internal/converters"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
"github.com/pojntfx/stfs/internal/encryption"
"github.com/pojntfx/stfs/internal/records"
"github.com/pojntfx/stfs/internal/signature"
@@ -37,7 +36,7 @@ func (o *Operations) Delete(name string) error {
return err
}
headersToDelete := []*models.Header{}
headersToDelete := []*config.Header{}
dbhdr, err := o.metadata.Metadata.GetHeader(context.Background(), name)
if err != nil {
return err
@@ -57,7 +56,7 @@ func (o *Operations) Delete(name string) error {
// Append deletion hdrs to the tape or tar file
hdrs := []tar.Header{}
for _, dbhdr := range headersToDelete {
hdr, err := converters.DBHeaderToTarHeader(dbhdr)
hdr, err := converters.DBHeaderToTarHeader(converters.ConfigHeaderToDBHeader(dbhdr))
if err != nil {
return err
}
@@ -77,7 +76,7 @@ func (o *Operations) Delete(name string) error {
o.onHeader(&config.HeaderEvent{
Type: config.HeaderEventTypeDelete,
Indexed: false,
Header: dbhdr,
Header: converters.DBHeaderToConfigHeader(dbhdr),
})
}
@@ -142,7 +141,7 @@ func (o *Operations) Delete(name string) error {
return nil // We sign above, no need to verify
},
func(hdr *models.Header) {
func(hdr *config.Header) {
o.onHeader(&config.HeaderEvent{
Type: config.HeaderEventTypeDelete,
Indexed: true,

9
pkg/operations/move.go
View File

@@ -8,7 +8,6 @@ import (
"strings"
"github.com/pojntfx/stfs/internal/converters"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
"github.com/pojntfx/stfs/internal/encryption"
"github.com/pojntfx/stfs/internal/records"
"github.com/pojntfx/stfs/internal/signature"
@@ -44,7 +43,7 @@ func (o *Operations) Move(from string, to string) error {
return err
}
headersToMove := []*models.Header{}
headersToMove := []*config.Header{}
dbhdr, err := o.metadata.Metadata.GetHeader(context.Background(), from)
if err != nil {
return err
@@ -74,7 +73,7 @@ func (o *Operations) Move(from string, to string) error {
// Append move headers to the tape or tar file
hdrs := []tar.Header{}
for _, dbhdr := range headersToMove {
hdr, err := converters.DBHeaderToTarHeader(dbhdr)
hdr, err := converters.DBHeaderToTarHeader(converters.ConfigHeaderToDBHeader(dbhdr))
if err != nil {
return err
}
@@ -96,7 +95,7 @@ func (o *Operations) Move(from string, to string) error {
o.onHeader(&config.HeaderEvent{
Type: config.HeaderEventTypeMove,
Indexed: false,
Header: dbhdr,
Header: converters.DBHeaderToConfigHeader(dbhdr),
})
}
@@ -161,7 +160,7 @@ func (o *Operations) Move(from string, to string) error {
return nil // We sign above, no need to verify
},
func(hdr *models.Header) {
func(hdr *config.Header) {
o.onHeader(&config.HeaderEvent{
Type: config.HeaderEventTypeMove,
Indexed: true,

3
pkg/operations/restore.go
View File

@@ -10,7 +10,6 @@ import (
"path/filepath"
"strings"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
"github.com/pojntfx/stfs/pkg/config"
"github.com/pojntfx/stfs/pkg/recovery"
)
@@ -28,7 +27,7 @@ func (o *Operations) Restore(
o.diskOperationLock.Lock()
defer o.diskOperationLock.Unlock()
headersToRestore := []*models.Header{}
headersToRestore := []*config.Header{}
src := strings.TrimSuffix(from, "/")
dbhdr, err := o.metadata.Metadata.GetHeader(context.Background(), src)
if err != nil {

7
pkg/operations/update.go
View File

@@ -9,7 +9,6 @@ import (
"github.com/pojntfx/stfs/internal/compression"
"github.com/pojntfx/stfs/internal/converters"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
"github.com/pojntfx/stfs/internal/encryption"
"github.com/pojntfx/stfs/internal/ioext"
"github.com/pojntfx/stfs/internal/mtio"
@@ -163,7 +162,7 @@ func (o *Operations) Update(
o.onHeader(&config.HeaderEvent{
Type: config.HeaderEventTypeUpdate,
Indexed: false,
Header: dbhdr,
Header: converters.DBHeaderToConfigHeader(dbhdr),
})
}
@@ -253,7 +252,7 @@ func (o *Operations) Update(
o.onHeader(&config.HeaderEvent{
Type: config.HeaderEventTypeUpdate,
Indexed: false,
Header: dbhdr,
Header: converters.DBHeaderToConfigHeader(dbhdr),
})
}
@@ -322,7 +321,7 @@ func (o *Operations) Update(
return nil // We sign above, no need to verify
},
func(hdr *models.Header) {
func(hdr *config.Header) {
o.onHeader(&config.HeaderEvent{
Type: config.HeaderEventTypeUpdate,
Indexed: true,

51
pkg/persisters/metadata.go
View File

@@ -11,6 +11,7 @@ import (
"path/filepath"
"strings"
"github.com/pojntfx/stfs/internal/converters"
"github.com/pojntfx/stfs/internal/db/sqlite/migrations/metadata"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
"github.com/pojntfx/stfs/internal/pathext"
@@ -43,8 +44,8 @@ func NewMetadataPersister(dbPath string) *MetadataPersister {
}
}
func (p *MetadataPersister) UpsertHeader(ctx context.Context, dbhdr *models.Header) error {
hdr := *dbhdr
func (p *MetadataPersister) UpsertHeader(ctx context.Context, dbhdr *config.Header) error {
hdr := converters.ConfigHeaderToDBHeader(dbhdr)
if _, err := models.FindHeader(ctx, p.DB, hdr.Name, models.HeaderColumns.Name); err != nil {
if err == sql.ErrNoRows {
@@ -70,11 +71,13 @@ func (p *MetadataPersister) UpsertHeader(ctx context.Context, dbhdr *models.Head
return nil
}
func (p *MetadataPersister) UpdateHeaderMetadata(ctx context.Context, dbhdr *models.Header) error {
if _, err := dbhdr.Update(ctx, p.DB, boil.Infer()); err != nil {
func (p *MetadataPersister) UpdateHeaderMetadata(ctx context.Context, dbhdr *config.Header) error {
idbhdr := converters.ConfigHeaderToDBHeader(dbhdr)
if _, err := idbhdr.Update(ctx, p.DB, boil.Infer()); err != nil {
if err == sql.ErrNoRows {
hdr := *dbhdr
hdr.Name = p.withRelativeRoot(ctx, dbhdr.Name)
hdr := *idbhdr
hdr.Name = p.withRelativeRoot(ctx, idbhdr.Name)
if _, err := hdr.Update(ctx, p.DB, boil.Infer()); err != nil {
return err
@@ -134,13 +137,23 @@ func (p *MetadataPersister) MoveHeader(ctx context.Context, oldName string, newN
return nil
}
func (p *MetadataPersister) GetHeaders(ctx context.Context) (models.HeaderSlice, error) {
return models.Headers(
func (p *MetadataPersister) GetHeaders(ctx context.Context) ([]*config.Header, error) {
dbhdrs, err := models.Headers(
qm.Where(models.HeaderColumns.Deleted+" != 1"),
).All(ctx, p.DB)
if err != nil {
return []*config.Header{}, err
}
hdrs := []*config.Header{}
for _, dbhdr := range dbhdrs {
hdrs = append(hdrs, converters.DBHeaderToConfigHeader(dbhdr))
}
return hdrs, nil
}
func (p *MetadataPersister) GetHeader(ctx context.Context, name string) (*models.Header, error) {
func (p *MetadataPersister) GetHeader(ctx context.Context, name string) (*config.Header, error) {
hdr, err := models.Headers(
qm.Where(models.HeaderColumns.Name+" = ?", name),
qm.Where(models.HeaderColumns.Deleted+" != 1"),
@@ -159,10 +172,10 @@ func (p *MetadataPersister) GetHeader(ctx context.Context, name string) (*models
}
}
return hdr, nil
return converters.DBHeaderToConfigHeader(hdr), nil
}
func (p *MetadataPersister) GetHeaderChildren(ctx context.Context, name string) (models.HeaderSlice, error) {
func (p *MetadataPersister) GetHeaderChildren(ctx context.Context, name string) ([]*config.Header, error) {
headers, err := models.Headers(
qm.Where(models.HeaderColumns.Name+" like ?", strings.TrimSuffix(name, "/")+"/%"), // Prevent double trailing slashes
qm.Where(models.HeaderColumns.Deleted+" != 1"),
@@ -181,11 +194,11 @@ func (p *MetadataPersister) GetHeaderChildren(ctx context.Context, name string)
}
}
outhdrs := models.HeaderSlice{}
outhdrs := []*config.Header{}
for _, hdr := range headers {
prefix := strings.TrimSuffix(hdr.Name, "/")
if name != prefix && name != prefix+"/" {
outhdrs = append(outhdrs, hdr)
outhdrs = append(outhdrs, converters.DBHeaderToConfigHeader(hdr))
}
}
@@ -214,10 +227,10 @@ func (p *MetadataPersister) GetRootPath(ctx context.Context) (string, error) {
return root.Name, nil
}
func (p *MetadataPersister) GetHeaderDirectChildren(ctx context.Context, name string, limit int) (models.HeaderSlice, error) {
func (p *MetadataPersister) GetHeaderDirectChildren(ctx context.Context, name string, limit int) ([]*config.Header, error) {
prefix := strings.TrimSuffix(name, "/") + "/"
rootDepth := 0
headers := models.HeaderSlice{}
headers := []*config.Header{}
// Root node
if pathext.IsRoot(name) {
@@ -243,7 +256,7 @@ func (p *MetadataPersister) GetHeaderDirectChildren(ctx context.Context, name st
rootDepth = int(depth.Depth)
}
getHeaders := func(prefix string) (models.HeaderSlice, error) {
getHeaders := func(prefix string) ([]*config.Header, error) {
query := fmt.Sprintf(
`select %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v,
length(replace(%v, ?, '')) - length(replace(replace(%v, ?, ''), '/', '')) as depth
@@ -336,7 +349,7 @@ where %v like ?
}
}
outhdrs := models.HeaderSlice{}
outhdrs := []*config.Header{}
for _, hdr := range headers {
prefix := strings.TrimSuffix(hdr.Name, "/")
if name != prefix && name != prefix+"/" {
@@ -351,7 +364,7 @@ where %v like ?
return outhdrs[:limit-1], nil
}
func (p *MetadataPersister) DeleteHeader(ctx context.Context, name string, lastknownrecord, lastknownblock int64) (*models.Header, error) {
func (p *MetadataPersister) DeleteHeader(ctx context.Context, name string, lastknownrecord, lastknownblock int64) (*config.Header, error) {
hdr, err := models.FindHeader(ctx, p.DB, name)
if err != nil {
if err == sql.ErrNoRows {
@@ -376,7 +389,7 @@ func (p *MetadataPersister) DeleteHeader(ctx context.Context, name string, lastk
return nil, err
}
return hdr, nil
return converters.DBHeaderToConfigHeader(hdr), nil
}
func (p *MetadataPersister) GetLastIndexedRecordAndBlock(ctx context.Context, recordSize int) (int64, int64, error) {

5
pkg/recovery/fetch.go
View File

@@ -10,7 +10,6 @@ import (
"github.com/pojntfx/stfs/internal/compression"
"github.com/pojntfx/stfs/internal/converters"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
"github.com/pojntfx/stfs/internal/encryption"
"github.com/pojntfx/stfs/internal/mtio"
"github.com/pojntfx/stfs/internal/records"
@@ -33,7 +32,7 @@ func Fetch(
to string,
preview bool,
onHeader func(hdr *models.Header),
onHeader func(hdr *config.Header),
) error {
to = filepath.ToSlash(to)
@@ -79,7 +78,7 @@ func Fetch(
return err
}
onHeader(dbhdr)
onHeader(converters.DBHeaderToConfigHeader(dbhdr))
}
if !preview {

12
pkg/recovery/index.go
View File

@@ -41,7 +41,7 @@ func Index(
isRegular bool,
) error,
onHeader func(hdr *models.Header),
onHeader func(hdr *config.Header),
) error {
if overwrite {
if err := metadata.Metadata.PurgeAllHeaders(context.Background()); err != nil {
@@ -239,7 +239,7 @@ func indexHeader(
metadataPersister config.MetadataPersister,
compressionFormat string,
encryptionFormat string,
onHeader func(hdr *models.Header),
onHeader func(hdr *config.Header),
) error {
uncompressedSize, ok := hdr.PAXRecords[records.STFSRecordUncompressedSize]
if ok {
@@ -265,7 +265,7 @@ func indexHeader(
return err
}
onHeader(dbhdr)
onHeader(converters.DBHeaderToConfigHeader(dbhdr))
}
stfsVersion, ok := hdr.PAXRecords[records.STFSRecordVersion]
@@ -287,7 +287,7 @@ func indexHeader(
return err
}
if err := metadataPersister.UpsertHeader(context.Background(), dbhdr); err != nil {
if err := metadataPersister.UpsertHeader(context.Background(), converters.DBHeaderToConfigHeader(dbhdr)); err != nil {
return err
}
case records.STFSRecordActionDelete:
@@ -312,7 +312,7 @@ func indexHeader(
newHdr = h
if err := metadataPersister.UpdateHeaderMetadata(context.Background(), newHdr); err != nil {
if err := metadataPersister.UpdateHeaderMetadata(context.Background(), converters.DBHeaderToConfigHeader(newHdr)); err != nil {
return err
}
} else {
@@ -326,7 +326,7 @@ func indexHeader(
newHdr = h
if err := metadataPersister.UpdateHeaderMetadata(context.Background(), newHdr); err != nil {
if err := metadataPersister.UpdateHeaderMetadata(context.Background(), converters.DBHeaderToConfigHeader(newHdr)); err != nil {
return err
}
}

7
pkg/recovery/query.go
View File

@@ -8,7 +8,6 @@ import (
"math"
"github.com/pojntfx/stfs/internal/converters"
models "github.com/pojntfx/stfs/internal/db/sqlite/models/metadata"
"github.com/pojntfx/stfs/internal/encryption"
"github.com/pojntfx/stfs/internal/ioext"
"github.com/pojntfx/stfs/internal/mtio"
@@ -25,7 +24,7 @@ func Query(
record int,
block int,
onHeader func(hdr *models.Header),
onHeader func(hdr *config.Header),
) ([]*tar.Header, error) {
headers := []*tar.Header{}
@@ -103,7 +102,7 @@ func Query(
return []*tar.Header{}, err
}
onHeader(dbhdr)
onHeader(converters.DBHeaderToConfigHeader(dbhdr))
}
headers = append(headers, hdr)
@@ -191,7 +190,7 @@ func Query(
return []*tar.Header{}, err
}
onHeader(dbhdr)
onHeader(converters.DBHeaderToConfigHeader(dbhdr))
}
headers = append(headers, hdr)