yatm/internal/dataformat/bundle.go

package dataformat

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"

	"gorm.io/gorm"
)

const (
	BundleFormat   = "yatm-job-bundle"
	BundleRevision = 1
)

var ErrBundleMissing = errors.New("job bundle is missing")

type Bundle struct {
	Format        string `json:"format"`
	FormatVersion int    `json:"format_version"`
	ID            int64  `json:"id"`
	CreatedAtNS   int64  `json:"created_at_ns,string"`
}

func NewBundle(id, createdAtNS int64) Bundle {
	return Bundle{Format: BundleFormat, FormatVersion: BundleRevision, ID: id, CreatedAtNS: createdAtNS}
}

// CheckBundle validates the published family before any Job database writes.
func CheckBundle(dir string, id int64) error {
	// Metadata must be a real file, not an indirect reference to another bundle.
	filename := filepath.Join(dir, "job.json")
	info, err := os.Lstat(filename)
	if err != nil {
		return fmt.Errorf("inspect Job bundle metadata failed, id=%d, %w", id, err)
	}
	if !info.Mode().IsRegular() {
		return fmt.Errorf("Job bundle metadata is not a regular file, id=%d", id)
	}
	data, err := os.ReadFile(filename)
	if err != nil {
		return fmt.Errorf("read Job bundle metadata failed, id=%d, %w", id, err)
	}

	// Decode the required timestamp shape and consume one complete metadata document.
	var metadata struct {
		Bundle
		CreatedAtNS *int64 `json:"created_at_ns,string"`
	}
	decoder := json.NewDecoder(bytes.NewReader(data))
	decoder.DisallowUnknownFields()
	if err := decoder.Decode(&metadata); err != nil {
		return fmt.Errorf("decode Job bundle metadata failed, id=%d, %w", id, err)
	}
	if err := decoder.Decode(new(any)); err != io.EOF {
		return fmt.Errorf("invalid Job bundle metadata trailing data, id=%d", id)
	}
	if metadata.CreatedAtNS == nil {
		return fmt.Errorf("unsupported Job bundle format, id=%d: created_at_ns is required", id)
	}

	// Validate bundle identity against the current published format.
	if metadata.Format != BundleFormat || metadata.FormatVersion != BundleRevision || metadata.ID != id {
		return fmt.Errorf("unsupported Job bundle format, id=%d format=%q revision=%d", id, metadata.Format, metadata.FormatVersion)
	}
	return nil
}

// CheckBundles checks catalog-owned bundle formats without changing incomplete creations.
func CheckBundles(db *gorm.DB, workRoot string) error {
	if !db.Migrator().HasTable("jobs") {
		return nil
	}
	var after int64
	for {
		// Bound startup/preflight reads without loading any execution manifests.
		var ids []int64
		if err := db.Table("jobs").Where("id > ? AND deleted_at_ns = 0", after).Order("id").Limit(100).Pluck("id", &ids).Error; err != nil {
			return fmt.Errorf("list Job bundle identities failed, %w", err)
		}
		if len(ids) == 0 {
			return nil
		}
		for _, id := range ids {
			dir := filepath.Join(workRoot, "jobs", strconv.FormatInt(id, 10))
			info, err := os.Lstat(dir)
			if err != nil {
				if errors.Is(err, os.ErrNotExist) {
					return fmt.Errorf("job_id=%d, %w: %w", id, ErrBundleMissing, err)
				}
				return fmt.Errorf("inspect Job bundle directory failed, id=%d, %w", id, err)
			}
			if !info.IsDir() {
				return fmt.Errorf("Job bundle path is not a directory, id=%d", id)
			}
			entries, err := os.ReadDir(dir)
			if err != nil {
				return fmt.Errorf("inspect Job bundle directory failed, id=%d, %w", id, err)
			}
			// Existing recovery owns literally empty interrupted creations.
			if len(entries) != 0 {
				if err := CheckBundle(dir, id); err != nil {
					return err
				}
			}
			after = id
		}
	}
}