lexgen

pkg/atproto/generate.go

@@ -3,17 +3,194 @@
 
 package main
 
-// CBOR Code Generator
+// Lexicon and CBOR Code Generator
 //
-// This generates optimized CBOR marshaling code for ATProto records.
+// This generates:
+// 1. Go types from lexicon JSON files (via lex/lexgen library)
+// 2. CBOR marshaling code for ATProto records (via cbor-gen)
+// 3. Type registration for lexutil (register.go)
 //
 // Usage:
 //   go generate ./pkg/atproto/...
 //
-// This creates pkg/atproto/cbor_gen.go which should be committed to git.
-// Only re-run when you modify types in pkg/atproto/types.go
-//
-// The //go:generate directive is in lexicon.go
+// Key insight: We use RegisterLexiconTypeID: false to avoid generating init()
+// blocks that require CBORMarshaler. This breaks the circular dependency between
+// lexgen and cbor-gen. See: https://github.com/bluesky-social/indigo/issues/931
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/bluesky-social/indigo/atproto/lexicon"
+	"github.com/bluesky-social/indigo/lex/lexgen"
+	"golang.org/x/tools/imports"
+)
+
+func main() {
+	// Find repo root
+	repoRoot, err := findRepoRoot()
+	if err != nil {
+		fmt.Printf("failed to find repo root: %v\n", err)
+		os.Exit(1)
+	}
+
+	pkgDir := filepath.Join(repoRoot, "pkg/atproto")
+	lexDir := filepath.Join(repoRoot, "lexicons")
+
+	// Step 0: Clean up old register.go to avoid conflicts
+	// (It will be regenerated at the end)
+	os.Remove(filepath.Join(pkgDir, "register.go"))
+
+	// Step 1: Load all lexicon schemas into catalog (for cross-references)
+	fmt.Println("Loading lexicons...")
+	cat := lexicon.NewBaseCatalog()
+	if err := cat.LoadDirectory(lexDir); err != nil {
+		fmt.Printf("failed to load lexicons: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Step 2: Generate Go code for each lexicon file
+	fmt.Println("Running lexgen...")
+	config := &lexgen.GenConfig{
+		RegisterLexiconTypeID: false, // KEY: no init() blocks generated
+		UnknownType:           "map-string-any",
+		WarningText:           "Code generated by generate.go; DO NOT EDIT.",
+	}
+
+	// Track generated types for register.go
+	var registeredTypes []typeInfo
+
+	// Walk lexicon directory and generate code for each file
+	err = filepath.Walk(lexDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() || !strings.HasSuffix(path, ".json") {
+			return nil
+		}
+
+		// Load and parse the schema file
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("failed to read %s: %w", path, err)
+		}
+
+		var sf lexicon.SchemaFile
+		if err := json.Unmarshal(data, &sf); err != nil {
+			return fmt.Errorf("failed to parse %s: %w", path, err)
+		}
+
+		if err := sf.FinishParse(); err != nil {
+			return fmt.Errorf("failed to finish parse %s: %w", path, err)
+		}
+
+		// Flatten the schema
+		flat, err := lexgen.FlattenSchemaFile(&sf)
+		if err != nil {
+			return fmt.Errorf("failed to flatten schema %s: %w", path, err)
+		}
+
+		// Generate code
+		var buf bytes.Buffer
+		gen := &lexgen.CodeGenerator{
+			Config: config,
+			Lex:    flat,
+			Cat:    &cat,
+			Out:    &buf,
+		}
+
+		if err := gen.WriteLexicon(); err != nil {
+			return fmt.Errorf("failed to generate code for %s: %w", path, err)
+		}
+
+		// Fix package name: lexgen generates "ioatcr" but we want "atproto"
+		code := bytes.Replace(buf.Bytes(), []byte("package ioatcr"), []byte("package atproto"), 1)
+
+		// Format with goimports
+		fileName := gen.FileName()
+		formatted, err := imports.Process(fileName, code, nil)
+		if err != nil {
+			// Write unformatted for debugging
+			outPath := filepath.Join(pkgDir, fileName)
+			os.WriteFile(outPath+".broken", code, 0644)
+			return fmt.Errorf("failed to format %s: %w (wrote to %s.broken)", fileName, err, outPath)
+		}
+
+		// Write output file
+		outPath := filepath.Join(pkgDir, fileName)
+		if err := os.WriteFile(outPath, formatted, 0644); err != nil {
+			return fmt.Errorf("failed to write %s: %w", outPath, err)
+		}
+
+		fmt.Printf("  Generated %s\n", fileName)
+
+		// Track type for registration - compute type name from NSID
+		typeName := nsidToTypeName(sf.ID)
+		registeredTypes = append(registeredTypes, typeInfo{
+			NSID:     sf.ID,
+			TypeName: typeName,
+		})
+
+		return nil
+	})
+	if err != nil {
+		fmt.Printf("lexgen failed: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Step 3: Run cbor-gen via exec.Command
+	// This must be a separate process so it can compile the freshly generated types
+	fmt.Println("Running cbor-gen...")
+	if err := runCborGen(repoRoot, pkgDir); err != nil {
+		fmt.Printf("cbor-gen failed: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Step 4: Generate register.go
+	fmt.Println("Generating register.go...")
+	if err := generateRegisterFile(pkgDir, registeredTypes); err != nil {
+		fmt.Printf("failed to generate register.go: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Println("Code generation complete!")
+}
+
+type typeInfo struct {
+	NSID     string
+	TypeName string
+}
+
+// nsidToTypeName converts an NSID to a Go type name
+// io.atcr.manifest → Manifest
+// io.atcr.hold.captain → HoldCaptain
+// io.atcr.sailor.profile → SailorProfile
+func nsidToTypeName(nsid string) string {
+	parts := strings.Split(nsid, ".")
+	if len(parts) < 3 {
+		return ""
+	}
+	// Skip the first two parts (authority, e.g., "io.atcr")
+	// and capitalize each remaining part
+	var result string
+	for _, part := range parts[2:] {
+		if len(part) > 0 {
+			result += strings.ToUpper(part[:1]) + part[1:]
+		}
+	}
+	return result
+}
+
+func runCborGen(repoRoot, pkgDir string) error {
+	// Create a temporary Go file that runs cbor-gen
+	cborGenCode := `//go:build ignore
+
+package main
 
 import (
 	"fmt"
@@ -25,14 +202,81 @@ import (
 )
 
 func main() {
-	// Generate map-style encoders for CrewRecord, CaptainRecord, LayerRecord, and TangledProfileRecord
 	if err := cbg.WriteMapEncodersToFile("cbor_gen.go", "atproto",
-		atproto.CrewRecord{},
-		atproto.CaptainRecord{},
-		atproto.LayerRecord{},
+		// Manifest types
+		atproto.Manifest{},
+		atproto.Manifest_BlobReference{},
+		atproto.Manifest_ManifestReference{},
+		atproto.Manifest_Platform{},
+		atproto.Manifest_Annotations{},
+		atproto.Manifest_BlobReference_Annotations{},
+		atproto.Manifest_ManifestReference_Annotations{},
+		// Tag
+		atproto.Tag{},
+		// Sailor types
+		atproto.SailorProfile{},
+		atproto.SailorStar{},
+		atproto.SailorStar_Subject{},
+		// Hold types
+		atproto.HoldCaptain{},
+		atproto.HoldCrew{},
+		atproto.HoldLayer{},
+		// External types
 		atproto.TangledProfileRecord{},
 	); err != nil {
-		fmt.Printf("Failed to generate CBOR encoders: %v\n", err)
+		fmt.Printf("cbor-gen failed: %v\n", err)
 		os.Exit(1)
 	}
 }
+`
+
+	// Write temp file
+	tmpFile := filepath.Join(pkgDir, "cborgen_tmp.go")
+	if err := os.WriteFile(tmpFile, []byte(cborGenCode), 0644); err != nil {
+		return fmt.Errorf("failed to write temp cbor-gen file: %w", err)
+	}
+	defer os.Remove(tmpFile)
+
+	// Run it
+	cmd := exec.Command("go", "run", tmpFile)
+	cmd.Dir = pkgDir
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func generateRegisterFile(pkgDir string, types []typeInfo) error {
+	var buf bytes.Buffer
+
+	buf.WriteString("// Code generated by generate.go; DO NOT EDIT.\n\n")
+	buf.WriteString("package atproto\n\n")
+	buf.WriteString("import lexutil \"github.com/bluesky-social/indigo/lex/util\"\n\n")
+	buf.WriteString("func init() {\n")
+
+	for _, t := range types {
+		fmt.Fprintf(&buf, "\tlexutil.RegisterType(%q, &%s{})\n", t.NSID, t.TypeName)
+	}
+
+	buf.WriteString("}\n")
+
+	outPath := filepath.Join(pkgDir, "register.go")
+	return os.WriteFile(outPath, buf.Bytes(), 0644)
+}
+
+func findRepoRoot() (string, error) {
+	dir, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+
+	for {
+		if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
+			return dir, nil
+		}
+		parent := filepath.Dir(dir)
+		if parent == dir {
+			return "", fmt.Errorf("go.mod not found")
+		}
+		dir = parent
+	}
+}

pkg/atproto/holdcaptain.go

@@ -0,0 +1,24 @@
+// Code generated by generate.go; DO NOT EDIT.
+
+// Lexicon schema: io.atcr.hold.captain
+
+package atproto
+
+// Represents the hold's ownership and metadata. Stored as a singleton record at rkey 'self' in the hold's embedded PDS.
+type HoldCaptain struct {
+	LexiconTypeID string `json:"$type" cborgen:"$type,const=io.atcr.hold.captain"`
+	// allowAllCrew: Allow any authenticated user to register as crew
+	AllowAllCrew bool `json:"allowAllCrew" cborgen:"allowAllCrew"`
+	// deployedAt: RFC3339 timestamp of when the hold was deployed
+	DeployedAt string `json:"deployedAt" cborgen:"deployedAt"`
+	// enableBlueskyPosts: Enable Bluesky posts when manifests are pushed
+	EnableBlueskyPosts bool `json:"enableBlueskyPosts" cborgen:"enableBlueskyPosts"`
+	// owner: DID of the hold owner
+	Owner string `json:"owner" cborgen:"owner"`
+	// provider: Deployment provider (e.g., fly.io, aws, etc.)
+	Provider *string `json:"provider,omitempty" cborgen:"provider,omitempty"`
+	// public: Whether this hold allows public blob reads (pulls) without authentication
+	Public bool `json:"public" cborgen:"public"`
+	// region: S3 region where blobs are stored
+	Region *string `json:"region,omitempty" cborgen:"region,omitempty"`
+}

pkg/atproto/holdcrew.go

@@ -0,0 +1,18 @@
+// Code generated by generate.go; DO NOT EDIT.
+
+// Lexicon schema: io.atcr.hold.crew
+
+package atproto
+
+// Crew member in a hold's embedded PDS. Grants access permissions to push blobs to the hold. Stored in the hold's embedded PDS (one record per member).
+type HoldCrew struct {
+	LexiconTypeID string `json:"$type" cborgen:"$type,const=io.atcr.hold.crew"`
+	// addedAt: RFC3339 timestamp of when the member was added
+	AddedAt string `json:"addedAt" cborgen:"addedAt"`
+	// member: DID of the crew member
+	Member string `json:"member" cborgen:"member"`
+	// permissions: Specific permissions granted to this member
+	Permissions []string `json:"permissions" cborgen:"permissions"`
+	// role: Member's role in the hold
+	Role string `json:"role" cborgen:"role"`
+}

pkg/atproto/holdlayer.go

@@ -0,0 +1,24 @@
+// Code generated by generate.go; DO NOT EDIT.
+
+// Lexicon schema: io.atcr.hold.layer
+
+package atproto
+
+// Represents metadata about a container layer stored in the hold. Stored in the hold's embedded PDS for tracking and analytics.
+type HoldLayer struct {
+	LexiconTypeID string `json:"$type" cborgen:"$type,const=io.atcr.hold.layer"`
+	// createdAt: RFC3339 timestamp of when the layer was uploaded
+	CreatedAt string `json:"createdAt" cborgen:"createdAt"`
+	// digest: Layer digest (e.g., sha256:abc123...)
+	Digest string `json:"digest" cborgen:"digest"`
+	// mediaType: Media type (e.g., application/vnd.oci.image.layer.v1.tar+gzip)
+	MediaType string `json:"mediaType" cborgen:"mediaType"`
+	// repository: Repository this layer belongs to
+	Repository string `json:"repository" cborgen:"repository"`
+	// size: Size in bytes
+	Size int64 `json:"size" cborgen:"size"`
+	// userDid: DID of user who uploaded this layer
+	UserDid string `json:"userDid" cborgen:"userDid"`
+	// userHandle: Handle of user (for display purposes)
+	UserHandle string `json:"userHandle" cborgen:"userHandle"`
+}

pkg/atproto/manifest.go

@@ -0,0 +1,103 @@
+// Code generated by generate.go; DO NOT EDIT.
+
+// Lexicon schema: io.atcr.manifest
+
+package atproto
+
+import (
+	lexutil "github.com/bluesky-social/indigo/lex/util"
+)
+
+// A container image manifest following OCI specification, stored in ATProto
+type Manifest struct {
+	LexiconTypeID string `json:"$type" cborgen:"$type,const=io.atcr.manifest"`
+	// annotations: Optional metadata annotations
+	Annotations *Manifest_Annotations `json:"annotations,omitempty" cborgen:"annotations,omitempty"`
+	// config: Reference to image configuration blob
+	Config *Manifest_BlobReference `json:"config,omitempty" cborgen:"config,omitempty"`
+	// createdAt: Record creation timestamp
+	CreatedAt string `json:"createdAt" cborgen:"createdAt"`
+	// digest: Content digest (e.g., 'sha256:abc123...')
+	Digest string `json:"digest" cborgen:"digest"`
+	// holdDid: DID of the hold service where blobs are stored (e.g., 'did:web:hold01.atcr.io'). Primary reference for hold resolution.
+	HoldDid *string `json:"holdDid,omitempty" cborgen:"holdDid,omitempty"`
+	// holdEndpoint: Hold service endpoint URL where blobs are stored. DEPRECATED: Use holdDid instead. Kept for backward compatibility.
+	HoldEndpoint *string `json:"holdEndpoint,omitempty" cborgen:"holdEndpoint,omitempty"`
+	// layers: Filesystem layers (for image manifests)
+	Layers []Manifest_BlobReference `json:"layers,omitempty" cborgen:"layers,omitempty"`
+	// manifestBlob: The full OCI manifest stored as a blob in ATProto.
+	ManifestBlob *lexutil.LexBlob `json:"manifestBlob,omitempty" cborgen:"manifestBlob,omitempty"`
+	// manifests: Referenced manifests (for manifest lists/indexes)
+	Manifests []Manifest_ManifestReference `json:"manifests,omitempty" cborgen:"manifests,omitempty"`
+	// mediaType: OCI media type
+	MediaType string `json:"mediaType" cborgen:"mediaType"`
+	// repository: Repository name (e.g., 'myapp'). Scoped to user's DID.
+	Repository string `json:"repository" cborgen:"repository"`
+	// schemaVersion: OCI schema version (typically 2)
+	SchemaVersion int64 `json:"schemaVersion" cborgen:"schemaVersion"`
+	// subject: Optional reference to another manifest (for attestations, signatures)
+	Subject *Manifest_BlobReference `json:"subject,omitempty" cborgen:"subject,omitempty"`
+}
+
+// Optional metadata annotations
+type Manifest_Annotations struct {
+}
+
+// Manifest_BlobReference is a "blobReference" in the io.atcr.manifest schema.
+//
+// Reference to a blob stored in S3 or external storage
+type Manifest_BlobReference struct {
+	LexiconTypeID string `json:"$type,omitempty" cborgen:"$type,const=io.atcr.manifest#blobReference,omitempty"`
+	// annotations: Optional metadata
+	Annotations *Manifest_BlobReference_Annotations `json:"annotations,omitempty" cborgen:"annotations,omitempty"`
+	// digest: Content digest (e.g., 'sha256:...')
+	Digest string `json:"digest" cborgen:"digest"`
+	// mediaType: MIME type of the blob
+	MediaType string `json:"mediaType" cborgen:"mediaType"`
+	// size: Size in bytes
+	Size int64 `json:"size" cborgen:"size"`
+	// urls: Optional direct URLs to blob (for BYOS)
+	Urls []string `json:"urls,omitempty" cborgen:"urls,omitempty"`
+}
+
+// Optional metadata
+type Manifest_BlobReference_Annotations struct {
+}
+
+// Manifest_ManifestReference is a "manifestReference" in the io.atcr.manifest schema.
+//
+// Reference to a manifest in a manifest list/index
+type Manifest_ManifestReference struct {
+	LexiconTypeID string `json:"$type,omitempty" cborgen:"$type,const=io.atcr.manifest#manifestReference,omitempty"`
+	// annotations: Optional metadata
+	Annotations *Manifest_ManifestReference_Annotations `json:"annotations,omitempty" cborgen:"annotations,omitempty"`
+	// digest: Content digest (e.g., 'sha256:...')
+	Digest string `json:"digest" cborgen:"digest"`
+	// mediaType: Media type of the referenced manifest
+	MediaType string `json:"mediaType" cborgen:"mediaType"`
+	// platform: Platform information for this manifest
+	Platform *Manifest_Platform `json:"platform,omitempty" cborgen:"platform,omitempty"`
+	// size: Size in bytes
+	Size int64 `json:"size" cborgen:"size"`
+}
+
+// Optional metadata
+type Manifest_ManifestReference_Annotations struct {
+}
+
+// Manifest_Platform is a "platform" in the io.atcr.manifest schema.
+//
+// Platform information describing OS and architecture
+type Manifest_Platform struct {
+	LexiconTypeID string `json:"$type,omitempty" cborgen:"$type,const=io.atcr.manifest#platform,omitempty"`
+	// architecture: CPU architecture (e.g., 'amd64', 'arm64', 'arm')
+	Architecture string `json:"architecture" cborgen:"architecture"`
+	// os: Operating system (e.g., 'linux', 'windows', 'darwin')
+	Os string `json:"os" cborgen:"os"`
+	// osFeatures: Optional OS features
+	OsFeatures []string `json:"osFeatures,omitempty" cborgen:"osFeatures,omitempty"`
+	// osVersion: Optional OS version
+	OsVersion *string `json:"osVersion,omitempty" cborgen:"osVersion,omitempty"`
+	// variant: Optional CPU variant (e.g., 'v7' for ARM)
+	Variant *string `json:"variant,omitempty" cborgen:"variant,omitempty"`
+}

pkg/atproto/register.go

@@ -0,0 +1,15 @@
+// Code generated by generate.go; DO NOT EDIT.
+
+package atproto
+
+import lexutil "github.com/bluesky-social/indigo/lex/util"
+
+func init() {
+	lexutil.RegisterType("io.atcr.hold.captain", &HoldCaptain{})
+	lexutil.RegisterType("io.atcr.hold.crew", &HoldCrew{})
+	lexutil.RegisterType("io.atcr.hold.layer", &HoldLayer{})
+	lexutil.RegisterType("io.atcr.manifest", &Manifest{})
+	lexutil.RegisterType("io.atcr.sailor.profile", &SailorProfile{})
+	lexutil.RegisterType("io.atcr.sailor.star", &SailorStar{})
+	lexutil.RegisterType("io.atcr.tag", &Tag{})
+}

pkg/atproto/sailorprofile.go

@@ -0,0 +1,16 @@
+// Code generated by generate.go; DO NOT EDIT.
+
+// Lexicon schema: io.atcr.sailor.profile
+
+package atproto
+
+// User profile for ATCR registry. Stores preferences like default hold for blob storage.
+type SailorProfile struct {
+	LexiconTypeID string `json:"$type" cborgen:"$type,const=io.atcr.sailor.profile"`
+	// createdAt: Profile creation timestamp
+	CreatedAt string `json:"createdAt" cborgen:"createdAt"`
+	// defaultHold: Default hold endpoint for blob storage. If null, user has opted out of defaults.
+	DefaultHold *string `json:"defaultHold,omitempty" cborgen:"defaultHold,omitempty"`
+	// updatedAt: Profile last updated timestamp
+	UpdatedAt *string `json:"updatedAt,omitempty" cborgen:"updatedAt,omitempty"`
+}

pkg/atproto/sailorstar.go

@@ -0,0 +1,25 @@
+// Code generated by generate.go; DO NOT EDIT.
+
+// Lexicon schema: io.atcr.sailor.star
+
+package atproto
+
+// A star (like) on a container image repository. Stored in the starrer's PDS, similar to Bluesky likes.
+type SailorStar struct {
+	LexiconTypeID string `json:"$type" cborgen:"$type,const=io.atcr.sailor.star"`
+	// createdAt: Star creation timestamp
+	CreatedAt string `json:"createdAt" cborgen:"createdAt"`
+	// subject: The repository being starred
+	Subject SailorStar_Subject `json:"subject" cborgen:"subject"`
+}
+
+// SailorStar_Subject is a "subject" in the io.atcr.sailor.star schema.
+//
+// Reference to a repository owned by a user
+type SailorStar_Subject struct {
+	LexiconTypeID string `json:"$type,omitempty" cborgen:"$type,const=io.atcr.sailor.star#subject,omitempty"`
+	// did: DID of the repository owner
+	Did string `json:"did" cborgen:"did"`
+	// repository: Repository name (e.g., 'myapp')
+	Repository string `json:"repository" cborgen:"repository"`
+}

pkg/atproto/tag.go

@@ -0,0 +1,20 @@
+// Code generated by generate.go; DO NOT EDIT.
+
+// Lexicon schema: io.atcr.tag
+
+package atproto
+
+// A named tag pointing to a specific manifest digest
+type Tag struct {
+	LexiconTypeID string `json:"$type" cborgen:"$type,const=io.atcr.tag"`
+	// createdAt: Tag creation timestamp
+	CreatedAt string `json:"createdAt" cborgen:"createdAt"`
+	// manifest: AT-URI of the manifest this tag points to (e.g., 'at://did:plc:xyz/io.atcr.manifest/abc123'). Preferred over manifestDigest for new records.
+	Manifest *string `json:"manifest,omitempty" cborgen:"manifest,omitempty"`
+	// manifestDigest: DEPRECATED: Digest of the manifest (e.g., 'sha256:...'). Kept for backward compatibility with old records. New records should use 'manifest' field instead.
+	ManifestDigest *string `json:"manifestDigest,omitempty" cborgen:"manifestDigest,omitempty"`
+	// repository: Repository name (e.g., 'myapp'). Scoped to user's DID.
+	Repository string `json:"repository" cborgen:"repository"`
+	// tag: Tag name (e.g., 'latest', 'v1.0.0', '12-slim')
+	Tag string `json:"tag" cborgen:"tag"`
+}