mirrors/at-container-registry
1
0
Fork 0
mirror of https://tangled.org/evan.jarrett.net/at-container-registry synced 2026-09-19 00:34:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

remove duplicate data from exporter

Browse Source
This commit is contained in:
Evan Jarrett
2026-01-08 10:24:33 -06:00
parent ab7e7c7abc
commit 263ec4b7af
2 changed files with 57 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/appview/db/export.go
-108
View File
@@ -4,8 +4,6 @@ import (
"database/sql"
"fmt"
"time"
"atcr.io/pkg/atproto"
)
// UserDataExport represents the GDPR-compliant data export for a user
@@ -18,7 +16,6 @@ type UserDataExport struct {
OAuthSessions []OAuthSessionExport `json:"oauth_sessions"`
UISessions []UISessionExport `json:"ui_sessions"`
HoldMemberships HoldMembershipsExport `json:"hold_memberships"`
KnownHolds KnownHoldsExport `json:"known_holds"`
CachedDataNote CachedDataNote `json:"cached_data_note"`
}
@@ -68,19 +65,6 @@ type HoldDenialExport struct {
LastDeniedAt time.Time `json:"last_denied_at"`
}
// KnownHoldsExport lists holds where the user has interacted
type KnownHoldsExport struct {
Note string `json:"note"`
Holds []KnownHoldExport `json:"holds"`
}
// KnownHoldExport represents a hold the user has interacted with
type KnownHoldExport struct {
HoldDID string `json:"hold_did"`
Relationship string `json:"relationship"` // "captain", "crew_member"
FirstSeen time.Time `json:"first_seen"`
ExportEndpoint string `json:"export_endpoint"`
}
// CachedDataNote explains what cached data exists and how to access it
type CachedDataNote struct {
@@ -127,13 +111,6 @@ func ExportUserData(db *sql.DB, did string) (*UserDataExport, error) {
}
export.HoldMemberships = memberships
// Get known holds (where user is captain or crew)
knownHolds, err := getKnownHoldsForExport(db, did)
if err != nil {
return nil, fmt.Errorf("failed to get known holds: %w", err)
}
export.KnownHolds = knownHolds
// Add cached data note
export.CachedDataNote = CachedDataNote{
Message: "We cache data from your PDS for performance. This cached data is NOT included in this export as it is under your direct control on your PDS.",
@@ -306,88 +283,3 @@ func getHoldMembershipsForExport(db *sql.DB, did string) (HoldMembershipsExport,
return memberships, denialRows.Err()
}
// getKnownHoldsForExport retrieves holds where user is captain or crew member
func getKnownHoldsForExport(db *sql.DB, did string) (KnownHoldsExport, error) {
known := KnownHoldsExport{
Note: "Hold services where you have interacted. Each hold stores its own records about you. Contact each hold directly to export that data.",
Holds: []KnownHoldExport{},
}
// Get holds where user is captain
captainRows, err := db.Query(`
SELECT hold_did, updated_at
FROM hold_captain_records
WHERE owner_did = ?
ORDER BY updated_at DESC
`, did)
if err != nil {
return known, err
}
defer captainRows.Close()
for captainRows.Next() {
var holdDID string
var updatedAt time.Time
err := captainRows.Scan(&holdDID, &updatedAt)
if err != nil {
return known, err
}
known.Holds = append(known.Holds, KnownHoldExport{
HoldDID: holdDID,
Relationship: "captain",
FirstSeen: updatedAt,
ExportEndpoint: resolveHoldExportEndpoint(holdDID),
})
}
if err := captainRows.Err(); err != nil {
return known, err
}
// Get holds where user is crew member
crewRows, err := db.Query(`
SELECT hold_did, created_at
FROM hold_crew_members
WHERE member_did = ?
ORDER BY created_at DESC
`, did)
if err != nil {
return known, err
}
defer crewRows.Close()
for crewRows.Next() {
var holdDID string
var createdAt time.Time
err := crewRows.Scan(&holdDID, &createdAt)
if err != nil {
return known, err
}
// Check if already added as captain
alreadyAdded := false
for _, h := range known.Holds {
if h.HoldDID == holdDID {
alreadyAdded = true
break
}
}
if !alreadyAdded {
known.Holds = append(known.Holds, KnownHoldExport{
HoldDID: holdDID,
Relationship: "crew_member",
FirstSeen: createdAt,
ExportEndpoint: resolveHoldExportEndpoint(holdDID),
})
}
}
return known, crewRows.Err()
}
// resolveHoldExportEndpoint converts a hold DID to its export endpoint URL
// Uses the shared ResolveHoldURL for did:web resolution
func resolveHoldExportEndpoint(holdDID string) string {
return atproto.ResolveHoldURL(holdDID) + atproto.HoldExportUserData
}
pkg/appview/handlers/export.go
+57 -33
View File
@@ -20,11 +20,13 @@ import (
// HoldExportResult represents the result of fetching export from a hold
type HoldExportResult struct {
HoldDID string `json:"hold_did"`
Endpoint string `json:"endpoint"`
Status string `json:"status"` // "success", "failed", "offline"
Error string `json:"error,omitempty"`
Data json.RawMessage `json:"data,omitempty"` // Raw JSON from hold
HoldDID string `json:"hold_did"`
Endpoint string `json:"endpoint"`
Relationship string `json:"relationship"` // "captain", "crew_member"
FirstSeen time.Time `json:"first_seen"`
Status string `json:"status"` // "success", "failed", "offline"
Error string `json:"error,omitempty"`
Data json.RawMessage `json:"data,omitempty"` // Raw JSON from hold
}
// FullUserDataExport represents the complete GDPR export including hold data
@@ -86,10 +88,37 @@ func (h *ExportUserDataHandler) ServeHTTP(w http.ResponseWriter, r *http.Request
"hold_count", len(holdExports))
}
// holdMetadata stores relationship info for a hold
type holdMetadata struct {
relationship string
firstSeen time.Time
}
// fetchHoldExports fetches export data from all holds where user is a member
func (h *ExportUserDataHandler) fetchHoldExports(ctx context.Context, user *db.User) []HoldExportResult {
var results []HoldExportResult
// Build metadata map: holdDID → (relationship, firstSeen)
holdMeta := make(map[string]holdMetadata)
// Get holds where user is captain
if h.DB != nil {
captainHolds, err := db.GetCaptainRecordsForOwner(h.DB, user.DID)
if err != nil {
slog.Warn("Failed to get captain records for export",
"component", "export",
"did", user.DID,
"error", err)
} else {
for _, hold := range captainHolds {
holdMeta[hold.HoldDID] = holdMetadata{
relationship: "captain",
firstSeen: hold.UpdatedAt,
}
}
}
}
// Get crew memberships from database
memberships, err := db.GetCrewMemberships(h.DB, user.DID)
if err != nil {
@@ -97,40 +126,33 @@ func (h *ExportUserDataHandler) fetchHoldExports(ctx context.Context, user *db.U
"component", "export",
"did", user.DID,
"error", err)
return results
}
if len(memberships) == 0 {
return results
}
// Collect unique hold DIDs
holdDIDs := make(map[string]bool)
for _, m := range memberships {
holdDIDs[m.HoldDID] = true
}
// Also check captain records (holds owned by user)
if h.DB != nil {
captainHolds, err := db.GetCaptainRecordsForOwner(h.DB, user.DID)
if err == nil {
for _, hold := range captainHolds {
holdDIDs[hold.HoldDID] = true
} else {
for _, m := range memberships {
// Don't overwrite captain relationship
if _, exists := holdMeta[m.HoldDID]; !exists {
holdMeta[m.HoldDID] = holdMetadata{
relationship: "crew_member",
firstSeen: m.CreatedAt,
}
}
}
}
if len(holdMeta) == 0 {
return results
}
// Fetch from each hold concurrently with timeout
var wg sync.WaitGroup
resultChan := make(chan HoldExportResult, len(holdDIDs))
resultChan := make(chan HoldExportResult, len(holdMeta))
for holdDID := range holdDIDs {
for holdDID, meta := range holdMeta {
wg.Add(1)
go func(holdDID string) {
go func(holdDID string, meta holdMetadata) {
defer wg.Done()
result := h.fetchSingleHoldExport(ctx, user, holdDID)
result := h.fetchSingleHoldExport(ctx, user, holdDID, meta)
resultChan <- result
}(holdDID)
}(holdDID, meta)
}
// Wait for all goroutines to complete
@@ -146,15 +168,17 @@ func (h *ExportUserDataHandler) fetchHoldExports(ctx context.Context, user *db.U
}
// fetchSingleHoldExport fetches export data from a single hold
func (h *ExportUserDataHandler) fetchSingleHoldExport(ctx context.Context, user *db.User, holdDID string) HoldExportResult {
func (h *ExportUserDataHandler) fetchSingleHoldExport(ctx context.Context, user *db.User, holdDID string, meta holdMetadata) HoldExportResult {
// Resolve hold DID to URL
holdURL := atproto.ResolveHoldURL(holdDID)
endpoint := holdURL + "/xrpc/io.atcr.hold.exportUserData"
result := HoldExportResult{
HoldDID: holdDID,
Endpoint: endpoint,
Status: "failed",
HoldDID: holdDID,
Endpoint: endpoint,
Relationship: meta.relationship,
FirstSeen: meta.firstSeen,
Status: "failed",
}
// Check if we have OAuth refresher (needed for service tokens)