From 7a53654b93a04106d2e37d486a786b4ba932b6ce Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sat, 25 Apr 2026 01:29:21 -0700
Subject: [PATCH] docs(parquet-design): add per-file descriptors to pushdown
 request

The previous Files []string was insufficient for two reasons:

- the server cannot validate that a cached side index still matches the
  data file without identity fields (size, record count, ETag);
- correct equality-delete scoping needs the Iceberg sequence number per
  data file, plus the set of position/equality delete files the
  client's Iceberg planner has attached to it.

Replace Files with DataFiles []DataFileDescriptor and document that
the client owns Iceberg planning while the server treats the resolved
set as authoritative.
---
 PARQUET_PUSHDOWN_DESIGN.md | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/PARQUET_PUSHDOWN_DESIGN.md b/PARQUET_PUSHDOWN_DESIGN.md
index 69aae415c..21c55736a 100644
--- a/PARQUET_PUSHDOWN_DESIGN.md
+++ b/PARQUET_PUSHDOWN_DESIGN.md
@@ -418,7 +418,14 @@ The pushdown planner picks per query: if the scalar predicate is aligned with a
 type ParquetPushdownRequest struct {
     Table          string
     SnapshotId     int64
-    Files          []string
+
+    // DataFiles is the authoritative list of files to scan. Each entry
+    // carries enough identity for the server to validate that its cached
+    // side indexes still apply, and enough delete-file context that the
+    // server can compute the correct visible row set without re-running
+    // Iceberg planning.
+    DataFiles      []DataFileDescriptor
+
     Columns        []string
     PredicateKind  PredicateKind // SUBSTRAIT or ICEBERG_EXPRESSION
     Predicate      []byte        // serialized per PredicateKind
@@ -428,6 +435,22 @@ type ParquetPushdownRequest struct {
     MaxRowIds      int           // cap on returned row refs; server may truncate
 }
 
+type DataFileDescriptor struct {
+    Path           string
+    SizeBytes      int64  // Iceberg manifest file_size_in_bytes
+    RecordCount    int64  // Iceberg manifest record_count
+    ETag           string // optional, used when no Iceberg manifest is available
+    SequenceNumber int64  // Iceberg sequence number, drives equality-delete scope
+    PositionDeletes []DeleteFileRef // delete files that apply to this data file
+    EqualityDeletes []DeleteFileRef
+}
+
+type DeleteFileRef struct {
+    Path           string
+    SizeBytes      int64
+    SequenceNumber int64
+}
+
 type PredicateKind int32
 
 const (
@@ -453,6 +476,12 @@ type VectorQuery struct {
 }
 ```
 
+The client is responsible for Iceberg planning (resolving the snapshot to data files and delete files) and passes the resolved set in `DataFiles`. The server treats this list as authoritative and does not re-read the catalog. Each descriptor carries:
+
+- enough identity (`SizeBytes`, `RecordCount`, optional `ETag`) for the server to verify a cached side index still matches the file,
+- the Iceberg `SequenceNumber` so equality-delete scope can be resolved correctly,
+- the position and equality delete files attached to this data file by the client's planner.
+
 v1 implementations should accept Substrait as the canonical wire format. Iceberg Expression JSON is supported as a convenience for connectors that already produce it.
 
 ### Pushdown Response