From b6691cbc9035aa35cec164fd25d55a740ae57aa0 Mon Sep 17 00:00:00 2001 From: Johanna Larsson Date: Sat, 17 Oct 2026 22:14:45 +0100 Subject: [PATCH] Set a maximum record size Left an explanation in the code as well but basically although the spec doesn't set a hard limit on the size of a CBOR record, there is a hard limit on individual commit event record blocks in the sync 1.1 spec. indigo-relay doesn't seem to validate the size of your commits, but hits a hard limit in the websocket library if the commit is too large, causing the connection to drop and instantly reconnect. I had a 10MiB record on my PDS and I had 9 relays hammering my server constantly trying to backfill that same event and choking on it over and over, until I cleaned it up. With this PR tranquil introduces a 1MB limit on CBOR records, ensuring we never go over the sync commit event limits. --- crates/tranquil-api/src/repo/record/batch.rs | 8 +++- .../src/repo/record/validation.rs | 22 +++++++++++ crates/tranquil-api/src/repo/record/write.rs | 8 +++- .../tranquil-pds/tests/it/repo_conformance.rs | 38 +++++++++++++++++++ 4 files changed, 74 insertions(+), 2 deletions(-) diff --git a/crates/tranquil-api/src/repo/record/batch.rs b/crates/tranquil-api/src/repo/record/batch.rs index e823a418..9649c5f7 100644 --- a/crates/tranquil-api/src/repo/record/batch.rs +++ b/crates/tranquil-api/src/repo/record/batch.rs @@ -1,4 +1,4 @@ -use super::validation::validate_record_with_status; +use super::validation::{validate_record_size, validate_record_with_status}; use super::validation_mode::{ValidationMode, deserialize_validation_mode}; use crate::repo::record::write::{CommitInfo, ensure_record_type}; use axum::{Json, extract::State}; @@ -85,6 +85,9 @@ async fn process_single_write( let record_ipld = tranquil_pds::util::json_to_ipld(value); let record_bytes = serde_ipld_dagcbor::to_vec(&record_ipld) .map_err(|_| ApiError::InvalidRecord("Failed to serialize record".into()))?; + + validate_record_size(&record_bytes)?; + let record_cid = tracking_store .put(&record_bytes) .await @@ -146,6 +149,9 @@ async fn process_single_write( let record_ipld = tranquil_pds::util::json_to_ipld(value); let record_bytes = serde_ipld_dagcbor::to_vec(&record_ipld) .map_err(|_| ApiError::InvalidRecord("Failed to serialize record".into()))?; + + validate_record_size(&record_bytes)?; + let record_cid = tracking_store .put(&record_bytes) .await diff --git a/crates/tranquil-api/src/repo/record/validation.rs b/crates/tranquil-api/src/repo/record/validation.rs index ecabeb0e..bbe935a6 100644 --- a/crates/tranquil-api/src/repo/record/validation.rs +++ b/crates/tranquil-api/src/repo/record/validation.rs @@ -19,6 +19,28 @@ pub async fn validate_record_with_status( .map_err(validation_error_to_api_error) } +// There is no hard limit on record sizes when creating records, just a recommendation to stay under +// 1MiB, however the Sync spec limits commit blocks to not be larger than 1MB. Anecdotally, creating a +// 10MiB record causes the relays to choke on the commit event with pretty bad outcomes. +// +// > A reasonable maximum record size limit (MAX_CBOR_RECORD_SIZE) is 1 MiByte (1 mebibyte, or 1,048,576 bytes). +// https://atproto.com/guides/data-validation#recommended-data-limits +// +// > individual record blocks within the blocks field have a hard limit of one million bytes +// https://atproto.com/specs/sync#commit-events +const MAX_CBOR_RECORD_SIZE: usize = 1_000_000; + +pub fn validate_record_size(record_bytes: &[u8]) -> Result<(), ApiError> { + if record_bytes.len() > MAX_CBOR_RECORD_SIZE { + return Err(ApiError::InvalidRecord(format!( + "Record is {} bytes; maximum size is {MAX_CBOR_RECORD_SIZE} bytes", + record_bytes.len() + ))); + } + + Ok(()) +} + fn validation_error_to_api_error(e: ValidationError) -> ApiError { let msg = match e { ValidationError::MissingType => "Record must have a $type field".to_string(), diff --git a/crates/tranquil-api/src/repo/record/write.rs b/crates/tranquil-api/src/repo/record/write.rs index e5966f78..c168e11f 100644 --- a/crates/tranquil-api/src/repo/record/write.rs +++ b/crates/tranquil-api/src/repo/record/write.rs @@ -1,4 +1,4 @@ -use super::validation::validate_record_with_status; +use super::validation::{validate_record_size, validate_record_with_status}; use super::validation_mode::{ValidationMode, deserialize_validation_mode}; use axum::{Json, extract::State}; use cid::Cid; @@ -236,6 +236,9 @@ async fn create_record_inner( let record_ipld = tranquil_pds::util::json_to_ipld(&input.record); let record_bytes = serde_ipld_dagcbor::to_vec(&record_ipld) .map_err(|_| ApiError::InvalidRecord("Failed to serialize record".into()))?; + + validate_record_size(&record_bytes)?; + let record_cid = ctx .tracking_store .put(&record_bytes) @@ -381,6 +384,9 @@ async fn put_record_inner( let record_ipld = tranquil_pds::util::json_to_ipld(&input.record); let record_bytes = serde_ipld_dagcbor::to_vec(&record_ipld) .map_err(|_| ApiError::InvalidRecord("Failed to serialize record".into()))?; + + validate_record_size(&record_bytes)?; + let record_cid = jacquard_repo::mst::util::compute_cid(&record_bytes) .map_err(|_| ApiError::InvalidRecord("Failed to compute record CID".into()))?; diff --git a/crates/tranquil-pds/tests/it/repo_conformance.rs b/crates/tranquil-pds/tests/it/repo_conformance.rs index 9993f584..e5c13e5a 100644 --- a/crates/tranquil-pds/tests/it/repo_conformance.rs +++ b/crates/tranquil-pds/tests/it/repo_conformance.rs @@ -143,6 +143,44 @@ async fn test_create_record_no_validation_status_when_validate_false() { ); } +#[tokio::test] +async fn test_create_record_max_size_validation() { + let client = client(); + let (did, jwt) = setup_new_user("conform-create-max-size").await; + let now = Utc::now().to_rfc3339(); + + let payload = json!({ + "validate": false, + "repo": did, + "collection": "com.test.feed.post", + "record": { + "$type": "com.test.feed.post", + "text": "x".repeat(1_000_001), + "createdAt": now + } + }); + + let res = client + .post(format!( + "{}/xrpc/com.atproto.repo.createRecord", + base_url().await + )) + .bearer_auth(&jwt) + .json(&payload) + .send() + .await + .expect("Failed to create record"); + + assert_eq!(res.status(), StatusCode::BAD_REQUEST); + let body: Value = res.json().await.unwrap(); + + assert!(body["error"].is_string(), "InvalidRecord"); + assert!( + body["error"].is_string(), + "Record is 1000081 bytes; maximum size is 1000000 bytes" + ); +} + #[tokio::test] async fn test_put_record_response_schema() { ensure_test_schemas();