From 9d0af3289af04e537e9f6192c63b8d11b301f887 Mon Sep 17 00:00:00 2001 From: Albert Date: Tue, 26 May 2026 11:09:02 -0400 Subject: [PATCH] feat: throttle Neon transfer with smaller batches + 1s delay Reduce BATCH_SIZE from 200 to 100 and add a 1-second pause between batches to avoid overwhelming the Neon production database during large incremental syncs. Co-Authored-By: Claude Opus 4.7 --- scripts/transfer-enriched-to-neon.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/transfer-enriched-to-neon.ts b/scripts/transfer-enriched-to-neon.ts index 0acc570..70668b1 100644 --- a/scripts/transfer-enriched-to-neon.ts +++ b/scripts/transfer-enriched-to-neon.ts @@ -113,7 +113,7 @@ async function main() { console.log(`šŸ”„ Incremental filter: updatedAt > ${transferSince.toISOString()}\n`); } - const BATCH_SIZE = 200; + const BATCH_SIZE = 100; const totalCount = await nasPrisma.church.count({ where: whereClause }); console.log(`Found ${totalCount} enriched churches (will process in batches of ${BATCH_SIZE})\n`); @@ -270,6 +270,9 @@ async function main() { console.error(`Error transferring ${church.name}:`, error instanceof Error ? error.message : error); } } + + // Brief pause between batches to avoid overwhelming Neon + await new Promise(resolve => setTimeout(resolve, 1000)); } // end batch loop console.log('\nā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•');