albert/ScraperControl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: throttle Neon transfer with smaller batches + 1s delay

Browse Source 
Reduce BATCH_SIZE from 200 to 100 and add a 1-second pause between
batches to avoid overwhelming the Neon production database during
large incremental syncs.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Albert
2026-05-26 11:09:02 -04:00
parent 6d1c7eb3c5
commit 9d0af3289a
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
scripts/transfer-enriched-to-neon.ts
View File

@@ -113,7 +113,7 @@ async function main() {
console.log(`🔄 Incremental filter: updatedAt > ${transferSince.toISOString()}\n`); console.log(`🔄 Incremental filter: updatedAt > ${transferSince.toISOString()}\n`);
} }
const BATCH_SIZE = 200; const BATCH_SIZE = 100;
const totalCount = await nasPrisma.church.count({ where: whereClause }); const totalCount = await nasPrisma.church.count({ where: whereClause });
console.log(`Found ${totalCount} enriched churches (will process in batches of ${BATCH_SIZE})\n`); console.log(`Found ${totalCount} enriched churches (will process in batches of ${BATCH_SIZE})\n`);
@@ -270,6 +270,9 @@ async function main() {
console.error(`Error transferring ${church.name}:`, error instanceof Error ? error.message : error); console.error(`Error transferring ${church.name}:`, error instanceof Error ? error.message : error);
} }
} }
// Brief pause between batches to avoid overwhelming Neon
await new Promise(resolve => setTimeout(resolve, 1000));
} // end batch loop } // end batch loop
console.log('\n════════════════════════════════════════════════════════════'); console.log('\n════════════════════════════════════════════════════════════');