albert/ScraperControl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: albert:6d1c7eb3c5c38f92aaced7ca0fd291e15607b591
Branches Tags
albert:master albert:main
...
compare: albert:main
Branches Tags
albert:main albert:master

2 Commits
6d1c7eb3c5 ... main

Author SHA1 Message Date
Albert
027ca59a01 feat: capture pastor name and phone from OrariMesse.it detail endpoint 
Populate church.pastorName from detail.parroco and church.phone from
detail.telefono during Pass 2 schedule import. Only updates when fields
are present and non-empty.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
 2026-05-26 22:43:04 -04:00
Albert
9d0af3289a feat: throttle Neon transfer with smaller batches + 1s delay 
Reduce BATCH_SIZE from 200 to 100 and add a 1-second pause between
batches to avoid overwhelming the Neon production database during
large incremental syncs.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
 2026-05-26 11:09:02 -04:00
2 changed files with 10 additions and 3 deletions
Expand all files Collapse all files

8
scripts/import-orarimesse.ts
View File

@@ -491,10 +491,14 @@ async function processSchedulesForDiocese(
})), })),
}); });
// Mark church as scraped // Update church metadata from detail (pastor, phone) if available
const churchUpdateData: Record<string, unknown> = { lastScrapedAt: new Date() };
if (detail.parroco) churchUpdateData.pastorName = detail.parroco;
if (detail.telefono) churchUpdateData.phone = detail.telefono;
await tx.church.update({ await tx.church.update({
where: { id: dbId }, where: { id: dbId },
data: { lastScrapedAt: new Date() }, data: churchUpdateData,
}); });
}); });

5
scripts/transfer-enriched-to-neon.ts
View File

@@ -113,7 +113,7 @@ async function main() {
console.log(`🔄 Incremental filter: updatedAt > ${transferSince.toISOString()}\n`); console.log(`🔄 Incremental filter: updatedAt > ${transferSince.toISOString()}\n`);
} }
const BATCH_SIZE = 200; const BATCH_SIZE = 100;
const totalCount = await nasPrisma.church.count({ where: whereClause }); const totalCount = await nasPrisma.church.count({ where: whereClause });
console.log(`Found ${totalCount} enriched churches (will process in batches of ${BATCH_SIZE})\n`); console.log(`Found ${totalCount} enriched churches (will process in batches of ${BATCH_SIZE})\n`);
@@ -270,6 +270,9 @@ async function main() {
console.error(`Error transferring ${church.name}:`, error instanceof Error ? error.message : error); console.error(`Error transferring ${church.name}:`, error instanceof Error ? error.message : error);
} }
} }
// Brief pause between batches to avoid overwhelming Neon
await new Promise(resolve => setTimeout(resolve, 1000));
} // end batch loop } // end batch loop
console.log('\n════════════════════════════════════════════════════════════'); console.log('\n════════════════════════════════════════════════════════════');