albert/ScraperControl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: sync with Gitea master and restore local-only files

Browse Source 
Reset local main to gitea/master (new source of truth) and restored
local-only files: web scrapers, admin dashboard, ChromaDB integration,
debug scripts, and utility libraries that aren't tracked in Gitea.

Gitea master adds: discovermass, buscarmisas-network, hk-parishes,
bohosluzby, kerknet, gottesdienstzeiten, miserend importers,
ClaimRequest model, forward geocoding, heartbeat healthcheck.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Albert
2026-04-12 19:11:22 -04:00
parent 76cca3ba75
commit 2c51513851
133 changed files with 30381 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
scripts/debug/check-osm-counts.ts Normal file
View File

@@ -0,0 +1,34 @@
import dotenv from 'dotenv';
import path from 'path';
dotenv.config({ path: path.resolve(process.cwd(), '.env.local') });
dotenv.config({ path: path.resolve(process.cwd(), '.env') });
import { Pool } from 'pg';
const pool = new Pool({ connectionString: process.env.DATABASE_URL });
async function main() {
const totalRes = await pool.query(`SELECT COUNT(*) as total FROM churches WHERE source = 'osm'`);
console.log('Total OSM churches:', totalRes.rows[0].total);
const countryRes = await pool.query(`SELECT country, COUNT(*) as count FROM churches WHERE source = 'osm' AND country IS NOT NULL GROUP BY country ORDER BY count DESC LIMIT 40`);
console.log('\nTop 40 countries by OSM church count:');
for (const row of countryRes.rows) {
console.log(` ${row.country}: ${row.count}`);
}
// Check key countries that were under-imported
const keyCountries = ['AT','HR','UA','RO','LV','BY','RS','BA','MK','AL','EE','GE','AM','RU','IN','JP','CA','US','MX','AR','CO','ID','CN'];
const keyRes = await pool.query(`SELECT country, COUNT(*) as count FROM churches WHERE source = 'osm' AND country = ANY($1) GROUP BY country ORDER BY count DESC`, [keyCountries]);
console.log('\nKey countries to check (were under-imported):');
const found = new Map(keyRes.rows.map((r: any) => [r.country, r.count]));
for (const c of keyCountries) {
console.log(` ${c}: ${found.get(c) || 0}`);
}
// Total countries
const countriesRes = await pool.query(`SELECT COUNT(DISTINCT country) as total FROM churches WHERE source = 'osm'`);
console.log(`\nTotal countries with OSM data: ${countriesRes.rows[0].total}`);
await pool.end();
}
main();