albert/ScraperControl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: sync with Gitea master and restore local-only files

Browse Source 
Reset local main to gitea/master (new source of truth) and restored
local-only files: web scrapers, admin dashboard, ChromaDB integration,
debug scripts, and utility libraries that aren't tracked in Gitea.

Gitea master adds: discovermass, buscarmisas-network, hk-parishes,
bohosluzby, kerknet, gottesdienstzeiten, miserend importers,
ClaimRequest model, forward geocoding, heartbeat healthcheck.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Albert
2026-04-12 19:11:22 -04:00
parent 76cca3ba75
commit 2c51513851
133 changed files with 30381 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
scripts/debug/data-overview.ts Normal file
View File

@@ -0,0 +1,48 @@
import { Pool } from 'pg';
const pool = new Pool({ connectionString: process.env.DATABASE_URL });
async function main() {
const c = await pool.connect();
const total = await c.query('SELECT count(*) FROM "Church"');
console.log('\n=== DATABASE OVERVIEW ===');
console.log('Churches total:', Number(total.rows[0].count).toLocaleString());
const withWebsite = await c.query('SELECT count(*) FROM "Church" WHERE website IS NOT NULL');
console.log('With website:', Number(withWebsite.rows[0].count).toLocaleString());
const withSchedules = await c.query('SELECT count(DISTINCT "churchId") FROM "MassSchedule"');
console.log('With mass schedules:', Number(withSchedules.rows[0].count).toLocaleString());
const enrichedGoogle = await c.query('SELECT count(*) FROM "Church" WHERE "googlePlaceId" IS NOT NULL');
console.log('Google Places enriched:', Number(enrichedGoogle.rows[0].count).toLocaleString());
const totalSchedules = await c.query('SELECT count(*) FROM "MassSchedule"');
console.log('Total mass schedules:', Number(totalSchedules.rows[0].count).toLocaleString());
const countries = await c.query('SELECT country, count(*) as cnt FROM "Church" GROUP BY country ORDER BY cnt DESC LIMIT 15');
console.log('\n=== TOP COUNTRIES ===');
for (const r of countries.rows) console.log(' ' + (r.country || '(null)') + ':', Number(r.cnt).toLocaleString());
const sources = await c.query('SELECT source, count(*) as cnt FROM "Church" GROUP BY source ORDER BY cnt DESC LIMIT 10');
console.log('\n=== CHURCH SOURCES ===');
for (const r of sources.rows) console.log(' ' + (r.source || '(null)') + ':', Number(r.cnt).toLocaleString());
const lastScrape = await c.query('SELECT "lastScrapedAt" FROM "Church" WHERE "lastScrapedAt" IS NOT NULL ORDER BY "lastScrapedAt" DESC LIMIT 1');
console.log('\n=== LAST SCRAPE ===');
console.log(lastScrape.rows[0]?.lastScrapedAt || 'No scrapes yet');
const jobs = await c.query('SELECT status, count(*) as cnt FROM "ScrapeJob" GROUP BY status ORDER BY cnt DESC');
console.log('\n=== JOB STATUS ===');
for (const r of jobs.rows) console.log(' ' + r.status + ':', Number(r.cnt).toLocaleString());
const schedulesByLang = await c.query('SELECT language, count(*) as cnt FROM "MassSchedule" GROUP BY language ORDER BY cnt DESC LIMIT 10');
console.log('\n=== SCHEDULES BY LANGUAGE ===');
for (const r of schedulesByLang.rows) console.log(' ' + (r.language || '(null)') + ':', Number(r.cnt).toLocaleString());
c.release();
await pool.end();
}
main().catch(e => { console.error(e.message); process.exit(1); });