Perf: Add batch transaction support for search index building
The search index build was extremely slow (25 v/s) due to individual INSERT statements without transactions. This refactors to use batch inserts with SQLite transactions, which should increase speed by 100-1000x. Changes: - Add insertVersesBatch() method in searchDatabase.js - Use BEGIN TRANSACTION / COMMIT for batch operations - Collect all verses for a version, then insert in one transaction - Removed per-verse insert calls from buildSearchIndex.js - Batch insert ~31,000 verses per version in single transaction Expected performance improvement: - Before: 25 verses/second (~82 minutes for 124k verses) - After: 2,000-5,000 verses/second (~30-60 seconds for 124k verses) SQLite is optimized for batched transactions - this is the standard pattern for bulk data loading. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -113,6 +113,38 @@ class SearchDatabase {
|
||||
});
|
||||
}
|
||||
|
||||
// Batch insert verses (MUCH faster - uses transactions)
|
||||
async insertVersesBatch(verses) {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.serialize(() => {
|
||||
this.db.run('BEGIN TRANSACTION');
|
||||
|
||||
const stmtVerses = this.db.prepare(
|
||||
`INSERT OR IGNORE INTO verses (book, chapter, verse_number, verse_text, version)
|
||||
VALUES (?, ?, ?, ?, ?)`
|
||||
);
|
||||
|
||||
const stmtFts = this.db.prepare(
|
||||
`INSERT INTO verses_fts (book, chapter, verse_number, verse_text, version)
|
||||
VALUES (?, ?, ?, ?, ?)`
|
||||
);
|
||||
|
||||
for (const verse of verses) {
|
||||
stmtVerses.run(verse.book, verse.chapter, verse.verse, verse.text, verse.version);
|
||||
stmtFts.run(verse.book, verse.chapter, verse.verse, verse.text, verse.version);
|
||||
}
|
||||
|
||||
stmtVerses.finalize();
|
||||
stmtFts.finalize();
|
||||
|
||||
this.db.run('COMMIT', (err) => {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Search using FTS5 with advanced features
|
||||
async search(query, options = {}) {
|
||||
const {
|
||||
|
||||
Reference in New Issue
Block a user