522 líneas
17 KiB
JavaScript
522 líneas
17 KiB
JavaScript
#!/usr/bin/env node
|
||
|
||
/**
|
||
* Hasher Indexer Script
|
||
*
|
||
* This script reads a text file with one word/phrase per line and indexes
|
||
* all the generated hashes into Redis.
|
||
*
|
||
* Usage:
|
||
* npx tsx scripts/index-file.ts <path-to-file.txt> [options]
|
||
* npm run index-file -- <path-to-file.txt> [options]
|
||
*
|
||
* Options:
|
||
* --batch-size=<number> Number of items to process in each batch (default: 100)
|
||
* --resume Resume from last saved state (default: true)
|
||
* --no-resume Start from beginning, ignore saved state
|
||
* --no-check Skip duplicate checking (faster, but may create duplicates)
|
||
* --state-file=<path> Custom state file path (default: .indexer-state-<filename>.json)
|
||
* --help, -h Show this help message
|
||
*/
|
||
|
||
import Redis from 'ioredis';
|
||
import { createReadStream, existsSync, readFileSync, writeFileSync, unlinkSync } from 'fs';
|
||
import { resolve, basename } from 'path';
|
||
import { createInterface } from 'readline';
|
||
import crypto from 'crypto';
|
||
|
||
const REDIS_HOST = process.env.REDIS_HOST || 'localhost';
|
||
const REDIS_PORT = parseInt(process.env.REDIS_PORT || '6379', 10);
|
||
const REDIS_PASSWORD = process.env.REDIS_PASSWORD || undefined;
|
||
const REDIS_DB = parseInt(process.env.REDIS_DB || '0', 10);
|
||
const INDEX_NAME = 'hasher';
|
||
const DEFAULT_BATCH_SIZE = 100;
|
||
|
||
interface HashDocument {
|
||
plaintext: string;
|
||
md5: string;
|
||
sha1: string;
|
||
sha256: string;
|
||
sha512: string;
|
||
created_at: string;
|
||
}
|
||
|
||
interface IndexerState {
|
||
filePath: string;
|
||
fileHash: string;
|
||
lastProcessedLine: number;
|
||
totalLines: number;
|
||
indexed: number;
|
||
skipped: number;
|
||
errors: number;
|
||
startTime: number;
|
||
lastUpdate: string;
|
||
}
|
||
|
||
interface ParsedArgs {
|
||
filePath: string | null;
|
||
batchSize: number;
|
||
resume: boolean;
|
||
checkDuplicates: boolean;
|
||
stateFile: string | null;
|
||
showHelp: boolean;
|
||
}
|
||
|
||
function parseArgs(args: string[]): ParsedArgs {
|
||
const result: ParsedArgs = {
|
||
filePath: null,
|
||
batchSize: DEFAULT_BATCH_SIZE,
|
||
resume: true,
|
||
checkDuplicates: true,
|
||
stateFile: null,
|
||
showHelp: false
|
||
};
|
||
|
||
for (let i = 0; i < args.length; i++) {
|
||
const arg = args[i];
|
||
|
||
if (arg === '--help' || arg === '-h') {
|
||
result.showHelp = true;
|
||
} else if (arg === '--resume') {
|
||
result.resume = true;
|
||
} else if (arg === '--no-resume') {
|
||
result.resume = false;
|
||
} else if (arg === '--no-check') {
|
||
result.checkDuplicates = false;
|
||
} else if (arg.startsWith('--batch-size=')) {
|
||
const value = arg.split('=')[1];
|
||
const parsed = parseInt(value, 10);
|
||
if (!isNaN(parsed) && parsed > 0) {
|
||
result.batchSize = parsed;
|
||
}
|
||
} else if (arg === '--batch-size') {
|
||
// Support --batch-size <value> format
|
||
const nextArg = args[i + 1];
|
||
if (nextArg && !nextArg.startsWith('-')) {
|
||
const parsed = parseInt(nextArg, 10);
|
||
if (!isNaN(parsed) && parsed > 0) {
|
||
result.batchSize = parsed;
|
||
i++; // Skip next argument
|
||
}
|
||
}
|
||
} else if (arg.startsWith('--state-file=')) {
|
||
result.stateFile = arg.split('=')[1];
|
||
} else if (arg === '--state-file') {
|
||
const nextArg = args[i + 1];
|
||
if (nextArg && !nextArg.startsWith('-')) {
|
||
result.stateFile = nextArg;
|
||
i++;
|
||
}
|
||
} else if (!arg.startsWith('-')) {
|
||
// Positional argument - treat as file path
|
||
result.filePath = arg;
|
||
}
|
||
}
|
||
|
||
return result;
|
||
}
|
||
|
||
function getFileHash(filePath: string): string {
|
||
// Create a hash based on file path and size for quick identification
|
||
const stats = require('fs').statSync(filePath);
|
||
const hashInput = `${filePath}:${stats.size}:${stats.mtime.getTime()}`;
|
||
return crypto.createHash('md5').update(hashInput).digest('hex').substring(0, 8);
|
||
}
|
||
|
||
function getDefaultStateFile(filePath: string): string {
|
||
const fileName = basename(filePath).replace(/\.[^.]+$/, '');
|
||
return resolve(`.indexer-state-${fileName}.json`);
|
||
}
|
||
|
||
function loadState(stateFile: string): IndexerState | null {
|
||
try {
|
||
if (existsSync(stateFile)) {
|
||
const data = readFileSync(stateFile, 'utf-8');
|
||
return JSON.parse(data) as IndexerState;
|
||
}
|
||
} catch (error) {
|
||
console.warn(`⚠️ Could not load state file: ${error}`);
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function saveState(stateFile: string, state: IndexerState): void {
|
||
try {
|
||
state.lastUpdate = new Date().toISOString();
|
||
writeFileSync(stateFile, JSON.stringify(state, null, 2), 'utf-8');
|
||
} catch (error) {
|
||
console.error(`❌ Could not save state file: ${error}`);
|
||
}
|
||
}
|
||
|
||
function deleteState(stateFile: string): void {
|
||
try {
|
||
if (existsSync(stateFile)) {
|
||
unlinkSync(stateFile);
|
||
}
|
||
} catch (error) {
|
||
console.warn(`⚠️ Could not delete state file: ${error}`);
|
||
}
|
||
}
|
||
|
||
async function generateHashes(plaintext: string): Promise<HashDocument> {
|
||
return {
|
||
plaintext,
|
||
md5: crypto.createHash('md5').update(plaintext).digest('hex'),
|
||
sha1: crypto.createHash('sha1').update(plaintext).digest('hex'),
|
||
sha256: crypto.createHash('sha256').update(plaintext).digest('hex'),
|
||
sha512: crypto.createHash('sha512').update(plaintext).digest('hex'),
|
||
created_at: new Date().toISOString()
|
||
};
|
||
}
|
||
|
||
function showHelp() {
|
||
console.log(`
|
||
Hasher Indexer Script
|
||
|
||
Usage:
|
||
npx tsx scripts/index-file.ts <path-to-file.txt> [options]
|
||
npm run index-file -- <path-to-file.txt> [options]
|
||
|
||
Options:
|
||
--batch-size=<number> Number of items to process in each batch (default: 100)
|
||
--batch-size <number> Alternative syntax for batch size
|
||
--resume Resume from last saved state (default)
|
||
--no-resume Start from beginning, ignore saved state
|
||
--no-check Skip duplicate checking (faster, but may create duplicates)
|
||
--state-file=<path> Custom state file path
|
||
--help, -h Show this help message
|
||
|
||
Environment Variables:
|
||
REDIS_HOST Redis host (default: localhost)
|
||
REDIS_PORT Redis port (default: 6379)
|
||
REDIS_PASSWORD Redis password (optional)
|
||
REDIS_DB Redis database number (default: 0)
|
||
|
||
Examples:
|
||
npx tsx scripts/index-file.ts wordlist.txt
|
||
npx tsx scripts/index-file.ts wordlist.txt --batch-size=500
|
||
npx tsx scripts/index-file.ts wordlist.txt --batch-size 500
|
||
npx tsx scripts/index-file.ts wordlist.txt --no-resume
|
||
npx tsx scripts/index-file.ts wordlist.txt --no-check
|
||
npm run index-file -- wordlist.txt --batch-size=500 --no-check
|
||
|
||
State Management:
|
||
The script automatically saves progress to a state file. If interrupted,
|
||
it will resume from where it left off on the next run. Use --no-resume
|
||
to start fresh.
|
||
|
||
Duplicate Checking:
|
||
By default, the script checks if each plaintext or hash already exists
|
||
in the index before inserting. Use --no-check to skip this verification
|
||
for faster indexing (useful when you're sure there are no duplicates).
|
||
`);
|
||
process.exit(0);
|
||
}
|
||
|
||
async function indexFile(filePath: string, batchSize: number, shouldResume: boolean, checkDuplicates: boolean, customStateFile: string | null) {
|
||
const client = new Redis({
|
||
host: REDIS_HOST,
|
||
port: REDIS_PORT,
|
||
password: REDIS_PASSWORD,
|
||
db: REDIS_DB,
|
||
retryStrategy: (times) => Math.min(times * 50, 2000),
|
||
});
|
||
|
||
const absolutePath = resolve(filePath);
|
||
const stateFile = customStateFile || getDefaultStateFile(absolutePath);
|
||
const fileHash = getFileHash(absolutePath);
|
||
|
||
// State management
|
||
let state: IndexerState = {
|
||
filePath: absolutePath,
|
||
fileHash,
|
||
lastProcessedLine: 0,
|
||
totalLines: 0,
|
||
indexed: 0,
|
||
skipped: 0,
|
||
errors: 0,
|
||
startTime: Date.now(),
|
||
lastUpdate: new Date().toISOString()
|
||
};
|
||
|
||
// Check for existing state
|
||
const existingState = loadState(stateFile);
|
||
let resumingFrom = 0;
|
||
|
||
if (shouldResume && existingState) {
|
||
if (existingState.fileHash === fileHash) {
|
||
state = existingState;
|
||
resumingFrom = state.lastProcessedLine;
|
||
state.startTime = Date.now(); // Reset start time for this session
|
||
console.log(`📂 Found existing state, resuming from line ${resumingFrom}`);
|
||
} else {
|
||
console.log(`⚠️ File has changed since last run, starting fresh`);
|
||
deleteState(stateFile);
|
||
}
|
||
} else if (!shouldResume) {
|
||
deleteState(stateFile);
|
||
}
|
||
|
||
console.log(`📚 Hasher Indexer`);
|
||
console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━`);
|
||
console.log(`Redis: ${REDIS_HOST}:${REDIS_PORT} (DB ${REDIS_DB})`);
|
||
console.log(`Index: ${INDEX_NAME}`);
|
||
console.log(`File: ${filePath}`);
|
||
console.log(`Batch size: ${batchSize}`);
|
||
console.log(`Check duplicates: ${checkDuplicates ? 'yes' : 'no (--no-check)'}`);
|
||
console.log(`State file: ${stateFile}`);
|
||
if (resumingFrom > 0) {
|
||
console.log(`Resuming from: line ${resumingFrom}`);
|
||
console.log(`Already indexed: ${state.indexed}`);
|
||
console.log(`Already skipped: ${state.skipped}`);
|
||
}
|
||
console.log('');
|
||
|
||
// Handle interruption signals
|
||
let isInterrupted = false;
|
||
const handleInterrupt = () => {
|
||
if (isInterrupted) {
|
||
console.log('\n\n⚡ Force quit. Progress saved.');
|
||
process.exit(1);
|
||
}
|
||
isInterrupted = true;
|
||
console.log('\n\n⏸️ Interrupted! Saving state...');
|
||
saveState(stateFile, state);
|
||
console.log(`💾 State saved to ${stateFile}`);
|
||
console.log(` Resume with: npx tsx scripts/index-file.ts ${filePath}`);
|
||
console.log(` Or start fresh with: npx tsx scripts/index-file.ts ${filePath} --no-resume`);
|
||
process.exit(0);
|
||
};
|
||
|
||
process.on('SIGINT', handleInterrupt);
|
||
process.on('SIGTERM', handleInterrupt);
|
||
|
||
try {
|
||
// Test connection
|
||
console.log('🔗 Connecting to Redis...');
|
||
await client.ping();
|
||
console.log('✅ Connected successfully\n');
|
||
|
||
// Process file line by line using streams
|
||
console.log('📖 Processing file...\n');
|
||
|
||
let currentLineNumber = 0;
|
||
let currentBatch: string[] = [];
|
||
let sessionIndexed = 0;
|
||
let sessionSkipped = 0;
|
||
let sessionErrors = 0;
|
||
const sessionStartTime = Date.now();
|
||
|
||
const fileStream = createReadStream(absolutePath, { encoding: 'utf-8' });
|
||
const rl = createInterface({
|
||
input: fileStream,
|
||
crlfDelay: Infinity
|
||
});
|
||
|
||
const processBatch = async (batch: string[], lineNumber: number) => {
|
||
if (batch.length === 0) return;
|
||
if (isInterrupted) return;
|
||
|
||
// Generate hashes for all items in batch first
|
||
const batchWithHashes = await Promise.all(
|
||
batch.map(async (plaintext: string) => ({
|
||
plaintext,
|
||
hashes: await generateHashes(plaintext)
|
||
}))
|
||
);
|
||
|
||
const pipeline = client.pipeline();
|
||
let toIndex: typeof batchWithHashes = [];
|
||
|
||
if (checkDuplicates) {
|
||
// Check which items already exist
|
||
const existenceChecks = await Promise.all(
|
||
batchWithHashes.map(async (item) => {
|
||
const plaintextExists = await client.exists(`hash:plaintext:${item.plaintext}`);
|
||
if (plaintextExists) return { item, exists: true };
|
||
|
||
// Check if any hash exists
|
||
const md5Exists = await client.exists(`hash:index:md5:${item.hashes.md5}`);
|
||
const sha1Exists = await client.exists(`hash:index:sha1:${item.hashes.sha1}`);
|
||
const sha256Exists = await client.exists(`hash:index:sha256:${item.hashes.sha256}`);
|
||
const sha512Exists = await client.exists(`hash:index:sha512:${item.hashes.sha512}`);
|
||
|
||
return {
|
||
item,
|
||
exists: md5Exists || sha1Exists || sha256Exists || sha512Exists
|
||
};
|
||
})
|
||
);
|
||
|
||
for (const check of existenceChecks) {
|
||
if (check.exists) {
|
||
state.skipped++;
|
||
sessionSkipped++;
|
||
} else {
|
||
toIndex.push(check.item);
|
||
}
|
||
}
|
||
} else {
|
||
// No duplicate checking - index everything
|
||
toIndex = batchWithHashes;
|
||
}
|
||
|
||
// Execute bulk operations
|
||
if (toIndex.length > 0) {
|
||
try {
|
||
for (const item of toIndex) {
|
||
const doc = item.hashes;
|
||
const key = `hash:plaintext:${doc.plaintext}`;
|
||
|
||
// Store main document
|
||
pipeline.set(key, JSON.stringify(doc));
|
||
|
||
// Create indexes for each hash type
|
||
pipeline.set(`hash:index:md5:${doc.md5}`, doc.plaintext);
|
||
pipeline.set(`hash:index:sha1:${doc.sha1}`, doc.plaintext);
|
||
pipeline.set(`hash:index:sha256:${doc.sha256}`, doc.plaintext);
|
||
pipeline.set(`hash:index:sha512:${doc.sha512}`, doc.plaintext);
|
||
|
||
// Update statistics
|
||
pipeline.hincrby('hash:stats', 'count', 1);
|
||
pipeline.hincrby('hash:stats', 'size', JSON.stringify(doc).length);
|
||
}
|
||
|
||
const results = await pipeline.exec();
|
||
|
||
// Count errors
|
||
const errorCount = results?.filter(([err]) => err !== null).length || 0;
|
||
|
||
if (errorCount > 0) {
|
||
state.errors += errorCount;
|
||
sessionErrors += errorCount;
|
||
const successCount = toIndex.length - errorCount;
|
||
state.indexed += successCount;
|
||
sessionIndexed += successCount;
|
||
} else {
|
||
state.indexed += toIndex.length;
|
||
sessionIndexed += toIndex.length;
|
||
}
|
||
} catch (error) {
|
||
console.error(`\n❌ Error processing batch:`, error);
|
||
state.errors += toIndex.length;
|
||
sessionErrors += toIndex.length;
|
||
}
|
||
}
|
||
|
||
// Update state
|
||
state.lastProcessedLine = lineNumber;
|
||
state.totalLines = lineNumber;
|
||
|
||
// Save state periodically (every 10 batches)
|
||
if (lineNumber % (batchSize * 10) === 0) {
|
||
saveState(stateFile, state);
|
||
}
|
||
|
||
// Progress indicator
|
||
const elapsed = ((Date.now() - sessionStartTime) / 1000).toFixed(0);
|
||
process.stdout.write(`\r⏳ Line: ${lineNumber} | Session: +${sessionIndexed} indexed, +${sessionSkipped} skipped | Total: ${state.indexed} indexed | Time: ${elapsed}s`);
|
||
};
|
||
|
||
for await (const line of rl) {
|
||
if (isInterrupted) break;
|
||
|
||
currentLineNumber++;
|
||
|
||
// Skip already processed lines
|
||
if (currentLineNumber <= resumingFrom) {
|
||
continue;
|
||
}
|
||
|
||
const trimmedLine = line.trim();
|
||
if (trimmedLine.length > 0) {
|
||
// Only take first word (no spaces or separators)
|
||
const firstWord = trimmedLine.split(/\s+/)[0];
|
||
if (firstWord) {
|
||
currentBatch.push(firstWord);
|
||
|
||
if (currentBatch.length >= batchSize) {
|
||
await processBatch(currentBatch, currentLineNumber);
|
||
currentBatch = [];
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Process remaining items in last batch
|
||
if (currentBatch.length > 0 && !isInterrupted) {
|
||
await processBatch(currentBatch, currentLineNumber);
|
||
}
|
||
|
||
if (isInterrupted) {
|
||
return;
|
||
}
|
||
|
||
// No refresh needed for Redis
|
||
console.log('\n\n✅ All data persisted to Redis');
|
||
|
||
// Delete state file on successful completion
|
||
deleteState(stateFile);
|
||
|
||
const duration = ((Date.now() - sessionStartTime) / 1000).toFixed(2);
|
||
const rate = sessionIndexed > 0 ? (sessionIndexed / parseFloat(duration)).toFixed(0) : '0';
|
||
|
||
console.log('\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
|
||
console.log('✅ Indexing complete!');
|
||
console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━`);
|
||
console.log(`Total lines processed: ${currentLineNumber}`);
|
||
if (resumingFrom > 0) {
|
||
console.log(`Lines skipped (resumed): ${resumingFrom}`);
|
||
console.log(`Lines processed this session: ${currentLineNumber - resumingFrom}`);
|
||
}
|
||
console.log(`Successfully indexed (total): ${state.indexed}`);
|
||
console.log(`Successfully indexed (session): ${sessionIndexed}`);
|
||
console.log(`Skipped duplicates (total): ${state.skipped}`);
|
||
console.log(`Skipped duplicates (session): ${sessionSkipped}`);
|
||
console.log(`Errors (total): ${state.errors}`);
|
||
console.log(`Session duration: ${duration}s`);
|
||
console.log(`Session rate: ${rate} docs/sec`);
|
||
console.log('');
|
||
|
||
} catch (error) {
|
||
// Save state on error
|
||
saveState(stateFile, state);
|
||
console.error(`\n💾 State saved to ${stateFile}`);
|
||
console.error('❌ Error:', error instanceof Error ? error.message : error);
|
||
process.exit(1);
|
||
} finally {
|
||
// Remove signal handlers
|
||
process.removeListener('SIGINT', handleInterrupt);
|
||
process.removeListener('SIGTERM', handleInterrupt);
|
||
}
|
||
}
|
||
|
||
// Parse command line arguments
|
||
const args = process.argv.slice(2);
|
||
const parsedArgs = parseArgs(args);
|
||
|
||
if (parsedArgs.showHelp || !parsedArgs.filePath) {
|
||
showHelp();
|
||
}
|
||
|
||
const filePath = parsedArgs.filePath as string;
|
||
|
||
// Validate file exists
|
||
if (!existsSync(filePath)) {
|
||
console.error(`❌ File not found: ${filePath}`);
|
||
process.exit(1);
|
||
}
|
||
|
||
console.log(`\n🔧 Configuration:`);
|
||
console.log(` File: ${filePath}`);
|
||
console.log(` Batch size: ${parsedArgs.batchSize}`);
|
||
console.log(` Resume: ${parsedArgs.resume}`);
|
||
console.log(` Check duplicates: ${parsedArgs.checkDuplicates}`);
|
||
if (parsedArgs.stateFile) {
|
||
console.log(` State file: ${parsedArgs.stateFile}`);
|
||
}
|
||
console.log('');
|
||
|
||
indexFile(filePath, parsedArgs.batchSize, parsedArgs.resume, parsedArgs.checkDuplicates, parsedArgs.stateFile).catch(console.error);
|