new redis migration

Signed-off-by: ale <ale@manalejandro.com>

scripts/index-file.ts

@@ -4,7 +4,7 @@
  * Hasher Indexer Script
  * 
  * This script reads a text file with one word/phrase per line and indexes
- * all the generated hashes into Elasticsearch.
+ * all the generated hashes into Redis.
  * 
  * Usage:
  *   npx tsx scripts/index-file.ts <path-to-file.txt> [options]
@@ -19,13 +19,16 @@
  *   --help, -h             Show this help message
  */
 
-import { Client } from '@elastic/elasticsearch';
+import Redis from 'ioredis';
 import { createReadStream, existsSync, readFileSync, writeFileSync, unlinkSync } from 'fs';
 import { resolve, basename } from 'path';
 import { createInterface } from 'readline';
 import crypto from 'crypto';
 
-const ELASTICSEARCH_NODE = process.env.ELASTICSEARCH_NODE || 'http://localhost:9200';
+const REDIS_HOST = process.env.REDIS_HOST || 'localhost';
+const REDIS_PORT = parseInt(process.env.REDIS_PORT || '6379', 10);
+const REDIS_PASSWORD = process.env.REDIS_PASSWORD || undefined;
+const REDIS_DB = parseInt(process.env.REDIS_DB || '0', 10);
 const INDEX_NAME = 'hasher';
 const DEFAULT_BATCH_SIZE = 100;
 
@@ -35,7 +38,6 @@ interface HashDocument {
   sha1: string;
   sha256: string;
   sha512: string;
-  bcrypt: string;
   created_at: string;
 }
 
@@ -158,16 +160,12 @@ function deleteState(stateFile: string): void {
 }
 
 async function generateHashes(plaintext: string): Promise<HashDocument> {
-  const bcrypt = await import('bcrypt');
-  const bcryptHash = await bcrypt.default.hash(plaintext, 10);
-  
   return {
     plaintext,
     md5: crypto.createHash('md5').update(plaintext).digest('hex'),
     sha1: crypto.createHash('sha1').update(plaintext).digest('hex'),
     sha256: crypto.createHash('sha256').update(plaintext).digest('hex'),
     sha512: crypto.createHash('sha512').update(plaintext).digest('hex'),
-    bcrypt: bcryptHash,
     created_at: new Date().toISOString()
   };
 }
@@ -190,7 +188,10 @@ Options:
   --help, -h             Show this help message
 
 Environment Variables:
-  ELASTICSEARCH_NODE     Elasticsearch node URL (default: http://localhost:9200)
+  REDIS_HOST             Redis host (default: localhost)
+  REDIS_PORT             Redis port (default: 6379)
+  REDIS_PASSWORD         Redis password (optional)
+  REDIS_DB               Redis database number (default: 0)
 
 Examples:
   npx tsx scripts/index-file.ts wordlist.txt
@@ -214,7 +215,14 @@ Duplicate Checking:
 }
 
 async function indexFile(filePath: string, batchSize: number, shouldResume: boolean, checkDuplicates: boolean, customStateFile: string | null) {
-  const client = new Client({ node: ELASTICSEARCH_NODE });
+  const client = new Redis({
+    host: REDIS_HOST,
+    port: REDIS_PORT,
+    password: REDIS_PASSWORD,
+    db: REDIS_DB,
+    retryStrategy: (times) => Math.min(times * 50, 2000),
+  });
+  
   const absolutePath = resolve(filePath);
   const stateFile = customStateFile || getDefaultStateFile(absolutePath);
   const fileHash = getFileHash(absolutePath);
@@ -252,7 +260,7 @@ async function indexFile(filePath: string, batchSize: number, shouldResume: bool
 
   console.log(`📚 Hasher Indexer`);
   console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━`);
-  console.log(`Elasticsearch: ${ELASTICSEARCH_NODE}`);
+  console.log(`Redis: ${REDIS_HOST}:${REDIS_PORT} (DB ${REDIS_DB})`);
   console.log(`Index: ${INDEX_NAME}`);
   console.log(`File: ${filePath}`);
   console.log(`Batch size: ${batchSize}`);
@@ -286,8 +294,8 @@ async function indexFile(filePath: string, batchSize: number, shouldResume: bool
 
   try {
     // Test connection
-    console.log('🔗 Connecting to Elasticsearch...');
-    await client.cluster.health({});
+    console.log('🔗 Connecting to Redis...');
+    await client.ping();
     console.log('✅ Connected successfully\n');
 
     // Process file line by line using streams
@@ -310,8 +318,6 @@ async function indexFile(filePath: string, batchSize: number, shouldResume: bool
       if (batch.length === 0) return;
       if (isInterrupted) return;
 
-      const bulkOperations: any[] = [];
-
       // Generate hashes for all items in batch first
       const batchWithHashes = await Promise.all(
         batch.map(async (plaintext: string) => ({
@@ -320,92 +326,82 @@ async function indexFile(filePath: string, batchSize: number, shouldResume: bool
         }))
       );
 
+      const pipeline = client.pipeline();
+      let toIndex: typeof batchWithHashes = [];
+
       if (checkDuplicates) {
-        // Check which items already exist (by plaintext or any hash)
-        const md5List = batchWithHashes.map((item: any) => item.hashes.md5);
-        const sha1List = batchWithHashes.map((item: any) => item.hashes.sha1);
-        const sha256List = batchWithHashes.map((item: any) => item.hashes.sha256);
-        const sha512List = batchWithHashes.map((item: any) => item.hashes.sha512);
+        // Check which items already exist
+        const existenceChecks = await Promise.all(
+          batchWithHashes.map(async (item) => {
+            const plaintextExists = await client.exists(`hash:plaintext:${item.plaintext}`);
+            if (plaintextExists) return { item, exists: true };
+            
+            // Check if any hash exists
+            const md5Exists = await client.exists(`hash:index:md5:${item.hashes.md5}`);
+            const sha1Exists = await client.exists(`hash:index:sha1:${item.hashes.sha1}`);
+            const sha256Exists = await client.exists(`hash:index:sha256:${item.hashes.sha256}`);
+            const sha512Exists = await client.exists(`hash:index:sha512:${item.hashes.sha512}`);
+            
+            return {
+              item,
+              exists: md5Exists || sha1Exists || sha256Exists || sha512Exists
+            };
+          })
+        );
 
-        const existingCheck = await client.search({
-          index: INDEX_NAME,
-          size: batchSize * 5,
-          query: {
-            bool: {
-              should: [
-                { terms: { 'plaintext.keyword': batch } },
-                { terms: { md5: md5List } },
-                { terms: { sha1: sha1List } },
-                { terms: { sha256: sha256List } },
-                { terms: { sha512: sha512List } },
-              ],
-              minimum_should_match: 1
-            }
-          },
-          _source: ['plaintext', 'md5', 'sha1', 'sha256', 'sha512']
-        });
-
-        // Create a set of existing hashes for quick lookup
-        const existingHashes = new Set<string>();
-        existingCheck.hits.hits.forEach((hit: any) => {
-          const src = hit._source;
-          existingHashes.add(src.plaintext);
-          existingHashes.add(src.md5);
-          existingHashes.add(src.sha1);
-          existingHashes.add(src.sha256);
-          existingHashes.add(src.sha512);
-        });
-
-        // Prepare bulk operations only for items that don't have any duplicate hash
-        for (const item of batchWithHashes) {
-          const isDuplicate = 
-            existingHashes.has(item.plaintext) ||
-            existingHashes.has(item.hashes.md5) ||
-            existingHashes.has(item.hashes.sha1) ||
-            existingHashes.has(item.hashes.sha256) ||
-            existingHashes.has(item.hashes.sha512);
-
-          if (!isDuplicate) {
-            bulkOperations.push({ index: { _index: INDEX_NAME } });
-            bulkOperations.push(item.hashes);
-          } else {
+        for (const check of existenceChecks) {
+          if (check.exists) {
             state.skipped++;
             sessionSkipped++;
+          } else {
+            toIndex.push(check.item);
           }
         }
       } else {
         // No duplicate checking - index everything
-        for (const item of batchWithHashes) {
-          bulkOperations.push({ index: { _index: INDEX_NAME } });
-          bulkOperations.push(item.hashes);
-        }
+        toIndex = batchWithHashes;
       }
 
-      // Execute bulk operation only if there are new items to insert
-      if (bulkOperations.length > 0) {
+      // Execute bulk operations
+      if (toIndex.length > 0) {
         try {
-          const bulkResponse = await client.bulk({
-            operations: bulkOperations,
-            refresh: false
-          });
+          for (const item of toIndex) {
+            const doc = item.hashes;
+            const key = `hash:plaintext:${doc.plaintext}`;
+            
+            // Store main document
+            pipeline.set(key, JSON.stringify(doc));
+            
+            // Create indexes for each hash type
+            pipeline.set(`hash:index:md5:${doc.md5}`, doc.plaintext);
+            pipeline.set(`hash:index:sha1:${doc.sha1}`, doc.plaintext);
+            pipeline.set(`hash:index:sha256:${doc.sha256}`, doc.plaintext);
+            pipeline.set(`hash:index:sha512:${doc.sha512}`, doc.plaintext);
+            
+            // Update statistics
+            pipeline.hincrby('hash:stats', 'count', 1);
+            pipeline.hincrby('hash:stats', 'size', JSON.stringify(doc).length);
+          }
 
-          if (bulkResponse.errors) {
-            const errorCount = bulkResponse.items.filter((item: any) => item.index?.error).length;
+          const results = await pipeline.exec();
+          
+          // Count errors
+          const errorCount = results?.filter(([err]) => err !== null).length || 0;
+          
+          if (errorCount > 0) {
             state.errors += errorCount;
             sessionErrors += errorCount;
-            const successCount = (bulkOperations.length / 2) - errorCount;
+            const successCount = toIndex.length - errorCount;
             state.indexed += successCount;
             sessionIndexed += successCount;
           } else {
-            const count = bulkOperations.length / 2;
-            state.indexed += count;
-            sessionIndexed += count;
+            state.indexed += toIndex.length;
+            sessionIndexed += toIndex.length;
           }
         } catch (error) {
           console.error(`\n❌ Error processing batch:`, error);
-          const count = bulkOperations.length / 2;
-          state.errors += count;
-          sessionErrors += count;
+          state.errors += toIndex.length;
+          sessionErrors += toIndex.length;
         }
       }
 
@@ -457,9 +453,8 @@ async function indexFile(filePath: string, batchSize: number, shouldResume: bool
       return;
     }
 
-    // Refresh index
-    console.log('\n\n🔄 Refreshing index...');
-    await client.indices.refresh({ index: INDEX_NAME });
+    // No refresh needed for Redis
+    console.log('\n\n✅ All data persisted to Redis');
 
     // Delete state file on successful completion
     deleteState(stateFile);