redis migration

Signed-off-by: ale <ale@manalejandro.com>

scripts/remove-duplicates.ts

@@ -3,7 +3,7 @@
 /**
  * Hasher Duplicate Remover Script
  * 
- * This script finds and removes duplicate entries from the Elasticsearch index.
+ * This script finds and removes duplicate entries from Redis.
  * It identifies duplicates by checking plaintext, md5, sha1, sha256, and sha512 fields.
  * 
  * Usage:
@@ -13,17 +13,28 @@
  * Options:
  *   --dry-run              Show duplicates without removing them (default)
  *   --execute              Actually remove the duplicates
- *   --batch-size=<number>  Number of items to process in each batch (default: 1000)
- *   --field=<field>        Check duplicates only on this field (plaintext, md5, sha1, sha256, sha512)
+ *   --batch-size=<number>  Number of keys to scan in each batch (default: 1000)
+ *   --field=<field>        Check duplicates only on this field (md5, sha1, sha256, sha512)
  *   --help, -h             Show this help message
  */
 
-import { Client } from '@elastic/elasticsearch';
+import Redis from 'ioredis';
 
-const ELASTICSEARCH_NODE = process.env.ELASTICSEARCH_NODE || 'http://localhost:9200';
-const INDEX_NAME = 'hasher';
+const REDIS_HOST = process.env.REDIS_HOST || 'localhost';
+const REDIS_PORT = parseInt(process.env.REDIS_PORT || '6379', 10);
+const REDIS_PASSWORD = process.env.REDIS_PASSWORD || undefined;
+const REDIS_DB = parseInt(process.env.REDIS_DB || '0', 10);
 const DEFAULT_BATCH_SIZE = 1000;
 
+interface HashDocument {
+  plaintext: string;
+  md5: string;
+  sha1: string;
+  sha256: string;
+  sha512: string;
+  created_at: string;
+}
+
 interface ParsedArgs {
   dryRun: boolean;
   batchSize: number;
@@ -34,9 +45,9 @@ interface ParsedArgs {
 interface DuplicateGroup {
   value: string;
   field: string;
-  documentIds: string[];
-  keepId: string;
-  deleteIds: string[];
+  plaintexts: string[];
+  keepPlaintext: string;
+  deletePlaintexts: string[];
 }
 
 function parseArgs(args: string[]): ParsedArgs {
@@ -96,302 +107,244 @@ Usage:
 Options:
   --dry-run              Show duplicates without removing them (default)
   --execute              Actually remove the duplicates
-  --batch-size=<number>  Number of items to process in each batch (default: 1000)
+  --batch-size=<number>  Number of keys to scan in each batch (default: 1000)
   --field=<field>        Check duplicates only on this field
-                         Valid fields: plaintext, md5, sha1, sha256, sha512
+                         Valid fields: md5, sha1, sha256, sha512
   --help, -h             Show this help message
 
 Environment Variables:
-  ELASTICSEARCH_NODE     Elasticsearch node URL (default: http://localhost:9200)
+  REDIS_HOST             Redis host (default: localhost)
+  REDIS_PORT             Redis port (default: 6379)
+  REDIS_PASSWORD         Redis password (optional)
+  REDIS_DB               Redis database number (default: 0)
 
 Examples:
-  npx tsx scripts/remove-duplicates.ts                    # Dry run, show all duplicates
-  npx tsx scripts/remove-duplicates.ts --execute          # Remove all duplicates
-  npx tsx scripts/remove-duplicates.ts --field=md5        # Check only md5 duplicates
-  npx tsx scripts/remove-duplicates.ts --execute --field=plaintext
+  # Dry run (show duplicates only)
+  npm run remove-duplicates
 
-Notes:
-  - The script keeps the OLDEST document (by created_at) and removes newer duplicates
-  - Always run with --dry-run first to review what will be deleted
-  - Duplicates are checked across all hash fields by default
+  # Actually remove duplicates
+  npm run remove-duplicates -- --execute
+
+  # Check only MD5 duplicates
+  npm run remove-duplicates -- --field=md5 --execute
+
+Description:
+  This script scans through all hash documents in Redis and identifies
+  duplicates based on hash values. When duplicates are found, it keeps
+  the oldest entry (by created_at) and marks the rest for deletion.
 `);
-  process.exit(0);
 }
 
 async function findDuplicatesForField(
-  client: Client, 
-  field: string, 
+  client: Redis,
+  field: 'md5' | 'sha1' | 'sha256' | 'sha512',
   batchSize: number
 ): Promise<DuplicateGroup[]> {
-  const duplicates: DuplicateGroup[] = [];
-  
-  // Use aggregation to find duplicate values
-  const fieldToAggregate = field === 'plaintext' ? 'plaintext.keyword' : field;
-  
-  // Use composite aggregation to handle large number of duplicates
-  let afterKey: any = undefined;
-  let hasMore = true;
-  
-  console.log(`   Scanning for duplicates...`);
-  
-  while (hasMore) {
-    const aggQuery: any = {
-      index: INDEX_NAME,
-      size: 0,
-      aggs: {
-        duplicates: {
-          composite: {
-            size: batchSize,
-            sources: [
-              { value: { terms: { field: fieldToAggregate } } }
-            ],
-            ...(afterKey && { after: afterKey })
-          },
-          aggs: {
-            doc_count_filter: {
-              bucket_selector: {
-                buckets_path: { count: '_count' },
-                script: 'params.count > 1'
-              }
-            }
-          }
-        }
-      }
-    };
+  const pattern = `hash:index:${field}:*`;
+  const hashToPlaintexts: Map<string, string[]> = new Map();
 
-    const response = await client.search(aggQuery);
-    const compositeAgg = response.aggregations?.duplicates as any;
-    const buckets = compositeAgg?.buckets || [];
+  console.log(`🔍 Scanning ${field} indexes...`);
 
-    for (const bucket of buckets) {
-      if (bucket.doc_count > 1) {
-        const value = bucket.key.value;
-        
-        // Use scroll API for large result sets
-        const documentIds: string[] = [];
-        
-        let scrollResponse = await client.search({
-          index: INDEX_NAME,
-          scroll: '1m',
-          size: 1000,
-          query: {
-            term: {
-              [fieldToAggregate]: value
-            }
-          },
-          sort: [
-            { created_at: { order: 'asc' } }
-          ],
-          _source: false
-        });
+  let cursor = '0';
+  let keysScanned = 0;
 
-        while (scrollResponse.hits.hits.length > 0) {
-          documentIds.push(...scrollResponse.hits.hits.map((hit: any) => hit._id));
-          
-          if (!scrollResponse._scroll_id) break;
-          
-          scrollResponse = await client.scroll({
-            scroll_id: scrollResponse._scroll_id,
-            scroll: '1m'
-          });
-        }
+  do {
+    const [nextCursor, keys] = await client.scan(cursor, 'MATCH', pattern, 'COUNT', batchSize);
+    cursor = nextCursor;
+    keysScanned += keys.length;
 
-        // Clear scroll
-        if (scrollResponse._scroll_id) {
-          await client.clearScroll({ scroll_id: scrollResponse._scroll_id }).catch(() => {});
-        }
-        
-        if (documentIds.length > 1) {
-          duplicates.push({
-            value: String(value),
-            field,
-            documentIds,
-            keepId: documentIds[0], // Keep the oldest
-            deleteIds: documentIds.slice(1) // Delete the rest
-          });
+    for (const key of keys) {
+      const hash = key.replace(`hash:index:${field}:`, '');
+      const plaintext = await client.get(key);
+      
+      if (plaintext) {
+        if (!hashToPlaintexts.has(hash)) {
+          hashToPlaintexts.set(hash, []);
         }
+        hashToPlaintexts.get(hash)!.push(plaintext);
       }
     }
 
-    // Check if there are more results
-    afterKey = compositeAgg?.after_key;
-    hasMore = buckets.length === batchSize && afterKey;
-    
-    if (hasMore) {
-      process.stdout.write(`\r   Found ${duplicates.length} duplicate groups so far...`);
+    process.stdout.write(`\r   Keys scanned: ${keysScanned}   `);
+  } while (cursor !== '0');
+
+  console.log('');
+
+  const duplicates: DuplicateGroup[] = [];
+
+  for (const [hash, plaintexts] of hashToPlaintexts.entries()) {
+    if (plaintexts.length > 1) {
+      // Fetch documents to get created_at timestamps
+      const docs = await Promise.all(
+        plaintexts.map(async (pt) => {
+          const data = await client.get(`hash:plaintext:${pt}`);
+          return data ? JSON.parse(data) as HashDocument : null;
+        })
+      );
+
+      const validDocs = docs.filter((doc): doc is HashDocument => doc !== null);
+      
+      if (validDocs.length > 1) {
+        // Sort by created_at, keep oldest
+        validDocs.sort((a, b) => a.created_at.localeCompare(b.created_at));
+        
+        duplicates.push({
+          value: hash,
+          field,
+          plaintexts: validDocs.map(d => d.plaintext),
+          keepPlaintext: validDocs[0].plaintext,
+          deletePlaintexts: validDocs.slice(1).map(d => d.plaintext)
+        });
+      }
     }
   }
 
   return duplicates;
 }
 
-async function removeDuplicates(parsedArgs: ParsedArgs) {
-  const client = new Client({ node: ELASTICSEARCH_NODE });
-  const fields = parsedArgs.field 
-    ? [parsedArgs.field] 
-    : ['plaintext', 'md5', 'sha1', 'sha256', 'sha512'];
+async function removeDuplicates(
+  client: Redis,
+  duplicates: DuplicateGroup[],
+  dryRun: boolean
+): Promise<{ deleted: number; errors: number }> {
+  let deleted = 0;
+  let errors = 0;
 
-  console.log(`🔍 Hasher Duplicate Remover`);
-  console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━`);
-  console.log(`Elasticsearch: ${ELASTICSEARCH_NODE}`);
-  console.log(`Index: ${INDEX_NAME}`);
-  console.log(`Mode: ${parsedArgs.dryRun ? '🔎 DRY RUN (no changes)' : '⚠️  EXECUTE (will delete)'}`);
-  console.log(`Batch size: ${parsedArgs.batchSize}`);
-  console.log(`Fields to check: ${fields.join(', ')}`);
+  console.log('');
+  console.log(`${dryRun ? '🔍 DRY RUN - Would delete:' : '🗑️  Deleting duplicates...'}`);
+  console.log('');
+
+  for (const dup of duplicates) {
+    console.log(`Duplicate ${dup.field}: ${dup.value}`);
+    console.log(`  Keep: ${dup.keepPlaintext} (oldest)`);
+    console.log(`  Delete: ${dup.deletePlaintexts.join(', ')}`);
+
+    if (!dryRun) {
+      for (const plaintext of dup.deletePlaintexts) {
+        try {
+          const docKey = `hash:plaintext:${plaintext}`;
+          const docData = await client.get(docKey);
+          
+          if (docData) {
+            const doc: HashDocument = JSON.parse(docData);
+            const pipeline = client.pipeline();
+
+            // Delete the main document
+            pipeline.del(docKey);
+            
+            // Delete all indexes
+            pipeline.del(`hash:index:md5:${doc.md5}`);
+            pipeline.del(`hash:index:sha1:${doc.sha1}`);
+            pipeline.del(`hash:index:sha256:${doc.sha256}`);
+            pipeline.del(`hash:index:sha512:${doc.sha512}`);
+            
+            // Update statistics
+            pipeline.hincrby('hash:stats', 'count', -1);
+            pipeline.hincrby('hash:stats', 'size', -JSON.stringify(doc).length);
+            
+            const results = await pipeline.exec();
+            
+            if (results && results.some(([err]) => err !== null)) {
+              errors++;
+            } else {
+              deleted++;
+            }
+          }
+        } catch (error) {
+          console.error(`  Error deleting ${plaintext}:`, error);
+          errors++;
+        }
+      }
+    }
+    console.log('');
+  }
+
+  return { deleted, errors };
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  const parsed = parseArgs(args);
+
+  if (parsed.showHelp) {
+    showHelp();
+    process.exit(0);
+  }
+
+  const validFields: Array<'md5' | 'sha1' | 'sha256' | 'sha512'> = ['md5', 'sha1', 'sha256', 'sha512'];
+  const fieldsToCheck = parsed.field 
+    ? [parsed.field as 'md5' | 'sha1' | 'sha256' | 'sha512']
+    : validFields;
+
+  // Validate field
+  if (parsed.field && !validFields.includes(parsed.field as any)) {
+    console.error(`❌ Invalid field: ${parsed.field}`);
+    console.error(`   Valid fields: ${validFields.join(', ')}`);
+    process.exit(1);
+  }
+
+  const client = new Redis({
+    host: REDIS_HOST,
+    port: REDIS_PORT,
+    password: REDIS_PASSWORD,
+    db: REDIS_DB,
+  });
+
+  console.log('');
+  console.log('🔍 Hasher Duplicate Remover');
+  console.log('━'.repeat(42));
+  console.log(`Redis: ${REDIS_HOST}:${REDIS_PORT}`);
+  console.log(`Mode: ${parsed.dryRun ? 'DRY RUN' : 'EXECUTE'}`);
+  console.log(`Batch size: ${parsed.batchSize}`);
+  console.log(`Fields to check: ${fieldsToCheck.join(', ')}`);
   console.log('');
 
   try {
-    // Test connection
-    console.log('🔗 Connecting to Elasticsearch...');
-    await client.cluster.health({});
+    console.log('🔗 Connecting to Redis...');
+    await client.ping();
     console.log('✅ Connected successfully\n');
 
-    // Get index stats
-    const countResponse = await client.count({ index: INDEX_NAME });
-    console.log(`📊 Total documents in index: ${countResponse.count}\n`);
-
     const allDuplicates: DuplicateGroup[] = [];
-    const seenDeleteIds = new Set<string>();
 
-    // Find duplicates for each field
-    for (const field of fields) {
-      console.log(`🔍 Checking duplicates for field: ${field}...`);
-      const fieldDuplicates = await findDuplicatesForField(client, field, parsedArgs.batchSize);
-      
-      // Filter out already seen delete IDs to avoid counting the same document multiple times
-      for (const dup of fieldDuplicates) {
-        const newDeleteIds = dup.deleteIds.filter(id => !seenDeleteIds.has(id));
-        if (newDeleteIds.length > 0) {
-          dup.deleteIds = newDeleteIds;
-          newDeleteIds.forEach(id => seenDeleteIds.add(id));
-          allDuplicates.push(dup);
-        }
-      }
-      
-      console.log(`   Found ${fieldDuplicates.length} duplicate groups for ${field}`);
+    for (const field of fieldsToCheck) {
+      const duplicates = await findDuplicatesForField(client, field, parsed.batchSize);
+      allDuplicates.push(...duplicates);
+      console.log(`   Found ${duplicates.length} duplicate groups for ${field}`);
     }
 
-    const totalToDelete = allDuplicates.reduce((sum, dup) => sum + dup.deleteIds.length, 0);
-
-    console.log(`\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━`);
-    console.log(`📋 Summary:`);
-    console.log(`   Duplicate groups found: ${allDuplicates.length}`);
-    console.log(`   Documents to delete: ${totalToDelete}`);
-    console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n`);
+    console.log('');
+    console.log(`📊 Total duplicate groups found: ${allDuplicates.length}`);
 
     if (allDuplicates.length === 0) {
-      console.log('✨ No duplicates found! Index is clean.\n');
-      return;
-    }
+      console.log('✅ No duplicates found!');
+    } else {
+      const totalToDelete = allDuplicates.reduce(
+        (sum, dup) => sum + dup.deletePlaintexts.length,
+        0
+      );
+      console.log(`   Total documents to delete: ${totalToDelete}`);
 
-    // Show sample of duplicates
-    console.log(`📝 Sample duplicates (showing first 10):\n`);
-    const samplesToShow = allDuplicates.slice(0, 10);
-    for (const dup of samplesToShow) {
-      const truncatedValue = dup.value.length > 50 
-        ? dup.value.substring(0, 50) + '...' 
-        : dup.value;
-      console.log(`   Field: ${dup.field}`);
-      console.log(`   Value: ${truncatedValue}`);
-      console.log(`   Keep: ${dup.keepId}`);
-      console.log(`   Delete: ${dup.deleteIds.length} document(s)`);
-      console.log('');
-    }
+      const { deleted, errors } = await removeDuplicates(client, allDuplicates, parsed.dryRun);
 
-    if (allDuplicates.length > 10) {
-      console.log(`   ... and ${allDuplicates.length - 10} more duplicate groups\n`);
-    }
-
-    if (parsedArgs.dryRun) {
-      console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━`);
-      console.log(`🔎 DRY RUN - No changes made`);
-      console.log(`   Run with --execute to remove ${totalToDelete} duplicate documents`);
-      console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n`);
-      return;
-    }
-
-    // Execute deletion
-    console.log(`\n🗑️  Removing ${totalToDelete} duplicate documents...\n`);
-
-    let deleted = 0;
-    let errors = 0;
-    const deleteIds = allDuplicates.flatMap(dup => dup.deleteIds);
-
-    // Delete in batches
-    for (let i = 0; i < deleteIds.length; i += parsedArgs.batchSize) {
-      const batch = deleteIds.slice(i, i + parsedArgs.batchSize);
-      
-      try {
-        const bulkOperations = batch.flatMap(id => [
-          { delete: { _index: INDEX_NAME, _id: id } }
-        ]);
-
-        const bulkResponse = await client.bulk({
-          operations: bulkOperations,
-          refresh: false
-        });
-
-        if (bulkResponse.errors) {
-          const errorCount = bulkResponse.items.filter((item: any) => item.delete?.error).length;
-          errors += errorCount;
-          deleted += batch.length - errorCount;
-        } else {
-          deleted += batch.length;
-        }
-
-        process.stdout.write(`\r⏳ Progress: ${Math.min(i + parsedArgs.batchSize, deleteIds.length)}/${deleteIds.length} - Deleted: ${deleted}, Errors: ${errors}`);
-      } catch (error) {
-        console.error(`\n❌ Error deleting batch:`, error);
-        errors += batch.length;
+      if (!parsed.dryRun) {
+        console.log('━'.repeat(42));
+        console.log('✅ Removal complete!');
+        console.log('');
+        console.log('📊 Statistics:');
+        console.log(`   Deleted: ${deleted}`);
+        console.log(`   Errors: ${errors}`);
+      } else {
+        console.log('━'.repeat(42));
+        console.log('💡 This was a dry run. Use --execute to actually remove duplicates.');
       }
     }
 
-    // Refresh index
-    console.log('\n\n🔄 Refreshing index...');
-    await client.indices.refresh({ index: INDEX_NAME });
-
-    // Get new count
-    const newCountResponse = await client.count({ index: INDEX_NAME });
-
-    console.log('\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
-    console.log('✅ Duplicate removal complete!');
-    console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━`);
-    console.log(`Documents deleted: ${deleted}`);
-    console.log(`Errors: ${errors}`);
-    console.log(`Previous document count: ${countResponse.count}`);
-    console.log(`New document count: ${newCountResponse.count}`);
-    console.log('');
-
+    await client.quit();
   } catch (error) {
-    console.error('\n❌ Error:', error instanceof Error ? error.message : error);
+    console.error('\n\n❌ Error:', error);
+    await client.quit();
     process.exit(1);
   }
 }
 
-// Parse command line arguments
-const args = process.argv.slice(2);
-const parsedArgs = parseArgs(args);
-
-if (parsedArgs.showHelp) {
-  showHelp();
-}
-
-// Validate field if provided
-const validFields = ['plaintext', 'md5', 'sha1', 'sha256', 'sha512'];
-if (parsedArgs.field && !validFields.includes(parsedArgs.field)) {
-  console.error(`❌ Invalid field: ${parsedArgs.field}`);
-  console.error(`   Valid fields: ${validFields.join(', ')}`);
-  process.exit(1);
-}
-
-console.log(`\n🔧 Configuration:`);
-console.log(`   Mode: ${parsedArgs.dryRun ? 'dry-run' : 'execute'}`);
-console.log(`   Batch size: ${parsedArgs.batchSize}`);
-if (parsedArgs.field) {
-  console.log(`   Field: ${parsedArgs.field}`);
-} else {
-  console.log(`   Fields: all (plaintext, md5, sha1, sha256, sha512)`);
-}
-console.log('');
-
-removeDuplicates(parsedArgs).catch(console.error);
+main();