refactor fediblock and taskdeletedup

lib/constant.js

@@ -9,6 +9,7 @@ module.exports = {
     elasticnode: 'http://fediblock-elasticsearch:9200', // elasticsearch connection
     workers: 3, // async concurrent workers to scan
     schedule: '5,11,17,23', // UTC hours to publish bot followers federated stats
+    taskdeletedup: '6', // task hour to delete dups index instances entries
     filterdomains: ['activitypub-troll.cf'], // domains filtered to scan
     initialscan: 'mastodon.social', // initial federated domain to scan
     timeout: 3000, // Timeout of each scan request

lib/fediblock.js

@@ -183,10 +183,10 @@ module.exports = async (client, apex, app) => {
 		},
 		scanReturn = async () => {
 			app.locals.scannum = 0
-			if (servers && servers.length > 0) {
-				return await scan(servers.shift())
+			if (servers?.length > 0) {
+				await scan(servers.shift())
 			} else {
-				return await scan(constant.initialscan)
+				await scan(constant.initialscan)
 			}
 		},
 		scan = async server => {
@@ -223,20 +223,19 @@ module.exports = async (client, apex, app) => {
 								console.log('Done: ' + server + ' - ' + i + ' parts with exit ' + exit + '.')
 							}
 						}))
-						return await scanReturn()
+						await scanReturn()
 					} else {
-						return await scanReturn()
+						await scanReturn()
 					}
 				} catch (e) {
 					// console.error(e)
-					return await scanReturn()
+					await scanReturn()
 				}
 			} else {
-				return await scanReturn()
+				await scanReturn()
 			}
 		},
 		job = schedule.scheduleJob('0 ' + constant.schedule + ' * * *', async () => {
-			return await util.sendFederatedMessage(constant.nick, null, 'Scanning ' + app.locals.server + ' instance with ' + app.locals.scantotal + ' peers\nScanned ' + app.locals.peers + ' peers from ' + app.locals.instances + ' instances, ' + app.locals.created + ' created, ' + app.locals.updated + ' updated\nhttps://' + constant.apexdomain)
+			await util.sendFederatedMessage(constant.nick, null, 'Scanning ' + app.locals.server + ' instance with ' + app.locals.scantotal + ' peers\nScanned ' + app.locals.peers + ' peers from ' + app.locals.instances + ' instances, ' + app.locals.created + ' created, ' + app.locals.updated + ' updated\nhttps://' + constant.apexdomain)
 		})
-	return await scanReturn()
 }

lib/taskdeletedup.js

@@ -0,0 +1,52 @@
+module.exports = client => {
+    const constant = require('./constant'),
+        schedule = require('node-schedule'),
+        size = 500,
+        deleteDup = async () => {
+            const count = { deleted: 0, total: 0 }
+            let lastsort = undefined, result = undefined, last = undefined
+            do {
+                result = await client.search({
+                    index: constant.index,
+                    size: size,
+                    body: {
+                        query: {
+                            match_all: {}
+                        },
+                        sort: [{
+                            "instance": {
+                                "order": "asc"
+                            },
+                            "last": {
+                                "order": "desc",
+                                "numeric_type": "date_nanos",
+                                "format": "strict_date_optional_time_nanos"
+                            }
+                        }],
+                        search_after: lastsort
+                    }
+                })
+                for (const status of result.hits.hits) {
+                    if (last && status._source && last.instance === status._source.instance) {
+                        await client.delete({ index: index, id: status._id })
+                        count.deleted++
+                        console.log('deleted ' + status._id + ': ' + status._source.instance)
+                    }
+                    else {
+                        last = status._source
+                    }
+                    if (status._id === result.hits.hits[result.hits.hits.length - 1]._id) {
+                        lastsort = status.sort
+                    }
+                }
+                count.total += result.hits.hits.length
+                if (result.hits.hits.length !== size) {
+                    break
+                }
+            } while (result.hits && result.hits.hits && result.hits.hits.length > 0)
+            return console.log('Index: ' + index + ' - Total: ' + count.total + ' - Deleted: ' + count.deleted)
+        },
+        job = schedule.scheduleJob('0 ' + constant.taskdeletedup + ' * * *', async () => {
+            await deleteDup()
+        })
+}

server.js

@@ -6,6 +6,7 @@ const apexinstance = require('./lib/apex'),
     logger = require('./lib/logger'),
     swagger = require('./lib/swagger'),
     fediblock = require('./lib/fediblock'),
+    taskdeletedup = require('./lib/taskdeletedup'),
     constant = require('./lib/constant'),
     http = require('http'),
     express = require('express'),
@@ -106,9 +107,11 @@ const apexinstance = require('./lib/apex'),
                     } else {
                         apex.systemUser = admin
                     }
+                    const fediblockscan = await fediblock(client, apex, app)
+                    taskdeletedup(client)
                     await apex.startDelivery()
+                    await fediblockscan.scanReturn()
                     console.log('Server listening on ' + server.address().address + ':' + server.address().port)
-                    await fediblock(client, apex, app)
                 } catch (e) {
                     console.error(e)
                 }