diff --git a/lib/constant.js b/lib/constant.js index 1d4c595..8a71690 100644 --- a/lib/constant.js +++ b/lib/constant.js @@ -9,6 +9,7 @@ module.exports = { elasticnode: 'http://fediblock-elasticsearch:9200', // elasticsearch connection workers: 3, // async concurrent workers to scan schedule: '5,11,17,23', // UTC hours to publish bot followers federated stats + taskdeletedup: '6', // task hour to delete dups index instances entries filterdomains: ['activitypub-troll.cf'], // domains filtered to scan initialscan: 'mastodon.social', // initial federated domain to scan timeout: 3000, // Timeout of each scan request diff --git a/lib/fediblock.js b/lib/fediblock.js index 8070f25..5c9e56d 100644 --- a/lib/fediblock.js +++ b/lib/fediblock.js @@ -183,10 +183,10 @@ module.exports = async (client, apex, app) => { }, scanReturn = async () => { app.locals.scannum = 0 - if (servers && servers.length > 0) { - return await scan(servers.shift()) + if (servers?.length > 0) { + await scan(servers.shift()) } else { - return await scan(constant.initialscan) + await scan(constant.initialscan) } }, scan = async server => { @@ -223,20 +223,19 @@ module.exports = async (client, apex, app) => { console.log('Done: ' + server + ' - ' + i + ' parts with exit ' + exit + '.') } })) - return await scanReturn() + await scanReturn() } else { - return await scanReturn() + await scanReturn() } } catch (e) { // console.error(e) - return await scanReturn() + await scanReturn() } } else { - return await scanReturn() + await scanReturn() } }, job = schedule.scheduleJob('0 ' + constant.schedule + ' * * *', async () => { - return await util.sendFederatedMessage(constant.nick, null, 'Scanning ' + app.locals.server + ' instance with ' + app.locals.scantotal + ' peers\nScanned ' + app.locals.peers + ' peers from ' + app.locals.instances + ' instances, ' + app.locals.created + ' created, ' + app.locals.updated + ' updated\nhttps://' + constant.apexdomain) + await util.sendFederatedMessage(constant.nick, null, 'Scanning ' + app.locals.server + ' instance with ' + app.locals.scantotal + ' peers\nScanned ' + app.locals.peers + ' peers from ' + app.locals.instances + ' instances, ' + app.locals.created + ' created, ' + app.locals.updated + ' updated\nhttps://' + constant.apexdomain) }) - return await scanReturn() } diff --git a/lib/taskdeletedup.js b/lib/taskdeletedup.js new file mode 100644 index 0000000..4d99abb --- /dev/null +++ b/lib/taskdeletedup.js @@ -0,0 +1,52 @@ +module.exports = client => { + const constant = require('./constant'), + schedule = require('node-schedule'), + size = 500, + deleteDup = async () => { + const count = { deleted: 0, total: 0 } + let lastsort = undefined, result = undefined, last = undefined + do { + result = await client.search({ + index: constant.index, + size: size, + body: { + query: { + match_all: {} + }, + sort: [{ + "instance": { + "order": "asc" + }, + "last": { + "order": "desc", + "numeric_type": "date_nanos", + "format": "strict_date_optional_time_nanos" + } + }], + search_after: lastsort + } + }) + for (const status of result.hits.hits) { + if (last && status._source && last.instance === status._source.instance) { + await client.delete({ index: index, id: status._id }) + count.deleted++ + console.log('deleted ' + status._id + ': ' + status._source.instance) + } + else { + last = status._source + } + if (status._id === result.hits.hits[result.hits.hits.length - 1]._id) { + lastsort = status.sort + } + } + count.total += result.hits.hits.length + if (result.hits.hits.length !== size) { + break + } + } while (result.hits && result.hits.hits && result.hits.hits.length > 0) + return console.log('Index: ' + index + ' - Total: ' + count.total + ' - Deleted: ' + count.deleted) + }, + job = schedule.scheduleJob('0 ' + constant.taskdeletedup + ' * * *', async () => { + await deleteDup() + }) +} diff --git a/server.js b/server.js index 985cb2f..b311c58 100644 --- a/server.js +++ b/server.js @@ -6,6 +6,7 @@ const apexinstance = require('./lib/apex'), logger = require('./lib/logger'), swagger = require('./lib/swagger'), fediblock = require('./lib/fediblock'), + taskdeletedup = require('./lib/taskdeletedup'), constant = require('./lib/constant'), http = require('http'), express = require('express'), @@ -106,9 +107,11 @@ const apexinstance = require('./lib/apex'), } else { apex.systemUser = admin } + const fediblockscan = await fediblock(client, apex, app) + taskdeletedup(client) await apex.startDelivery() + await fediblockscan.scanReturn() console.log('Server listening on ' + server.address().address + ':' + server.address().port) - await fediblock(client, apex, app) } catch (e) { console.error(e) }