From 77f736950f63b91f578c45edf77171dd5278255e Mon Sep 17 00:00:00 2001 From: ale Date: Fri, 20 Sep 2024 01:41:46 +0200 Subject: [PATCH] function generator --- lib/fediblock.js | 30 ++++++++---- lib/worker-fediblock.js | 106 ---------------------------------------- 2 files changed, 20 insertions(+), 116 deletions(-) delete mode 100644 lib/worker-fediblock.js diff --git a/lib/fediblock.js b/lib/fediblock.js index 16bcaad..ed0dde9 100644 --- a/lib/fediblock.js +++ b/lib/fediblock.js @@ -127,18 +127,24 @@ module.exports = async (client, apex, app) => { return } }, - scanPart = async (instancesall, index) => { - for (const instance of instancesall) { - try { - app.locals.scan.emit('data', 'data: ' + (app.locals.scannum > 0 ? '(' + app.locals.scannum-- + ':' + (index + 1) + '): ' + instance : ': ' + instance) + '\n\n') - app.locals.peers++ - await scanInstance(instance) - } catch (e) { - console.error(e) + scanPart = async function* (instancesall, index) { + try { + for (const instance of instancesall) { + try { + app.locals.scan.emit('data', 'data: ' + (app.locals.scannum > 0 ? '(' + app.locals.scannum-- + ':' + (index + 1) + '): ' + instance : ': ' + instance) + '\n\n') + app.locals.peers++ + await scanInstance(instance) + global.gc() + } catch (e) { + console.error(e) + } } + } catch (e) { + console.error(e) + yield 1 } global.gc() - return + yield 0 }, scanIndex = async (server, instancesall) => { try { @@ -241,7 +247,11 @@ module.exports = async (client, apex, app) => { const chunk = instancessorted.slice(i, i + chunkSize) parts.push(chunk) } - await Promise.all(parts.map(async (p, i) => await scanPart(p, i))) + await Promise.all(parts.map(async (p, i) => { + for await (const exit of scanPart(p, i)) { + console.log(`Scan finished ${server} - index ${i} - exit ${exit}`) + } + })) return await scanReturn() } else { return await scanReturn() diff --git a/lib/worker-fediblock.js b/lib/worker-fediblock.js deleted file mode 100644 index 65b184e..0000000 --- a/lib/worker-fediblock.js +++ /dev/null @@ -1,106 +0,0 @@ -module.exports = async (client, apex, app) => { - util = require('./util')(apex), - constant = require('./constant'), - { parentPort } = require('worker_threads'), - scanInstance = async instance => { - try { - const json = await util.requestPart(`https://${instance}/api/v1/instance/domain_blocks`) - if (Array.isArray(json) && json.length > 0) { - const result = await client.search({ - index: constant.index, - size: 1, - query: { - term: { - instance: instance - } - } - }), - instancelocated = result.hits && result.hits.hits ? result.hits.hits : [], - blocks = json.map(block => { - if (block.comment && block.comment.length > 8190) { - block.comment = block.comment.slice(0, 8190) - } - return block - }), - [api, nodeinfo, peers] = await Promise.all([ - util.requestPart(`https://${instance}/api/v1/instance`), - util.requestPart(`https://${instance}/nodeinfo/2.0`), - util.requestPart(`https://${instance}/api/v1/instance/peers`) - ]) - if (instancelocated.length === 0) { - await client.index({ - index: constant.index, - body: { - instance, - api, - nodeinfo, - blocks, - peers, - last: new Date() - } - }) - app.locals.created++ - return await util.sendFederatedMessage(constant.nick, 'New Fediblock Instance', `Fediblock Instance ${instance} with ${json.length} blocks - https://${constant.apexdomain}#${instance}`, util.getAccount(api)) - } else { - const elasticinstance = instancelocated[0]._source.blocks || [] - if (Array.isArray(elasticinstance)) { - if (json.length !== elasticinstance.length - || (instancelocated[0]._source.last && instancelocated[0]._source.last < new Date(Date.now() - 2678400000)) - || !instancelocated[0]._source.api - || !instancelocated[0]._source.nodeinfo - || !instancelocated[0]._source.peers) { - await client.update({ - index: constant.index, - id: instancelocated[0]._id, - doc: { - api: api ? api : instancelocated[0]._source.api, - nodeinfo: nodeinfo ? nodeinfo : instancelocated[0]._source.nodeinfo, - blocks: blocks && blocks.length > 0 ? blocks : elasticinstance, - peers: peers && peers.length > 0 ? peers : instancelocated[0]._source.peers, - last: new Date() - }, - doc_as_upsert: true - }) - app.locals.updated++ - if (instancelocated[0]._source.api && instancelocated[0]._source.api.uri && instancelocated[0]._source.api.contact_account && instancelocated[0]._source.api.contact_account.acct) { - const difference = blocks.filter(block => block.domain && block.domain.trim().length > 0 && !elasticinstance.some(instance => block.domain === instance.domain)) - if (difference.length > 0 && difference.length < 50) { - return await util.sendFederatedMessage(constant.nick, 'Detected #Fediblock by Fediblock Instance', `You blocked new instances: ${difference.map(d => d.domain).join(', ')} - https://${constant.apexdomain}#${instance}`, util.getAccount(instancelocated[0]._source.api)) - } else { - return - } - } else { - return - } - } else { - return - } - } else { - return - } - } - } else { - return - } - } catch (e) { - console.error(e) - return - } - }, - scanPart = async (instancesall, index) => { - for (const instance of instancesall) { - try { - app.locals.scan.emit('data', 'data: ' + (app.locals.scannum > 0 ? '(' + app.locals.scannum-- + ':' + (index + 1) + '): ' + instance : ': ' + instance) + '\n\n') - app.locals.peers++ - await scanInstance(instance) - } catch (e) { - console.error(e) - } - } - return - }, - work = (part, index) => { - return parentPort.postMessage({ func: scanPart, args: [part, index] }) - } - -} \ No newline at end of file