From 33b42beb5bbf42e8737a93a5b9d4ffcd9f9d69b8 Mon Sep 17 00:00:00 2001 From: ale Date: Wed, 18 Sep 2024 00:58:34 +0200 Subject: [PATCH 1/4] README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6b48dc0..66e7cc6 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Fediblock Instance [![Build Status](https://drone.manalejandro.com/api/badges/ale/fediblock-instance/status.svg)](https://drone.manalejandro.com/ale/fediblock-instance) +# Fediblock Instance [![Build Status](https://drone.manalejandro.com/api/badges/ale/fediblock-instance/status.svg)](https://drone.manalejandro.com/ale/fediblock-instance) [![Quality Gate Status](https://sonar.manalejandro.com/api/project_badges/measure?project=fediblock-instance&metric=alert_status&token=sqb_748f7401d83ff11dc7f36ec805b1c9a73c41f1e5)](https://sonar.manalejandro.com/dashboard?id=fediblock-instance) ## Another instance to search public blocks and do stats and more, like one FBA but with NodeJS powers. From 8018069c3c9c5a2f77cfdcc0c86fb93aa95bd205 Mon Sep 17 00:00:00 2001 From: ale Date: Wed, 18 Sep 2024 01:57:34 +0200 Subject: [PATCH 2/4] build and quality --- README.md | 2 +- public/index.html | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 66e7cc6..8ee6f59 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Fediblock Instance [![Build Status](https://drone.manalejandro.com/api/badges/ale/fediblock-instance/status.svg)](https://drone.manalejandro.com/ale/fediblock-instance) [![Quality Gate Status](https://sonar.manalejandro.com/api/project_badges/measure?project=fediblock-instance&metric=alert_status&token=sqb_748f7401d83ff11dc7f36ec805b1c9a73c41f1e5)](https://sonar.manalejandro.com/dashboard?id=fediblock-instance) +# Fediblock Instance [![Build Status](https://drone.manalejandro.com/api/badges/ale/fediblock-instance/status.svg)](https://drone.manalejandro.com/ale/fediblock-instance) [![Quality Gate Status](https://sonar.manalejandro.com/api/project_badges/measure?project=ale%3Afediblock-instance&metric=alert_status&token=sqb_b85f22162c314a3c5d781cf356e90ce304ce6a13)](https://sonar.manalejandro.com/dashboard?id=ale%3Afediblock-instance) ## Another instance to search public blocks and do stats and more, like one FBA but with NodeJS powers. diff --git a/public/index.html b/public/index.html index 5c5a238..445b14e 100644 --- a/public/index.html +++ b/public/index.html @@ -30,7 +30,10 @@

Search in public instances - API docs - Git

+ href="https://git.manalejandro.com/ale/fediblock-instance" target="_blank">Git + +
From 77f736950f63b91f578c45edf77171dd5278255e Mon Sep 17 00:00:00 2001 From: ale Date: Fri, 20 Sep 2024 01:41:46 +0200 Subject: [PATCH 3/4] function generator --- lib/fediblock.js | 30 ++++++++---- lib/worker-fediblock.js | 106 ---------------------------------------- 2 files changed, 20 insertions(+), 116 deletions(-) delete mode 100644 lib/worker-fediblock.js diff --git a/lib/fediblock.js b/lib/fediblock.js index 16bcaad..ed0dde9 100644 --- a/lib/fediblock.js +++ b/lib/fediblock.js @@ -127,18 +127,24 @@ module.exports = async (client, apex, app) => { return } }, - scanPart = async (instancesall, index) => { - for (const instance of instancesall) { - try { - app.locals.scan.emit('data', 'data: ' + (app.locals.scannum > 0 ? '(' + app.locals.scannum-- + ':' + (index + 1) + '): ' + instance : ': ' + instance) + '\n\n') - app.locals.peers++ - await scanInstance(instance) - } catch (e) { - console.error(e) + scanPart = async function* (instancesall, index) { + try { + for (const instance of instancesall) { + try { + app.locals.scan.emit('data', 'data: ' + (app.locals.scannum > 0 ? '(' + app.locals.scannum-- + ':' + (index + 1) + '): ' + instance : ': ' + instance) + '\n\n') + app.locals.peers++ + await scanInstance(instance) + global.gc() + } catch (e) { + console.error(e) + } } + } catch (e) { + console.error(e) + yield 1 } global.gc() - return + yield 0 }, scanIndex = async (server, instancesall) => { try { @@ -241,7 +247,11 @@ module.exports = async (client, apex, app) => { const chunk = instancessorted.slice(i, i + chunkSize) parts.push(chunk) } - await Promise.all(parts.map(async (p, i) => await scanPart(p, i))) + await Promise.all(parts.map(async (p, i) => { + for await (const exit of scanPart(p, i)) { + console.log(`Scan finished ${server} - index ${i} - exit ${exit}`) + } + })) return await scanReturn() } else { return await scanReturn() diff --git a/lib/worker-fediblock.js b/lib/worker-fediblock.js deleted file mode 100644 index 65b184e..0000000 --- a/lib/worker-fediblock.js +++ /dev/null @@ -1,106 +0,0 @@ -module.exports = async (client, apex, app) => { - util = require('./util')(apex), - constant = require('./constant'), - { parentPort } = require('worker_threads'), - scanInstance = async instance => { - try { - const json = await util.requestPart(`https://${instance}/api/v1/instance/domain_blocks`) - if (Array.isArray(json) && json.length > 0) { - const result = await client.search({ - index: constant.index, - size: 1, - query: { - term: { - instance: instance - } - } - }), - instancelocated = result.hits && result.hits.hits ? result.hits.hits : [], - blocks = json.map(block => { - if (block.comment && block.comment.length > 8190) { - block.comment = block.comment.slice(0, 8190) - } - return block - }), - [api, nodeinfo, peers] = await Promise.all([ - util.requestPart(`https://${instance}/api/v1/instance`), - util.requestPart(`https://${instance}/nodeinfo/2.0`), - util.requestPart(`https://${instance}/api/v1/instance/peers`) - ]) - if (instancelocated.length === 0) { - await client.index({ - index: constant.index, - body: { - instance, - api, - nodeinfo, - blocks, - peers, - last: new Date() - } - }) - app.locals.created++ - return await util.sendFederatedMessage(constant.nick, 'New Fediblock Instance', `Fediblock Instance ${instance} with ${json.length} blocks - https://${constant.apexdomain}#${instance}`, util.getAccount(api)) - } else { - const elasticinstance = instancelocated[0]._source.blocks || [] - if (Array.isArray(elasticinstance)) { - if (json.length !== elasticinstance.length - || (instancelocated[0]._source.last && instancelocated[0]._source.last < new Date(Date.now() - 2678400000)) - || !instancelocated[0]._source.api - || !instancelocated[0]._source.nodeinfo - || !instancelocated[0]._source.peers) { - await client.update({ - index: constant.index, - id: instancelocated[0]._id, - doc: { - api: api ? api : instancelocated[0]._source.api, - nodeinfo: nodeinfo ? nodeinfo : instancelocated[0]._source.nodeinfo, - blocks: blocks && blocks.length > 0 ? blocks : elasticinstance, - peers: peers && peers.length > 0 ? peers : instancelocated[0]._source.peers, - last: new Date() - }, - doc_as_upsert: true - }) - app.locals.updated++ - if (instancelocated[0]._source.api && instancelocated[0]._source.api.uri && instancelocated[0]._source.api.contact_account && instancelocated[0]._source.api.contact_account.acct) { - const difference = blocks.filter(block => block.domain && block.domain.trim().length > 0 && !elasticinstance.some(instance => block.domain === instance.domain)) - if (difference.length > 0 && difference.length < 50) { - return await util.sendFederatedMessage(constant.nick, 'Detected #Fediblock by Fediblock Instance', `You blocked new instances: ${difference.map(d => d.domain).join(', ')} - https://${constant.apexdomain}#${instance}`, util.getAccount(instancelocated[0]._source.api)) - } else { - return - } - } else { - return - } - } else { - return - } - } else { - return - } - } - } else { - return - } - } catch (e) { - console.error(e) - return - } - }, - scanPart = async (instancesall, index) => { - for (const instance of instancesall) { - try { - app.locals.scan.emit('data', 'data: ' + (app.locals.scannum > 0 ? '(' + app.locals.scannum-- + ':' + (index + 1) + '): ' + instance : ': ' + instance) + '\n\n') - app.locals.peers++ - await scanInstance(instance) - } catch (e) { - console.error(e) - } - } - return - }, - work = (part, index) => { - return parentPort.postMessage({ func: scanPart, args: [part, index] }) - } - -} \ No newline at end of file From 8655727009f9da473c6e172cf675e27062eccf40 Mon Sep 17 00:00:00 2001 From: ale Date: Fri, 20 Sep 2024 02:44:59 +0200 Subject: [PATCH 4/4] Math.ceil --- lib/fediblock.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fediblock.js b/lib/fediblock.js index ed0dde9..f6ddde7 100644 --- a/lib/fediblock.js +++ b/lib/fediblock.js @@ -230,7 +230,7 @@ module.exports = async (client, apex, app) => { instancessorted = instancesall.sort((a, b) => a < b ? -1 : a > b ? 1 : 0), parts = [], split = workers, - chunkSize = Math.floor(instancessorted.length / split) + chunkSize = Math.ceil(instancessorted.length / split) await scanIndex(server, instancesall) if (instancesall.length > 0) { if (!servers || servers.length === 0) {