let servers module.exports = async (client, apex, app) => { const util = require('./util')(apex), constant = require('./constant'), schedule = require('node-schedule'), workers = constant.workers, getAccount = api => { if (api && api.contact_account.acct) { const acct = api.contact_account.acct.split('@') if (acct.length > 1) { return `https://${acct[1]}/users/${acct[0]}` } else { return `https://${api.uri.replace(/https?:\/\//, '')}/users/${acct.join('')}` } } else { return '' } }, requestPart = async uri => { const ac = new AbortController(), timeout = setTimeout(() => { setImmediate(() => ac.abort()) }, constant.abort_timeout) try { const response = await fetch(uri, { headers: { 'User-Agent': constant.agent }, signal: ac.signal, keepalive: false, timeout: constant.timeout, cache: 'no-cache', mode: 'same-origin', priority: 'high', redirect: 'error' }) setImmediate(() => clearTimeout(timeout)) return await response.json() } catch (e) { setImmediate(() => { clearTimeout(timeout) ac.abort() }) // console.error(e) } }, scanInstance = async instance => { const json = await requestPart(`https://${instance}/api/v1/instance/domain_blocks`) if (Array.isArray(json) && json.length > 0) { const result = await client.search({ index: constant.index, size: 1, query: { term: { instance: instance } } }), instancelocated = result.hits && result.hits.hits ? result.hits.hits : [], blocks = json.map(block => { if (block.comment && block.comment.length > 8190) { block.comment = block.comment.slice(0, 8190) } return block }), [api, nodeinfo, peers] = await Promise.all([ requestPart(`https://${instance}/api/v1/instance`), requestPart(`https://${instance}/nodeinfo/2.0`), requestPart(`https://${instance}/api/v1/instance/peers`) ]) if (instancelocated.length === 0) { await client.index({ index: constant.index, body: { instance, api, nodeinfo, blocks, peers, last: new Date() } }) app.locals.created++ return await util.sendFederatedMessage(constant.nick, 'New Fediblock Instance', `Fediblock Instance ${instance} with ${json.length} blocks - https://${constant.apexdomain}#${instance}`, getAccount(api)) } else { const elasticinstance = instancelocated[0]._source.blocks || [] if (Array.isArray(elasticinstance)) { if (json.length !== elasticinstance.length || (instancelocated[0]._source.last && instancelocated[0]._source.last < new Date(Date.now() - 2678400000)) || !instancelocated[0]._source.api || !instancelocated[0]._source.nodeinfo || !instancelocated[0]._source.peers) { await client.update({ index: constant.index, id: instancelocated[0]._id, doc: { api: api ? api : instancelocated[0]._source.api, nodeinfo: nodeinfo ? nodeinfo : instancelocated[0]._source.nodeinfo, blocks: blocks && blocks.length > 0 ? blocks : elasticinstance, peers: peers && peers.length > 0 ? peers : instancelocated[0]._source.peers, last: new Date() }, doc_as_upsert: true }) app.locals.updated++ if (instancelocated[0]._source.api && instancelocated[0]._source.api.uri && instancelocated[0]._source.api.contact_account && instancelocated[0]._source.api.contact_account.acct) { const difference = blocks.filter(block => block.domain && block.domain.trim().length > 0 && !elasticinstance.some(instance => block.domain === instance.domain)) if (difference.length > 0 && difference.length < 50) { return await util.sendFederatedMessage(constant.nick, 'Detected #Fediblock by Fediblock Instance', `You blocked new instances: ${difference.map(d => d.domain).join(', ')} - https://${constant.apexdomain}#${instance}`, getAccount(instancelocated[0]._source.api)) } } } } } } }, scanPart = async function* (instancesall, index) { try { for (const instance of instancesall) { try { app.locals.scan.emit('data', 'data: ' + (app.locals.scannum > 0 ? '(' + app.locals.scannum-- + ':' + (index + 1) + '): ' + instance : ': ' + instance) + '\n\n') app.locals.peers++ await scanInstance(instance) } catch (e) { console.error('Index ' + index + ':' + instance + ': ' + e) } } yield 0 } catch (e) { console.error(e) yield 1 } }, scanIndex = async (server, instancesall) => { const [api, nodeinfo, blocks] = await Promise.all([ requestPart(`https://${server}/api/v1/instance`), requestPart(`https://${server}/nodeinfo/2.0`), requestPart(`https://${server}/api/v1/instance/domain_blocks`) ]) if (api && typeof api === 'object' && api.version) { const result = await client.search({ index: constant.index, size: 1, query: { term: { instance: server } } }), instancelocated = result.hits && result.hits.hits ? result.hits.hits : [] if (instancelocated.length === 0) { await client.index({ index: constant.index, body: { instance: server, peers: instancesall, api, nodeinfo, blocks: blocks && blocks.length > 0 ? blocks : [], last: new Date() } }) app.locals.created++ } else { const elasticinstance = instancelocated[0]._source.peers || [] if (Array.isArray(elasticinstance) && Array.isArray(instancesall) && instancesall.length > 0) { if (instancesall.length !== elasticinstance.filter(i => !constant.filterdomains.some(d => i.endsWith(d))).length) { await client.update({ index: constant.index, id: instancelocated[0]._id, doc: { peers: instancesall.length > 0 ? instancesall : elasticinstance, api: api ? api : instancelocated[0]._source.api, nodeinfo: nodeinfo ? nodeinfo : instancelocated[0]._source.nodeinfo, blocks: blocks && blocks.length > 0 ? blocks : instancelocated[0]._source.blocks, last: new Date() }, doc_as_upsert: true }) app.locals.updated++ } } } } }, scanReturn = async () => { app.locals.scannum = 0 if (servers?.length > 0) { await scan(servers.shift()) } else { await scan(constant.initialscan) } }, scan = async server => { if (server) { try { console.log(server) app.locals.scan.emit('data', 'data: ' + server + ' peers...\n\n') const instances = await requestPart(`https://${server}/api/v1/instance/peers`), instancesall = Array.isArray(instances) && instances.length > 0 ? instances.filter(i => !constant.filterdomains.some(d => i.endsWith(d))) : [], instancessorted = instancesall.sort((a, b) => a < b ? -1 : a > b ? 1 : 0), parts = [], split = workers, chunkSize = Math.ceil(instancessorted.length / split) await scanIndex(server, instancesall) if (instancesall.length > 0) { if (!servers || servers.length === 0) { servers = instancesall.sort(() => Math.random() - 0.5) } if (server === constant.initialscan) { await scan(servers.shift()) } app.locals.scannum = instancessorted.length app.locals.scantotal = instancessorted.length app.locals.server = server app.locals.instances++ for (let i = 0; i < instancessorted.length; i += chunkSize) { const chunk = instancessorted.slice(i, i + chunkSize) parts.push(chunk) } await Promise.all(parts.map(async (p, i) => { for await (const exit of scanPart(p, i)) { console.log('Done: ' + server + ' - ' + i + ' parts with exit ' + exit + '.') } })) await scanReturn() } else { await scanReturn() } } catch (e) { // console.error(e) await scanReturn() } } else { await scanReturn() } }, job = schedule.scheduleJob('0 ' + constant.schedule + ' * * *', async () => { await util.sendFederatedMessage(constant.nick, null, 'Scanning ' + app.locals.server + ' instance with ' + app.locals.scantotal + ' peers\nScanned ' + app.locals.peers + ' peers from ' + app.locals.instances + ' instances, ' + app.locals.created + ' created, ' + app.locals.updated + ' updated\nhttps://' + constant.apexdomain) }) await scanReturn() }