fediblock-instance/lib/fediblock.js

let servers
module.exports = async (client, apex, app) => {
	const util = require('./util')(apex),
		constant = require('./constant'),
		https = require('https'),
		schedule = require('node-schedule'),
		workers = constant.workers,
		getAccount = api => {
			if (api && api.contact_account.acct) {
				const acct = api.contact_account.acct.split('@')
				if (acct.length > 1) {
					return `https://${acct[1]}/users/${acct[0]}`
				} else {
					return `https://${api.uri.replace(/https?:\/\//, '')}/users/${acct.join('')}`
				}
			} else {
				return ''
			}
		},
		requestPart = async uri => {
			const ac = new AbortController()
			try {
				setTimeout(() => {
					ac.abort()
				}, 5000)
				const response = await fetch(uri, {
					headers: { 'User-Agent': constant.agent },
					agent: new https.Agent({
						rejectUnauthorized: false,
						keepAlive: false
					}),
					signal: ac.signal,
					keepalive: false,
					timeout: 4000
				}),
					json = await response.json()
				setImmediate(() => { ac.abort() })
				return json
			} catch (e) {
				setImmediate(() => { ac.abort() })
				// console.error(e)
				return
			}
		},
		scanInstance = async instance => {
			try {
				const json = await requestPart(`https://${instance}/api/v1/instance/domain_blocks`)
				if (Array.isArray(json) && json.length > 0) {
					const result = await client.search({
						index: constant.index,
						size: 1,
						query: {
							term: {
								instance: instance
							}
						}
					}),
						instancelocated = result.hits && result.hits.hits ? result.hits.hits : [],
						blocks = json.map(block => {
							if (block.comment && block.comment.length > 8190) {
								block.comment = block.comment.slice(0, 8190)
							}
							return block
						}),
						[api, nodeinfo, peers] = await Promise.all([
							requestPart(`https://${instance}/api/v1/instance`),
							requestPart(`https://${instance}/nodeinfo/2.0`),
							requestPart(`https://${instance}/api/v1/instance/peers`)
						])
					if (instancelocated.length === 0) {
						try {
							await client.index({
								index: constant.index,
								body: {
									instance,
									api,
									nodeinfo,
									blocks,
									peers,
									last: new Date()
								}
							})
						} catch (e) {
							console.error(e)
							return
						}
						app.locals.created++
						return await util.sendFederatedMessage(constant.nick, 'New Fediblock Instance', `Fediblock Instance ${instance} with ${json.length} blocks - https://${constant.apexdomain}#${instance}`, getAccount(api))
					} else {
						const elasticinstance = instancelocated[0]._source.blocks || []
						if (Array.isArray(elasticinstance)) {
							if (json.length !== elasticinstance.length
								|| (instancelocated[0]._source.last && instancelocated[0]._source.last < new Date(Date.now() - 2678400000))
								|| !instancelocated[0]._source.api
								|| !instancelocated[0]._source.nodeinfo
								|| !instancelocated[0]._source.peers) {
								try {
									await client.update({
										index: constant.index,
										id: instancelocated[0]._id,
										doc: {
											api: api ? api : instancelocated[0]._source.api,
											nodeinfo: nodeinfo ? nodeinfo : instancelocated[0]._source.nodeinfo,
											blocks: blocks && blocks.length > 0 ? blocks : elasticinstance,
											peers: peers && peers.length > 0 ? peers : instancelocated[0]._source.peers,
											last: new Date()
										},
										doc_as_upsert: true
									})
								} catch (e) {
									console.error(e)
									return
								}
								app.locals.updated++
								if (instancelocated[0]._source.api && instancelocated[0]._source.api.uri && instancelocated[0]._source.api.contact_account && instancelocated[0]._source.api.contact_account.acct) {
									const difference = blocks.filter(block => block.domain && block.domain.trim().length > 0 && !elasticinstance.some(instance => block.domain === instance.domain))
									if (difference.length > 0 && difference.length < 50) {
										return await util.sendFederatedMessage(constant.nick, 'Detected #Fediblock by Fediblock Instance', `You blocked new instances: ${difference.map(d => d.domain).join(', ')} - https://${constant.apexdomain}#${instance}`, getAccount(instancelocated[0]._source.api))
									} else {
										return
									}
								} else {
									return
								}
							} else {
								return
							}
						} else {
							return
						}
					}
				} else {
					return
				}
			} catch (e) {
				console.error(e)
				return
			}
		},
		scanPart = async function* (instancesall, index) {
			try {
				for (const instance of instancesall) {
					try {
						app.locals.scan.emit('data', 'data: ' + (app.locals.scannum > 0 ? '(' + app.locals.scannum-- + ':' + (index + 1) + '): ' + instance : ': ' + instance) + '\n\n')
						app.locals.peers++
						await scanInstance(instance)
					} catch (e) {
						console.error(e)
					}
				}
				global.gc()
				yield 0
			} catch (e) {
				global.gc()
				console.error(e)
				yield 1
			}
		},
		scanIndex = async (server, instancesall) => {
			try {
				const [api, nodeinfo, blocks] = await Promise.all([
					requestPart(`https://${server}/api/v1/instance`),
					requestPart(`https://${server}/nodeinfo/2.0`),
					requestPart(`https://${server}/api/v1/instance/domain_blocks`)
				])
				if (api && typeof api === 'object' && api.version) {
					const result = await client.search({
						index: constant.index,
						size: 1,
						query: {
							term: {
								instance: server
							}
						}
					}),
						instancelocated = result.hits && result.hits.hits ? result.hits.hits : []
					if (instancelocated.length === 0) {
						await client.index({
							index: constant.index,
							body: {
								instance: server,
								peers: instancesall,
								api,
								nodeinfo,
								blocks: blocks && blocks.length > 0 ? blocks : [],
								last: new Date()
							}
						})
						app.locals.created++
					} else {
						const elasticinstance = instancelocated[0]._source.peers || []
						if (Array.isArray(elasticinstance) && Array.isArray(instancesall) && instancesall.length > 0) {
							if (instancesall.length !== elasticinstance.filter(i => !constant.filterdomains.some(d => i.endsWith(d))).length) {
								await client.update({
									index: constant.index,
									id: instancelocated[0]._id,
									doc: {
										peers: instancesall.length > 0 ? instancesall : elasticinstance,
										api: api ? api : instancelocated[0]._source.api,
										nodeinfo: nodeinfo ? nodeinfo : instancelocated[0]._source.nodeinfo,
										blocks: blocks && blocks.length > 0 ? blocks : instancelocated[0]._source.blocks,
										last: new Date()
									},
									doc_as_upsert: true
								})
								app.locals.updated++
							} else {
								return
							}
						} else {
							return
						}
					}
				} else {
					return
				}
			} catch (e) {
				console.error(e)
				return
			}
		},
		scanReturn = async () => {
			global.gc()
			app.locals.scannum = 0
			if (servers && servers.length > 0) {
				return await scan(servers.shift())
			} else {
				return await scan(constant.initialscan)
			}
		},
		scan = async server => {
			if (server) {
				try {
					console.log(server)
					app.locals.scan.emit('data', 'data:  ' + server + ' peers...\n\n')
					const instances = await requestPart(`https://${server}/api/v1/instance/peers`),
						instancesall = Array.isArray(instances) && instances.length > 0
							? instances.filter(i => !constant.filterdomains.some(d => i.endsWith(d)))
							: [],
						instancessorted = instancesall.sort((a, b) => a < b ? -1 : a > b ? 1 : 0),
						parts = [],
						split = workers,
						chunkSize = Math.ceil(instancessorted.length / split)
					await scanIndex(server, instancesall)
					if (instancesall.length > 0) {
						if (!servers || servers.length === 0) {
							servers = instancesall.sort(() => Math.random() - 0.5)
						}
						if (server && server === constant.initialscan) {
							return await scan(servers.shift())
						}
						app.locals.scannum = instancessorted.length
						app.locals.scantotal = instancessorted.length
						app.locals.server = server
						app.locals.instances++
						for (let i = 0; i < instancessorted.length; i += chunkSize) {
							const chunk = instancessorted.slice(i, i + chunkSize)
							parts.push(chunk)
						}
						await Promise.all(parts.map(async (p, i) => {
							for await (const exit of scanPart(p, i)) {
								console.log('Done: ' + server + ' - ' + i + ' parts with exit ' + exit + '.')
							}
						}))
						return await scanReturn()
					} else {
						return await scanReturn()
					}
				} catch (e) {
					console.error(e)
					return await scanReturn()
				}
			} else {
				return await scanReturn()
			}
		},
		job = schedule.scheduleJob('0 ' + constant.schedule + ' * * *', async () => {
			return await util.sendFederatedMessage(constant.nick, null, 'Scanning ' + app.locals.server + ' instance with ' + app.locals.scantotal + ' peers\nScanned ' + app.locals.peers + ' peers from ' + app.locals.instances + ' instances, ' + app.locals.created + ' created, ' + app.locals.updated + ' updated\nhttps://' + constant.apexdomain)
		})
	return await scanReturn()
}