initial commit

Signed-off-by: ale <ale@manalejandro.com>

back/constant.js

@@ -0,0 +1,17 @@
+module.exports = {
+    index: 'fediblock', // elasticsearch index name
+    nick: 'fediblockbot', // nick of bot
+    icon: 'https://manalejandro.com/favicon.png', // icon of the instance
+    dburl: 'mongodb://fediblock-mongodb:27017', // mongodb connection
+    dbname: 'fediblock', // mongodb database name
+    apexdomain: 'fediblock.manalejandro.com', // domain of the instance
+    agent: 'fediblock.manalejandro.com', // agent of fetch requests
+    elasticnode: 'http://fediblock-elasticsearch:9200', // elasticsearch connection
+    workers: 3, // async concurrent workers to scan
+    threads: 2, // threads of each worker
+    schedule: '5,11,17,23', // UTC hours to publish bot followers federated stats
+    taskdeletedup: '6', // task hour to delete dups index instances entries
+    filterdomains: ['activitypub-troll.cf'], // domains filtered to scan
+    initialscan: 'mastodon.social', // initial federated domain to scan
+    abort_timeout: 5000 // Timeout of abort scan request
+}

back/index.js

@@ -0,0 +1,3 @@
+const server = require('./lib/express')
+
+console.log('Server listening on ' + server.address().address + ':' + server.address().port)

back/lib/apex.js

@@ -0,0 +1,33 @@
+const ActivitypubExpress = require('activitypub-express'),
+    constant = require('../constant'),
+    routes = {
+        actor: '/u/:actor',
+        object: '/o/:id',
+        activity: '/s/:id',
+        inbox: '/u/:actor/inbox',
+        outbox: '/u/:actor/outbox',
+        followers: '/u/:actor/followers',
+        following: '/u/:actor/following',
+        liked: '/u/:actor/liked',
+        collections: '/u/:actor/c/:id',
+        blocked: '/u/:actor/blocked',
+        rejections: '/u/:actor/rejections',
+        rejected: '/u/:actor/rejected',
+        shares: '/s/:id/shares',
+        likes: '/s/:id/likes'
+    },
+    apex = ActivitypubExpress({
+        name: 'Fediblock Instance',
+        version: '1.0.0',
+        domain: constant.apexdomain,
+        actorParam: 'actor',
+        objectParam: 'id',
+        activityParam: 'id',
+        routes,
+        endpoints: {
+            proxyUrl: 'https://' + constant.apexdomain + '/proxy'
+        },
+        itemsPerPage: 200
+    })
+
+module.exports = { routes, apex }

back/lib/apexcustom.js

@@ -0,0 +1,97 @@
+// custom side-effects for your app
+module.exports = (app, apex, client) => {
+    const { sendFederatedMessage, requestPart } = require('./util'),
+        constant = require('../constant')
+    app.on('apex-outbox', msg => {
+        if (typeof msg === 'object'
+            && !Object.keys(msg).filter(prop => msg[prop]
+                && typeof msg[prop] === 'object').some(prop => !(msg[prop].hasOwnProperty('id')
+                    && msg[prop].hasOwnProperty('type')))
+            && msg.activity
+            && msg.activity.type) {
+            console.log(`New ${msg.activity.type} from ${msg.actor.id} to ${msg.recipient.id}`)
+        } else {
+            console.log(JSON.stringify(msg, '', 2))
+        }
+    })
+    app.on('apex-inbox', async msg => {
+        if (typeof msg === 'object'
+            && !Object.keys(msg).filter(prop => msg[prop]
+                && typeof msg[prop] === 'object').some(prop => !(msg[prop].hasOwnProperty('id')
+                    && msg[prop].hasOwnProperty('type')))
+            && msg.activity
+            && msg.activity.type) {
+            const type = msg.activity.type.toLowerCase()
+            if (type === 'follow' && msg.recipient.type.toLowerCase() === 'person') {
+                await apex.acceptFollow(msg.recipient, msg.activity)
+                const act = await apex.buildActivity('Accept', apex.utils.usernameToIRI(constant.nick), ['https://www.w3.org/ns/activitystreams#Public'].concat([msg.actor.id]), {
+                    actor: msg.recipient,
+                    object: msg.activity
+                })
+                await apex.addToOutbox(await apex.store.getObject(apex.utils.usernameToIRI(constant.nick), true), act)
+            }
+            if (type === 'create' && msg.object.type.toLowerCase() === 'note' && msg.actor.preferredUsername && msg.object.content) {
+                const name = Array.isArray(msg.actor.preferredUsername) ? msg.actor.preferredUsername[0] : msg.actor.preferredUsername,
+                    content = ('' + msg.object.content).replace(/<[^>]*>?/gm, '').split(' ').filter(token => !token.startsWith('@'))
+                if (msg.recipient.id === apex.utils.usernameToIRI(constant.nick)) {
+                    if (content.length === 1) {
+                        try {
+                            const instance = content.join('').trim()
+                            if (instance === 'stats') {
+                                const statsres = await requestPart('https://' + constant.apexdomain + '/api/stats')
+                                await sendFederatedMessage(constant.nick, null, `STATS\n
+Statuses AVG: ${Math.round(statsres.status_avg)}
+Statuses MAX: ${statsres.status_max}
+Domain AVG: ${Math.round(statsres.domain_avg)}
+Domain MAX: ${statsres.domain_max}
+Users AVG: ${Math.round(statsres.user_avg)}
+Users MAX: ${statsres.user_max}
+Stats Instances: ${statsres.stats_filtered}
+Total Instances: ${statsres.instance_count}
+Users by Instance: ${(Math.round(statsres.user_avg) / statsres.instance_count).toFixed(2)}
+Statuses by Domain: ${(Math.round(statsres.status_avg) / Math.round(statsres.domain_avg)).toFixed(2)}
+Statuses by User: ${(Math.round(statsres.status_avg) / Math.round(statsres.user_avg)).toFixed(2)}
+https://${constant.apexdomain}`, msg.actor.id, msg.object.id)
+                            } else {
+                                const json = await requestPart('https://' + constant.apexdomain + '/api/detail/' + instance)
+                                if (json && json.blocks && Array.isArray(json.blocks) && json.blocks.length > 0) {
+                                    const res = await requestPart('https://' + constant.apexdomain + '/api/list/' + instance)
+                                    if (res && res.instances && Array.isArray(res.instances)) {
+                                        res.instances.map(async r => {
+                                            if (r.domain === instance) {
+                                                await sendFederatedMessage(constant.nick, null, 'Instance ' + instance + ' has ' + json.blocks.length + ' blocks\n'
+                                                    + (r.api.title ? '\n'
+                                                        + r.api.title + ' - ' + r.api.uri + '\n'
+                                                        + (r.api.email ? 'Email: ' + r.api.email + '\n' : '')
+                                                        + 'Registration: ' + (r.api.registrations ? 'open' : 'closed') + ' - Version: ' + r.api.version + '\n'
+                                                        + (r.api.stats ? 'Users: ' + r.api.stats.user_count + ' - Statuses: ' + r.api.stats.status_count + ' - Domains: ' + r.api.stats.domain_count + '\n' : '')
+                                                        + (r.api.description ? 'Description: ' + r.api.description + '\n' : '') : '')
+                                                    + 'https://' + constant.apexdomain + '/#' + instance, msg.actor.id, msg.object.id)
+                                            }
+                                        })
+                                    } else {
+                                        await sendFederatedMessage(constant.nick, null, 'Instance ' + instance + ' has ' + json.blocks.length + ' blocks - https://' + constant.apexdomain + '/#' + instance, msg.actor.id, msg.object.id)
+                                    }
+                                } else {
+                                    await sendFederatedMessage(constant.nick, null, 'Instance ' + instance + ' not found, next try.', msg.actor.id, msg.object.id)
+                                }
+                            }
+                        } catch (e) {
+                            await sendFederatedMessage(constant.nick, null, e.message, msg.actor.id, msg.object.id)
+                            // console.error(e)
+                        }
+                    } else {
+                        await sendFederatedMessage(constant.nick, 'Hi ' + name, 'I know ' + (await client.count({ index: constant.index })).count + ' Federated Instances\nScanning ' + app.locals.server + ' instance with ' + app.locals.scantotal + ' peers\nScanned ' + app.locals.peers + ' peers from ' + app.locals.instances + ' instances, ' + app.locals.created + ' created, ' + app.locals.updated + ' updated\nhttps://' + constant.apexdomain, msg.actor.id, msg.object.id)
+                    }
+                }
+                console.log(`New note from ${name} to ${msg.recipient.id}: ${content.join(' ')}`)
+            } else if (type !== 'delete') {
+                console.log(`New ${msg.activity.type} from ${msg.actor.id} to ${msg.recipient.id}`)
+            } else {
+                console.log(`New ${msg.activity.type} from ${msg.actor.id} to ${msg.recipient.id}`)
+            }
+        } else {
+            console.log(JSON.stringify(msg, '', 2))
+        }
+    })
+}

back/lib/api.js

@@ -0,0 +1,54 @@
+module.exports = (app, apex) => {
+    const constant = require('../constant'),
+        { readFile, writeFile } = require('fs'),
+        asyncHandler = require('express-async-handler')
+    app.get('/api/scan', (req, res) => {
+        res.header('Content-Type', 'text/event-stream')
+        res.header('Connection', 'keep-alive')
+        res.header('Cache-Control', 'no-cache')
+        res.header('X-Accel-Buffering', 'no')
+        res.write(Buffer.from('data: ...\n\n'))
+        req.app.locals.scan.on('data', data => {
+            res.write(Buffer.from(data))
+        })
+        res.setTimeout(20000, () => {
+            res.write(Buffer.from('data: ...\n\n'))
+        })
+    })
+    app.get('/api/served', (req, res, next) => {
+        readFile(__dirname + '/../served.txt', async (err, data) => {
+            if (err) {
+                next(err)
+            } else {
+                const num = parseInt(data.toString()) + 1
+                writeFile(__dirname + '/../served.txt', num.toString(), 'utf8', (err) => {
+                    if (err) {
+                        next(err)
+                    } else {
+                        res.json({ served: num, lastscan: req.app.locals.scantotal, server: req.app.locals.server, instances: req.app.locals.instances, peers: req.app.locals.peers, created: req.app.locals.created, updated: req.app.locals.updated })
+                    }
+                })
+            }
+        })
+    })
+    app.get('/api/outbox', asyncHandler(async (req, res, next) => {
+        const outbox = await apex.getOutbox(await apex.store.getObject(apex.utils.usernameToIRI(constant.nick), true), 'true'),
+            notes = outbox.orderedItems.filter(e => e.object
+                && e.object.length > 0
+                && e.object[0].type === 'Note'
+                && e.object[0].content
+                && e.object[0].content[0].startsWith('<p>You')).slice(0, 20).map(e => ({
+                    content: e.object[0].content[0],
+                    published: e.published[0]
+                }))
+        res.json(notes)
+    }))
+    app.get('/api/inbox', asyncHandler(async (req, res, next) => {
+        const actor = await apex.store.getObject(apex.utils.usernameToIRI(constant.nick), true)
+        actor._local = {}
+        actor._local.blockList = []
+        const inbox = await apex.getInbox(actor, 'true'),
+            notes = inbox.orderedItems.filter(e => e.object && e.object.length > 0 && e.object[0].type === 'Note').slice(0, 20).map(e => e.object[0].content[0])
+        res.json(notes)
+    }))
+}

back/lib/apiswagger.js

@@ -0,0 +1,545 @@
+module.exports = (app, client) => {
+    const constant = require('../constant'),
+        zlib = require('zlib'),
+        asyncHandler = require('express-async-handler'),
+        { param, validationResult } = require('express-validator')
+    /**
+     * @swagger
+     * /api/stats:
+     *   get:
+     *     summary: Retrieve stats of instances.
+     *     description: Retrieve stats of total instances.
+     *     responses:
+     *       200:
+     *         description: A object with stats parameters.
+     *         content:
+     *           application/json:
+     *             schema:
+     *               type: object
+     *               properties:
+     *                 data:
+     *                   type: object
+     *                   properties:
+     *                     instance_count:
+     *                       type: integer
+     *                       description: Count of instances.
+     *                       example: 0
+     *                     status_avg:
+     *                       type: float
+     *                       description: Average statuses of total instances.
+     *                       example: 0
+     *                     status_max:
+     *                       type: integer
+     *                       description: Max of total statuses instances.
+     *                       example: 0
+     *                     user_avg:
+     *                       type: float
+     *                       description: Average users of total instances.
+     *                       example: 0
+     *                     user_max:
+     *                       type: integer
+     *                       description: Max of total users instances.
+     *                       example: 0
+     *                     domain_avg:
+     *                       type: float
+     *                       description: Number of average domains instances.
+     *                       example: 0
+     *                     domain_max:
+     *                       type: integer
+     *                       description: Number of max domains instances.
+     *                       example: 0
+     *                     stats_filtered:
+     *                       type: integer
+     *                       description: Number of instances stats.
+     *                       example: 0
+     */
+    app.get('/api/stats', asyncHandler(async (req, res) => {
+        const result = await client.search({
+            index: constant.index,
+            body: {
+                size: 0,
+                aggs: {
+                    instance_count: {
+                        value_count: {
+                            field: '_id'
+                        }
+                    },
+                    stats_filtered: {
+                        filter: {
+                            exists: {
+                                field: 'api.stats'
+                            }
+                        }
+                    },
+                    user_avg: {
+                        avg: {
+                            field: 'api.stats.user_count'
+                        }
+                    },
+                    user_max: {
+                        max: {
+                            field: 'api.stats.user_count'
+                        }
+                    },
+                    domain_avg: {
+                        avg: {
+                            field: 'api.stats.domain_count'
+                        }
+                    },
+                    domain_max: {
+                        max: {
+                            field: 'api.stats.domain_count'
+                        }
+                    },
+                    status_avg: {
+                        avg: {
+                            field: 'api.stats.status_count'
+                        }
+                    },
+                    status_max: {
+                        max: {
+                            field: 'api.stats.status_count'
+                        }
+                    }
+
+                }
+            }
+        })
+        res.json(Object.keys(result.aggregations).reduce((prev, curr) => ({
+            ...prev, [curr]: result.aggregations[curr][Object.keys(result.aggregations[curr])[0]]
+        }), {}))
+    }))
+    /**
+     * @swagger
+     * /api/count:
+     *   get:
+     *     summary: Retrieve count of instances.
+     *     description: Retrieve a number of total instances.
+     *     responses:
+     *       200:
+     *         description: A object with count parameter.
+     *         content:
+     *           application/json:
+     *             schema:
+     *               type: object
+     *               properties:
+     *                 data:
+     *                   type: object
+     *                   properties:
+     *                     count:
+     *                       type: integer
+     *                       description: Number of instances.
+     *                       example: 0
+     */
+    app.get('/api/count', asyncHandler(async (req, res) => {
+        res.json({ count: (await client.count({ index: constant.index })).count })
+    }))
+    /**
+     * @swagger
+     * /api/ranking:
+     *   get:
+     *     summary: Retrieve one ranking of instances.
+     *     description: Retrieve a top ten ranking of total fediblock instances.
+     *     responses:
+     *       200:
+     *         description: A object with count parameter.
+     *         content:
+     *           application/json:
+     *             schema:
+     *               type: object
+     *               properties:
+     *                 data:
+     *                   type: object
+     *                   properties:
+     *                     domain:
+     *                       type: string
+     *                       description: Domain of the instance.
+     *                       example: "mastodon.social"
+     *                     count:
+     *                       type: integer
+     *                       description: Number of fediblocks.
+     *                       example: 0
+     */
+    app.get('/api/ranking', asyncHandler(async (req, res) => {
+        const result = await client.search({
+            index: constant.index,
+            body: {
+                size: 0,
+                aggs: {
+                    blocks: {
+                        nested: {
+                            path: 'blocks'
+                        },
+                        aggs: {
+                            ranking: {
+                                terms: {
+                                    field: 'blocks.domain',
+                                    size: 100
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        })
+        const ranking = result.aggregations?.blocks?.ranking?.buckets
+        res.json(ranking.map(r => ({ domain: r.key, count: r.doc_count })))
+    }))
+    /**
+    * @swagger
+    * /api/list/{instance}:
+    *   get:
+    *     summary: Search result array of intances.
+    *     description: Retrieve a result array of instances matching search input.
+    *     parameters:
+    *       - in: path
+    *         name: instance
+    *         required: true
+    *         description: String for search instance
+    *         schema:
+    *           type: string
+    *     responses:
+    *       200:
+    *         description: A list of instances.
+    *         content:
+    *           application/json:
+    *             schema:
+    *               type: object
+    *               properties:
+    *                 data:
+    *                   type: array
+    *                   items:
+    *                     type: object
+    *                     properties:
+    *                       instances:
+    *                         type: array
+    *                         items:
+    *                           type: data
+    *                           description: List of instances.
+    *                           example: "mastodon.social"
+    *                       suggests:
+    *                         type: array
+    *                         items:
+    *                           type: data
+    *                           description: Suggest of the instance.
+    *                           example: "mastodon.social"
+    */
+    app.get('/api/list/:instance', param('instance').notEmpty().trim().escape(), asyncHandler(async (req, res) => {
+        const errors = validationResult(req)
+        if (!errors.isEmpty()) {
+            return res.status(400).json({
+                errors: errors.array()
+            })
+        }
+        if (req.params.instance && req.params.instance.length > 0) {
+            const result = await client.search({
+                index: constant.index,
+                size: 10,
+                query: {
+                    wildcard: {
+                        instance: {
+                            value: `*${req.params.instance}*`
+                        }
+                    },
+                },
+                suggest: {
+                    suggests: {
+                        text: req.params.instance,
+                        term: {
+                            field: 'instance',
+                            size: 3,
+                            sort: 'score',
+                            suggest_mode: 'always',
+                            max_edits: 2,
+                            min_word_length: 1
+                        }
+                    }
+                }
+            })
+            const instances = result.hits?.hits?.length > 0 ? result.hits.hits : [],
+                suggests = result.suggest.suggests && result.suggest.suggests.length > 0 && result.suggest.suggests[0].options.length > 0 ? result.suggest.suggests[0].options : []
+            res.json({
+                instances: instances.map(instance => ({
+                    domain: instance._source.instance,
+                    api: instance._source.api ? instance._source.api : null,
+                    blocks: instance._source.blocks && instance._source.blocks.length > 0 ? instance._source.blocks.length : null,
+                    last: instance._source.last ? instance._source.last : null,
+                    nodeinfo: instance._source.nodeinfo ? true : false
+                })), suggests: suggests.map(instance => instance.text)
+            })
+        } else {
+            res.status(404).end()
+        }
+    }))
+    /**
+    * @swagger
+    * /api/detail/{instance}:
+    *   get:
+    *     summary: Search result array of fediblocks intances.
+    *     description: Retrieve a result array of fediblock instances matching search input.
+    *     parameters:
+    *       - in: path
+    *         name: instance
+    *         required: true
+    *         description: String to detail of the instance
+    *         schema:
+    *           type: string
+    *     responses:
+    *       200:
+    *         description: A list of instances.
+    *         content:
+    *           application/json:
+    *             schema:
+    *               type: object
+    *               properties:
+    *                 data:
+    *                   type: array
+    *                   items:
+    *                     type: object
+    *                     properties:
+    *                       domain:
+    *                         type: string
+    *                         description: Domain of the block instance.
+    *                         example: "mastodon.social"
+    *                       comment:
+    *                         type: string
+    *                         description: Comment of the block instance.
+    *                         example: "mastodon.social"
+    *                       severity:
+    *                         type: string
+    *                         description: Severity of the block instance.
+    *                         example: "mastodon.social"
+    * 
+    */
+    app.get('/api/detail/:instance', param('instance').notEmpty().trim().escape(), asyncHandler(async (req, res) => {
+        const errors = validationResult(req)
+        if (!errors.isEmpty()) {
+            return res.status(400).json({
+                errors: errors.array()
+            })
+        }
+        if (req.params.instance && req.params.instance.length > 0) {
+            const result = await client.search({
+                index: constant.index,
+                size: 1,
+                query: {
+                    term: {
+                        instance: req.params.instance
+                    }
+                }
+            })
+            const instances = result.hits?.hits?.length > 0 ? result.hits.hits : []
+            res.json(instances.length > 0 && instances[0]._source.blocks && instances[0]._source.blocks.length > 0 ?
+                {
+                    blocks: instances[0]._source.blocks.map(instance => ({ domain: instance.domain, comment: instance.comment, severity: instance.severity })),
+                    last: instances[0]._source.last,
+                    instance: instances[0]._source.instance,
+                    nodeinfo: instances[0]._source.nodeinfo,
+                    api: instances[0]._source.api,
+                    took: result.took
+                } : [])
+        } else {
+            res.status(404).end()
+        }
+    }))
+    /**
+    * @swagger
+    * /api/detail_api/{instance}:
+    *   get:
+    *     summary: Search result of detail's api intance.
+    *     description: Retrieve a result of detail's api instance.
+    *     parameters:
+    *       - in: path
+    *         name: instance
+    *         required: true
+    *         description: String to detail of the instance
+    *         schema:
+    *           type: string
+    *     responses:
+    *       200:
+    *         description: Detail of the api instance.
+    *         content:
+    *           application/json:
+    *             schema:
+    *               type: object
+    *               properties:
+    *                 data:
+    *                   type: object
+    */
+    app.get('/api/detail_api/:instance', param('instance').notEmpty().trim().escape(), asyncHandler(async (req, res) => {
+        const errors = validationResult(req)
+        if (!errors.isEmpty()) {
+            return res.status(400).json({
+                errors: errors.array()
+            })
+        }
+        if (req.params.instance && req.params.instance.length > 0) {
+            const result = await client.search({
+                index: constant.index,
+                size: 1,
+                query: {
+                    term: {
+                        instance: req.params.instance
+                    }
+                }
+            })
+            const instances = result.hits?.hits?.length > 0 ? result.hits.hits : []
+            res.json(instances.length > 0 ? instances[0]._source.api : {})
+        } else {
+            res.status(404).end()
+        }
+    }))
+    /**
+    * @swagger
+    * /api/detail_nodeinfo/{instance}:
+    *   get:
+    *     summary: Search result of detail's nodeinfo intance.
+    *     description: Retrieve a result of detail's nodeinfo instance.
+    *     parameters:
+    *       - in: path
+    *         name: instance
+    *         required: true
+    *         description: String to detail of the instance
+    *         schema:
+    *           type: string
+    *     responses:
+    *       200:
+    *         description: Detail of the nodeinfo instance.
+    *         content:
+    *           application/json:
+    *             schema:
+    *               type: object
+    *               properties:
+    *                 data:
+    *                   type: object
+    */
+    app.get('/api/detail_nodeinfo/:instance', param('instance').notEmpty().trim().escape(), asyncHandler(async (req, res) => {
+        const errors = validationResult(req)
+        if (!errors.isEmpty()) {
+            return res.status(400).json({
+                errors: errors.array()
+            })
+        }
+        if (req.params.instance && req.params.instance.length > 0) {
+            const result = await client.search({
+                index: constant.index,
+                size: 1,
+                query: {
+                    term: {
+                        instance: req.params.instance
+                    }
+                }
+            })
+            const instances = result.hits?.hits?.length > 0 ? result.hits.hits : []
+            res.json(instances.length > 0 ? instances[0]._source.nodeinfo : {})
+        } else {
+            res.status(404).end()
+        }
+    }))
+    /**
+    * @swagger
+    * /api/block_count/{instance}:
+    *   get:
+    *     summary: Retrieve count of fediblocked instances.
+    *     description: Retrieve a number of total fediblocked instances.
+    *     parameters:
+    *       - in: path
+    *         name: instance
+    *         required: true
+    *         description: String to search fediblocks of the instance
+    *         schema:
+    *           type: string
+    *     responses:
+    *       200:
+    *         description: A object with block_count parameter.
+    *         content:
+    *           application/json:
+    *             schema:
+    *               type: object
+    *               properties:
+    *                 data:
+    *                   type: object
+    *                   properties:
+    *                     block_count:
+    *                       type: integer
+    *                       description: Number of fediblock instances.
+    *                       example: 0
+    *                     instances:
+    *                       type: array
+    *                       items:
+    *                         type: object
+    *                         properties:
+    *                           instance:
+    *                             type: string
+    *                             description: Instance of the block.
+    *                             example: mastodon.social
+    *                           comment:
+    *                             type: string
+    *                             description: Comment about the block.
+    *                             example: spam
+    */
+    app.get('/api/block_count/:instance', param('instance').notEmpty().trim().escape(), asyncHandler(async (req, res) => {
+        const errors = validationResult(req)
+        if (!errors.isEmpty()) {
+            return res.status(400).json({
+                errors: errors.array()
+            })
+        }
+        if (req.params.instance && req.params.instance.length > 0) {
+            const result = await client.search({
+                index: constant.index,
+                size: 9999,
+                query: {
+                    nested: {
+                        path: 'blocks',
+                        query: {
+                            term: {
+                                'blocks.domain': req.params.instance
+                            }
+                        }
+                    }
+                }
+            })
+            const instances = result.hits?.hits?.length > 0 ? result.hits.hits : [],
+                instancescomment = instances.map(i => ({
+                    instance: i._source.instance, comment: i._source.blocks.find(block => block.domain === req.params.instance).comment
+                }))
+            res.json({
+                block_count: instances.length,
+                instances: instancescomment,
+                took: result.took
+            })
+        } else {
+            res.status(404).end()
+        }
+    }))
+    /**
+    * @swagger
+    * /api/download_index:
+    *   get:
+    *     summary: Retrieve all content of ElasticSearch index.
+    *     description: Retrieve all content of ElasticSearch index.
+    *     responses:
+    *       200:
+    *         description: A file compressed with gzip.
+    *         content:
+    *           application/gzip:
+    */
+    app.get('/api/download_index', asyncHandler(async (req, res) => {
+        try {
+            res.setHeader('Content-Type', 'application/gzip')
+            res.setHeader('Content-disposition', 'attachment; filename=fediblock-index.json.gz')
+            const result = await client.search({
+                index: constant.index,
+                size: 9999,
+                query: {
+                    match_all: {}
+                }
+            }, { asStream: true, meta: false })
+            result.pipe(zlib.createGzip()).pipe(res)
+        } catch (e) {
+            console.error(e)
+            res.status(404).end()
+        }
+    }))
+}

back/lib/express.js

@@ -0,0 +1,158 @@
+const http = require('http'),
+    express = require('express'),
+    rateLimit = require('express-rate-limit'),
+    app = express(),
+    events = require('events'),
+    constant = require('../constant'),
+    logger = require('./logger'),
+    { routes, apex } = require('./apex'),
+    apexcustom = require('./apexcustom'),
+    apiswagger = require('./apiswagger'),
+    api = require('./api'),
+    swagger = require('./swagger'),
+    fediblock = require('./fediblock'),
+    taskdeletedup = require('./taskdeletedup'),
+    { generateKeyPairSync } = require('crypto'),
+    { MongoClient } = require('mongodb'),
+    mongoclient = new MongoClient(constant.dburl),
+    { Client } = require('@elastic/elasticsearch'),
+    client = new Client({
+        node: constant.elasticnode,
+        pingTimeout: 10000,
+        requestTimeout: 60000,
+        retryOnTimeout: true,
+        maxRetries: 3
+    }),
+    register = async () => {
+        const admin = await apex.store.getObject(apex.utils.usernameToIRI('admin'), true)
+        if (!admin) {
+            const adminaccount = await apex.createActor('admin', 'Fediblock Admin', 'Fediblock Admin - https://' + constant.apexdomain,
+                { type: 'Image', mediaType: 'image/png', url: constant.icon }, 'Service'),
+                keys = generateKeyPairSync('rsa', {
+                    modulusLength: 2048,
+                    publicKeyEncoding: {
+                        type: 'spki',
+                        format: 'pem'
+                    },
+                    privateKeyEncoding: {
+                        type: 'pkcs8',
+                        format: 'pem'
+                    }
+                })
+            adminaccount.publicKey[0].publicKeyPem[0] = keys.publicKey
+            adminaccount._meta.privateKey = keys.privateKey
+            await apex.store.saveObject(adminaccount)
+            const bot = await apex.createActor(constant.nick, 'Fediblock Bot', 'Fediblock #Bot - https://' + constant.apexdomain,
+                { type: 'Image', mediaType: 'image/png', url: constant.icon }, 'Person'),
+                keysbot = generateKeyPairSync('rsa', {
+                    modulusLength: 2048,
+                    publicKeyEncoding: {
+                        type: 'spki',
+                        format: 'pem'
+                    },
+                    privateKeyEncoding: {
+                        type: 'pkcs8',
+                        format: 'pem'
+                    }
+                })
+            bot.publicKey[0].publicKeyPem[0] = keysbot.publicKey
+            bot._meta.privateKey = keysbot.privateKey
+            await apex.store.saveObject(bot)
+            apex.systemUser = adminaccount
+            apex.store.setup(adminaccount)
+        } else {
+            apex.systemUser = admin
+        }
+    },
+    connect = () => {
+        mongoclient.connect()
+            .then(async () => {
+                const exists = await client.indices.exists({ index: constant.index })
+                if (!exists) {
+                    await client.indices.create({
+                        index: constant.index,
+                        body: require('../fediblock-mapping.json')
+                    })
+                }
+                apex.store.db = mongoclient.db(constant.dbname)
+                return apex.store.db
+            })
+            .then(async () => {
+                try {
+                    await register()
+                    taskdeletedup(client)
+                    await apex.startDelivery()
+                    await fediblock(client, app)
+                } catch (e) {
+                    console.error(e)
+                }
+            })
+    },
+    server = http.createServer(app).listen(4000, () => {
+        connect()
+    })
+
+app.locals.scan = new events.EventEmitter()
+app.locals.scannum = 0
+app.locals.scantotal = 0
+app.locals.server = ''
+app.locals.peers = 0
+app.locals.instances = 0
+app.locals.created = 0
+app.locals.updated = 0
+app.disable('x-powered-by')
+app.set('json spaces', 2)
+app.set('trust proxy', 1)
+app.use(logger)
+app.use(rateLimit({
+    windowMs: 1 * 60 * 1000, // 1 minutes
+    limit: 120, // each IP can make up to 120 requests per `windowsMs` (5 minutes)
+    standardHeaders: true, // add the `RateLimit-*` headers to the response
+    legacyHeaders: false,
+    delayAfter: 30, // allow 30 requests per `windowMs` (5 minutes) without slowing them down
+    delayMs: (hits) => hits * 200, // add 200 ms of delay to every request after the 10th
+    maxDelayMs: 5000
+}))
+app.use(
+    express.json({ type: apex.consts.jsonldTypes }),
+    express.urlencoded({ extended: true }),
+    apex
+)
+app.route(routes.inbox)
+    .get(apex.net.inbox.get)
+    .post(apex.net.inbox.post)
+app.route(routes.outbox)
+    .get(apex.net.outbox.get)
+    .post(apex.net.outbox.post)
+app.get(routes.actor, apex.net.actor.get)
+app.get(routes.followers, apex.net.followers.get)
+app.get(routes.following, apex.net.following.get)
+app.get(routes.liked, apex.net.liked.get)
+app.get(routes.object, apex.net.object.get)
+app.get(routes.activity, apex.net.activityStream.get)
+app.get(routes.shares, apex.net.shares.get)
+app.get(routes.likes, apex.net.likes.get)
+app.get('/.well-known/webfinger', apex.net.webfinger.get)
+app.get('/.well-known/nodeinfo', apex.net.nodeInfoLocation.get)
+app.get('/nodeinfo/:version', apex.net.nodeInfo.get)
+app.post('/proxy', apex.net.proxy.post)
+apexcustom(app, apex, client)
+api(app, apex, client)
+apiswagger(app, client)
+swagger(app)
+app.get('/u/' + constant.nick, (req, res) => {
+    res.redirect('/api/inbox')
+})
+app.get(Object.keys(routes).map(route => routes[route]), (req, res) => {
+    res.redirect('/')
+})
+app.use(express.static(__dirname + '/../../front/build'))
+    .use((err, req, res, next) => {
+        if (res.headersSent) {
+            console.error(err.stack)
+            return next(err)
+        }
+        res.status(500).end('Error')
+    })
+
+module.exports = server

back/lib/fediblock.js

@@ -0,0 +1,211 @@
+module.exports = async (client, app) => {
+	const { sendFederatedMessage, requestPart } = require('./util'),
+		constant = require('../constant'),
+		schedule = require('node-schedule'),
+		{ PromisePool } = require('@supercharge/promise-pool'),
+		workers = constant.workers,
+		threads = constant.threads,
+		getAccount = api => {
+			if (api && api.contact_account.acct) {
+				const acct = api.contact_account.acct.split('@')
+				if (acct.length > 1) {
+					return `https://${acct[1]}/users/${acct[0]}`
+				} else {
+					return `https://${api.uri.replace(/https?:\/\//, '')}/users/${acct.join('')}`
+				}
+			} else {
+				return ''
+			}
+		},
+		scanInstance = async instance => {
+			const json = await requestPart(`https://${instance}/api/v1/instance/domain_blocks`)
+			if (Array.isArray(json) && json.length > 0) {
+				const result = await client.search({
+					index: constant.index,
+					size: 1,
+					query: {
+						term: {
+							instance: instance
+						}
+					}
+				}),
+					instancelocated = result.hits && result.hits.hits ? result.hits.hits : [],
+					blocks = json.map(block => {
+						if (block.comment && block.comment.length > 8190) {
+							block.comment = block.comment.slice(0, 8190)
+						}
+						return block
+					}),
+					[api, nodeinfo, peers] = await Promise.all([
+						requestPart(`https://${instance}/api/v1/instance`),
+						requestPart(`https://${instance}/nodeinfo/2.0`),
+						requestPart(`https://${instance}/api/v1/instance/peers`)
+					])
+				if (result && result.hits) {
+					if (instancelocated.length === 0) {
+						await client.index({
+							index: constant.index,
+							body: {
+								instance,
+								api,
+								nodeinfo,
+								blocks,
+								peers,
+								last: new Date()
+							}
+						})
+						app.locals.created++
+						return sendFederatedMessage(constant.nick, 'New Fediblock Instance', `Fediblock Instance ${instance} with ${json.length} blocks - https://${constant.apexdomain}#${instance}`, getAccount(api))
+					} else {
+						const elasticinstance = instancelocated[0]._source.blocks || []
+						if (Array.isArray(elasticinstance)) {
+							if (json.length !== elasticinstance.length
+								|| (instancelocated[0]._source.last && instancelocated[0]._source.last < new Date(Date.now() - 2678400000))
+								|| !instancelocated[0]._source.api
+								|| !instancelocated[0]._source.nodeinfo
+								|| !instancelocated[0]._source.peers) {
+								await client.update({
+									index: constant.index,
+									id: instancelocated[0]._id,
+									doc: {
+										api: api ? api : instancelocated[0]._source.api,
+										nodeinfo: nodeinfo ? nodeinfo : instancelocated[0]._source.nodeinfo,
+										blocks: blocks && blocks.length > 0 ? blocks : elasticinstance,
+										peers: peers && peers.length > 0 ? peers : instancelocated[0]._source.peers,
+										last: new Date()
+									},
+									doc_as_upsert: true
+								})
+								app.locals.updated++
+								if (instancelocated[0]._source.api && instancelocated[0]._source.api.uri && instancelocated[0]._source.api.contact_account && instancelocated[0]._source.api.contact_account.acct) {
+									const difference = blocks.filter(block => block.domain && block.domain.trim().length > 0 && !elasticinstance.some(instance => block.domain === instance.domain))
+									if (difference.length > 0 && difference.length < 20) {
+										return sendFederatedMessage(constant.nick, 'Detected #Fediblock by Fediblock Instance', `You blocked new instances: ${difference.map(d => d.domain).join(', ')} - https://${constant.apexdomain}#${instance}`, getAccount(instancelocated[0]._source.api))
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+		},
+		scanPart = async instancesall => PromisePool
+			.withConcurrency(threads)
+			.for(instancesall)
+			.process(async instance => {
+				app.locals.scan.emit('data', 'data: ' + (app.locals.scannum > 0 ? '(' + app.locals.scannum-- + '): ' + instance : ': ' + instance) + '\n\n')
+				app.locals.peers++
+				return scanInstance(instance)
+			}),
+		scanIndex = async (server, instancesall) => {
+			const [api, nodeinfo, blocks] = await Promise.all([
+				requestPart(`https://${server}/api/v1/instance`),
+				requestPart(`https://${server}/nodeinfo/2.0`),
+				requestPart(`https://${server}/api/v1/instance/domain_blocks`)
+			])
+			if (api && typeof api === 'object' && api.version) {
+				const result = await client.search({
+					index: constant.index,
+					size: 1,
+					query: {
+						term: {
+							instance: server
+						}
+					}
+				}),
+					instancelocated = result.hits && result.hits.hits ? result.hits.hits : []
+				if (result && result.hits) {
+					if (instancelocated.length === 0) {
+						await client.index({
+							index: constant.index,
+							body: {
+								instance: server,
+								peers: instancesall,
+								api,
+								nodeinfo,
+								blocks: blocks && blocks.length > 0 ? blocks : [],
+								last: new Date()
+							}
+						})
+						app.locals.created++
+					} else {
+						const elasticinstance = instancelocated[0]._source.peers || []
+						if (Array.isArray(elasticinstance) && Array.isArray(instancesall) && instancesall.length > 0) {
+							if (instancesall.length !== elasticinstance.filter(i => !constant.filterdomains.some(d => i.endsWith(d))).length) {
+								await client.update({
+									index: constant.index,
+									id: instancelocated[0]._id,
+									doc: {
+										peers: instancesall.length > 0 ? instancesall : elasticinstance,
+										api: api ? api : instancelocated[0]._source.api,
+										nodeinfo: nodeinfo ? nodeinfo : instancelocated[0]._source.nodeinfo,
+										blocks: blocks && blocks.length > 0 ? blocks : instancelocated[0]._source.blocks,
+										last: new Date()
+									},
+									doc_as_upsert: true
+								})
+								app.locals.updated++
+							}
+						}
+					}
+				}
+			}
+		},
+		scanReturn = async () => {
+			app.locals.scannum = 0
+			if (app.locals.servers?.length > 0) {
+				await scan(app.locals.servers.shift())
+			} else {
+				await scan(constant.initialscan)
+			}
+		},
+		scan = async server => {
+			if (server) {
+				try {
+					console.log(server)
+					app.locals.scan.emit('data', 'data:  ' + server + ' peers...\n\n')
+					const instances = await requestPart(`https://${server}/api/v1/instance/peers`),
+						instancesall = Array.isArray(instances) && instances.length > 0
+							? instances.filter(i => !constant.filterdomains.some(d => i.endsWith(d)))
+							: [],
+						instancessorted = instancesall.sort((a, b) => a < b ? -1 : a > b ? 1 : 0),
+						parts = [],
+						split = workers,
+						chunkSize = Math.ceil(instancessorted.length / split)
+					if (instancesall.length > 0) {
+						await scanIndex(server, instancesall)
+						if (!app.locals.servers || app.locals.servers.length === 0) {
+							app.locals.servers = instancesall.sort(() => Math.random() - 0.5)
+						}
+						if (server === constant.initialscan) {
+							return scan(app.locals.servers.shift())
+						}
+						app.locals.scannum = instancessorted.length
+						app.locals.scantotal = instancessorted.length
+						app.locals.server = server
+						app.locals.instances++
+						for (let i = 0; i < instancessorted.length; i += chunkSize) {
+							const chunk = instancessorted.slice(i, i + chunkSize)
+							parts.push(chunk)
+						}
+						await PromisePool
+							.withConcurrency(workers)
+							.for(parts)
+							.process(scanPart)
+						await scanReturn()
+					} else {
+						await scanReturn()
+					}
+				} catch (e) {
+					// console.error(e)
+					await scanReturn()
+				}
+			} else {
+				await scanReturn()
+			}
+		},
+		job = schedule.scheduleJob('0 ' + constant.schedule + ' * * *', async () => {
+			await sendFederatedMessage(constant.nick, null, 'Scanning ' + app.locals.server + ' instance with ' + app.locals.scantotal + ' peers\nScanned ' + app.locals.peers + ' peers from ' + app.locals.instances + ' instances, ' + app.locals.created + ' created, ' + app.locals.updated + ' updated\nhttps://' + constant.apexdomain)
+		})
+	return scanReturn()
+}

back/lib/logger.js

@@ -0,0 +1,6 @@
+const morgan = require('morgan'),
+    rfs = require('rotating-file-stream'),
+    accessLogStream = rfs.createStream('access.log', { interval: '1d', path: __dirname + '/../logs' }),
+    logger = morgan('combined', { stream: accessLogStream })
+
+module.exports = logger

back/lib/swagger.js

@@ -0,0 +1,25 @@
+module.exports = app => {
+    const swaggerJsdoc = require('swagger-jsdoc'),
+        swaggerUi = require('swagger-ui-express'),
+        options = {
+            swaggerOptions: {
+                withCredentials: false
+            },
+            swaggerDefinition: {
+                restapi: '3.1.0',
+                info: {
+                    title: 'Fediblock Instance API',
+                    version: '1.0.0',
+                    description: '#Fediblock Instance REST API',
+                },
+                servers: [
+                    {
+                        url: 'http://localhost:3000',
+                    },
+                ],
+            },
+            apis: ['**/apiswagger.js'],
+        },
+        specs = swaggerJsdoc(options)
+    app.use('/api-docs', swaggerUi.serve, swaggerUi.setup(specs, { explorer: false }))
+}

back/lib/taskdeletedup.js

@@ -0,0 +1,69 @@
+module.exports = client => {
+    const constant = require('../constant'),
+        schedule = require('node-schedule'),
+        { pick } = require('stream-json/filters/Pick'),
+        { parser } = require('stream-json'),
+        { streamArray } = require('stream-json/streamers/StreamArray'),
+        { chain } = require('stream-chain'),
+        size = 500,
+        deleteDup = async () => {
+            const count = { deleted: 0, total: 0, current: 0 }
+            let lastsort = undefined, last = undefined
+            await searchDup(count, lastsort, last)
+        },
+        searchDup = async (count, lastsort, last) => {
+            count.current = 0
+            const result = await client.search({
+                index: constant.index,
+                size: size,
+                body: {
+                    query: {
+                        match_all: {}
+                    },
+                    sort: [{
+                        "instance": {
+                            "order": "asc"
+                        },
+                        "last": {
+                            "order": "desc",
+                            "numeric_type": "date_nanos",
+                            "format": "strict_date_optional_time_nanos"
+                        }
+                    }],
+                    search_after: lastsort
+                }
+            }, { asStream: true, meta: false }),
+                pipeline = chain([
+                    parser(),
+                    pick({ filter: 'hits.hits' }),
+                    streamArray(),
+                    data => data.value
+                ])
+            pipeline.on('data', async data => {
+                count.current++
+                if (last && last.instance === data._source.instance) {
+                    await client.delete({ index: constant.index, id: data._id })
+                    count.deleted++
+                    console.log('deleted ' + data._id + ': ' + data._source.instance)
+                }
+                else {
+                    last = data._source
+                }
+                if (count.current === size) {
+                    lastsort = data.sort
+                }
+            })
+            pipeline.on('end', async () => {
+                count.total += count.current
+                if (count.current === size) {
+                    await searchDup(count, lastsort, last)
+                } else {
+                    console.log('Index: ' + constant.index + ' - Total: ' + count.total + ' - Deleted: ' + count.deleted)
+                }
+            })
+            result.pipe(pipeline)
+        },
+        job = schedule.scheduleJob('0 ' + constant.taskdeletedup + ' * * *', async () => {
+            await deleteDup()
+        })
+}

back/lib/util.js

@@ -0,0 +1,83 @@
+const constant = require("../constant"),
+    { apex } = require('./apex'),
+    https = require('node:https'),
+    json = 'application/json',
+    urlToId = url => {
+        try {
+            if (typeof new URL(url) === 'object' && url.split('/').length === 5 && url.split('/')[3].length > 0) {
+                return '@' + url.split('/')[4] + '@' + url.split('/')[2]
+            } else {
+                return url
+            }
+        } catch (e) {
+            return url
+        }
+    },
+    sendFederatedMessage = async (id, summary, message, recipient, reply) => {
+        const users = message.split(' ').filter(token => token.startsWith('@') && token.split('@').length === 3)
+            .map(token => `https://${token.split('@')[2]}/users/${token.split('@')[1]}`),
+            mentions = message.split(' ').filter(token => token.startsWith('@') && token.split('@').length === 3)
+                .map(token => { return { type: 'Mention', href: `https://${token.split('@')[2]}/users/${token.split('@')[1]}`, name: `@${token.split('@')[1]}` } }),
+            hashtags = message.split(' ').filter(token => token.startsWith('#'))
+                .map(token => { return { id: `https://${constant.apexdomain}/tags/${token.split('#')[1]}`, name: token } }),
+            images = message.split(' ').filter(token => token.match(/^https?:\/\//i) && token.match(/\.(jpg|png|gif|jpeg|ppm)$/i))
+                .map(token => { return { type: 'Image', url: token } }),
+            audio = message.split(' ').filter(token => token.match(/^https?:\/\//i) && token.match(/\.(mp3|wav|flac)$/i))
+                .map(token => { return { type: 'Audio', url: token } }),
+            video = message.split(' ').filter(token => token.match(/^https?:\/\//i) && token.match(/\.(mp4|flv|avi)$/i))
+                .map(token => { return { type: 'Video', url: token } }),
+            links = message.split(' ').filter(token => token.match(/^https?:\/\//i) && !images.concat(audio).concat(video)
+                .some(link => link.url === token)).map(token => { return { type: 'Link', href: token } }),
+            allusers = users.concat([recipient || (await apex.store.getObject(apex.utils.usernameToIRI(id), true)).followers[0]]),
+            act = await apex.buildActivity('Create', apex.utils.usernameToIRI(id), ['https://www.w3.org/ns/activitystreams#Public'].concat(allusers), {
+                object: {
+                    type: 'Note',
+                    name: `Fediblock Instance`,
+                    summary: summary || null,
+                    content: `<p>${message.replace(/(\b(https?|ftp|file):\/\/([-A-Z0-9+&@#%?=~_|!:,.;]*)([-A-Z0-9+&@#%?/=~_|!:,.;]*))/ig,
+                        '<a href="$1" target="_blank">$3</a>').replace(/\s#(\S+)/g, '<a href="https://mastodon.social/tags/$1">#$1</a>').replace(/\r?\n/g, '<br />')}</p>`,
+                    tag: hashtags.concat(mentions),
+                    attachment: images.concat(links).concat(audio).concat(video),
+                    inReplyTo: reply || null
+                }
+            })
+        act.object[0].id = act.id
+        await apex.addToOutbox(await apex.store.getObject(apex.utils.usernameToIRI(id), true), act)
+    },
+    requestPart = uri => new Promise((resolve, reject) => {
+        try {
+            const data = [],
+                req = https.request(uri, {
+                    headers: {
+                        'User-Agent': constant.agent,
+                        'Content-Type': json
+                    },
+                    signal: AbortSignal.timeout(constant.abort_timeout),
+                }, res => {
+                    res.setEncoding('utf8')
+                    res.headers['Content-Type'] = json
+                    res.on('data', chunk => {
+                        data.push(chunk)
+                    })
+                    res.on('error', error => {
+                        reject(error)
+                    })
+                    res.on('end', () => {
+                        try {
+                            resolve(JSON.parse(data.join('')))
+                        } catch (error) {
+                            reject(error)
+                        }
+                    })
+                })
+            req.on('error', error => {
+                reject(error)
+            })
+            req.end()
+        } catch (error) {
+            reject(error)
+        }
+    })
+
+module.exports = { urlToId, sendFederatedMessage, requestPart }
+

back/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "fediblock-instance",
+  "version": "1.0.0",
+  "description": "Fediblock Instance",
+  "author": "ale",
+  "repository": {
+    "type": "git",
+    "url": "https://git.manalejandro.com/ale/fediblock-instance"
+  },
+  "license": "MIT",
+  "scripts": {
+    "start": "node index.js",
+    "build": "cd ../front && yarn build",
+    "install": "cd node_modules && rm -rf http-signature && rm -rf request/node_modules/http-signature && mv @peertube/http-signature . && cd ../../front && yarn"
+  },
+  "private": true,
+  "dependencies": {
+    "@elastic/elasticsearch": "^8.17.1",
+    "@peertube/http-signature": "^1.7.0",
+    "@supercharge/promise-pool": "^3.2.0",
+    "@types/morgan": "^1.9.9",
+    "activitypub-express": "^4.4.2",
+    "express": "^4.21.2",
+    "express-async-handler": "^1.2.0",
+    "express-rate-limit": "^7.5.0",
+    "express-validator": "^7.2.1",
+    "mongodb": "^4.17.2",
+    "morgan": "^1.10.0",
+    "node-schedule": "^2.1.1",
+    "rotating-file-stream": "^3.2.6",
+    "stream-json": "^1.9.1",
+    "swagger-jsdoc": "^6.2.8",
+    "swagger-ui-express": "^5.0.1"
+  }
+}

back/served.txt

@@ -0,0 +1 @@
+0