diff --git a/lib/apiswagger.js b/lib/apiswagger.js index dd94694..afb36b7 100644 --- a/lib/apiswagger.js +++ b/lib/apiswagger.js @@ -3,6 +3,10 @@ const nodeinfo = require('activitypub-express/pub/nodeinfo') module.exports = (app, client) => { const constant = require('./constant'), zlib = require('zlib'), + { pick } = require('stream-json/filters/Pick'), + { parser } = require('stream-json'), + { streamArray } = require('stream-json/streamers/StreamArray'), + { chain } = require('stream-chain'), clean = str => { return str.replace(/[/\\^$+?()`'¡¿¨!"·%&=;,\|\[\]{}]+/gmi, '') } @@ -472,15 +476,24 @@ module.exports = (app, client) => { } } } - }) - const instances = result.hits?.hits?.length > 0 ? result.hits.hits : [], - instancescomment = instances.map(i => ({ - instance: i._source.instance, comment: i._source.blocks.find(block => block.domain === clean(req.params.instance)).comment - })) + }, { asStream: true, meta: false }), + instancescomment = [], + block_count = 0, + pipeline = chain([ + parser(), + pick({ filter: 'hits.hits' }), + streamArray(), + data => { + block_count++ + instancescomment.push({ + instance: data.value._source.instance, comment: data.value._source.blocks.find(block => block.domain === clean(req.params.instance)).comment + }) + } + ]) + result.pipe(pipeline) res.json({ - block_count: instances.length, - instances: instancescomment, - took: result.took + block_count, + instances: instancescomment }) } else { res.status(404).end() @@ -501,15 +514,22 @@ module.exports = (app, client) => { app.use('/api/download_index', async (req, res) => { try { res.setHeader('Content-Type', 'application/gzip') - res.setHeader('Content-disposition', 'attachment; filename=fediblock-index.json.gz') + res.setHeader('Content-disposition', 'attachment; filename=fediblock-index.jsonl.gz') const result = await client.search({ index: constant.index, size: 9999, query: { match_all: {} } - }, { asStream: true, meta: false }) - result.pipe(zlib.createGzip()).pipe(res) + }, { asStream: true, meta: false }), + pipeline = chain([ + parser(), + pick({ filter: 'hits.hits' }), + streamArray(), + data => JSON.stringify(data.value._source) + '\n', + zlib.createGzip() + ]) + result.pipe(pipeline).pipe(res) } catch (e) { console.error(e) res.status(404).end() diff --git a/package.json b/package.json index aa1501c..d89f769 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "node-schedule": "^2.1.1", "parcel": "^2.12.0", "rotating-file-stream": "^3.2.5", + "stream-json": "^1.8.0", "swagger-jsdoc": "^6.2.8", "swagger-ui-express": "^5.0.1" } diff --git a/public/index.html b/public/index.html index b6c63fb..9469cbf 100644 --- a/public/index.html +++ b/public/index.html @@ -74,7 +74,7 @@
public instances are block
-
(search in ms)
+