lib/apiswagger.js

@@ -3,6 +3,10 @@ const nodeinfo = require('activitypub-express/pub/nodeinfo')
 module.exports = (app, client) => {
     const constant = require('./constant'),
         zlib = require('zlib'),
+        { chain } = require('stream-chain'),
+        { parser } = require('stream-json'),
+        { stringer } = require('stream-json/jsonl/Stringer'),
+        { streamArray } = require('stream-json/streamers/StreamArray'),
         clean = str => {
             return str.replace(/[\/\\^$+?()`'¡¿¨!"·%&=;,\|\[\]{}]+/gmi, '')
         }
@@ -501,15 +505,28 @@ module.exports = (app, client) => {
     app.use('/api/download_index', async (req, res) => {
         try {
             res.setHeader('Content-Type', 'application/x-gzip')
-            res.setHeader('Content-disposition', 'attachment; filename=fediblock-index.json.gz')
+            res.setHeader('Content-disposition', 'attachment; filename=fediblock-index.jsonl.gz')
             const result = await client.search({
                 index: constant.index,
                 size: 9999,
                 query: {
                     match_all: {}
                 }
-            }, { asStream: true, meta: false })
-            result.pipe(zlib.createGzip()).pipe(res, { end: true })
+            }, { asStream: true })
+            result.pipe(chain([
+                parser(),
+                data => data.value.hits.hits,
+                streamArray(),
+                data => ({
+                    instance: data.value._source.instance,
+                    blocks: data.value._source.blocks,
+                    api: data.value._source.api,
+                    last: data.value._source.last,
+                    nodeinfo: data.value._source.nodeinfo
+                }),
+                stringer(),
+                zlib.createGzip()
+            ])).pipe(res, { end: true })
         } catch (e) {
             console.error(e)
             res.status(404).end()

package.json

@@ -25,6 +25,7 @@
     "node-schedule": "^2.1.1",
     "parcel": "^2.12.0",
     "rotating-file-stream": "^3.2.5",
+    "stream-json": "^1.8.0",
     "swagger-jsdoc": "^6.2.8",
     "swagger-ui-express": "^5.0.1"
   }

public/main.js

@@ -115,7 +115,7 @@ document.addEventListener('DOMContentLoaded', function () {
         fetch('/api/download_index', { signal: ac.signal }).then(async function (result) {
             var res = await result.blob(),
                 a = document.createElement('a')
-            a.download = 'fediblock-index.json.gz'
+            a.download = 'fediblock-index.jsonl.gz'
             a.href = URL.createObjectURL(res)
             a.type = 'application/x-gzip'
             a.target = '_blank'