stream-json out
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
ale 2024-09-29 09:41:37 +02:00
parent e95886eea2
commit ddfd30d157
3 changed files with 4 additions and 14 deletions

View File

@ -3,9 +3,6 @@ const nodeinfo = require('activitypub-express/pub/nodeinfo')
module.exports = (app, client) => { module.exports = (app, client) => {
const constant = require('./constant'), const constant = require('./constant'),
zlib = require('zlib'), zlib = require('zlib'),
Pick = require('stream-json/filters/Pick'),
{ streamArray } = require('stream-json/streamers/StreamArray'),
{ chain } = require('stream-chain'),
clean = str => { clean = str => {
return str.replace(/[/\\^$+?()`'¡¿¨!"·%&=;,\|\[\]{}]+/gmi, '') return str.replace(/[/\\^$+?()`'¡¿¨!"·%&=;,\|\[\]{}]+/gmi, '')
} }
@ -511,14 +508,8 @@ module.exports = (app, client) => {
query: { query: {
match_all: {} match_all: {}
} }
}, { asStream: true, meta: false }), }, { asStream: true, meta: false })
pipeline = chain([ result.pipe(zlib.createGzip()).pipe(res)
Pick.withParser({ filter: 'hits.hits' }),
streamArray(),
data => data.value._source,
zlib.createGzip()
])
result.pipe(pipeline).pipe(res)
} catch (e) { } catch (e) {
console.error(e) console.error(e)
res.status(404).end() res.status(404).end()

View File

@ -19,7 +19,7 @@ module.exports = async (client, apex, app) => {
requestPart = async uri => { requestPart = async uri => {
const ac = new AbortController(), const ac = new AbortController(),
timeout = setTimeout(() => { timeout = setTimeout(() => {
ac.abort() setImmediate(ac.abort())
}, constant.abort_timeout), }, constant.abort_timeout),
response = await fetch(uri, { response = await fetch(uri, {
headers: { 'User-Agent': constant.agent }, headers: { 'User-Agent': constant.agent },
@ -27,7 +27,7 @@ module.exports = async (client, apex, app) => {
keepalive: false, keepalive: false,
timeout: constant.timeout, timeout: constant.timeout,
}) })
clearTimeout(timeout) setImmediate(clearTimeout(timeout))
return await response.json() return await response.json()
}, },
scanInstance = async instance => { scanInstance = async instance => {

View File

@ -25,7 +25,6 @@
"node-schedule": "^2.1.1", "node-schedule": "^2.1.1",
"parcel": "^2.12.0", "parcel": "^2.12.0",
"rotating-file-stream": "^3.2.5", "rotating-file-stream": "^3.2.5",
"stream-json": "^1.8.0",
"swagger-jsdoc": "^6.2.8", "swagger-jsdoc": "^6.2.8",
"swagger-ui-express": "^5.0.1" "swagger-ui-express": "^5.0.1"
} }