module.exports = client => { const constant = require('./constant'), schedule = require('node-schedule'), size = 500, deleteDup = async () => { const count = { deleted: 0, total: 0 } let lastsort = undefined, result = undefined, last = undefined do { result = await client.search({ index: constant.index, size: size, body: { query: { match_all: {} }, sort: [{ "instance": { "order": "asc" }, "last": { "order": "desc", "numeric_type": "date_nanos", "format": "strict_date_optional_time_nanos" } }], search_after: lastsort } }) for (const status of result.hits.hits) { if (last && status._source && last.instance === status._source.instance) { await client.delete({ index: index, id: status._id }) count.deleted++ console.log('deleted ' + status._id + ': ' + status._source.instance) } else { last = status._source } if (status._id === result.hits.hits[result.hits.hits.length - 1]._id) { lastsort = status.sort } } count.total += result.hits.hits.length if (result.hits.hits.length !== size) { break } } while (result.hits && result.hits.hits && result.hits.hits.length > 0) return console.log('Index: ' + index + ' - Total: ' + count.total + ' - Deleted: ' + count.deleted) }, job = schedule.scheduleJob('0 ' + constant.taskdeletedup + ' * * *', async () => { await deleteDup() }) }