async bulk(body) { const response = await this.client .bulk({ body: body }); return (response.items.length > 0); }
_queueMetadata(callback) { if (this.recordTorrentQueue.length / 2 >= this.batchSizeTorrent) { this.client.bulk({ body: this.recordTorrentQueue }, function (err, resp) { }); this.recordTorrentQueue = []; console.log('Elasticsearch Class: Metadata Indexed') if (callback != null) callback(); } }
client.bulk({ body: batchData }, ( err, resp, status ) => { count+= docs.length; if ( count % 1000 === 0 ) { process.stdout.clearLine(); process.stdout.cursorTo(0); process.stdout.write( `${count} records imported` ); } callback(); });
async addOptionalFiles(siteId, optionals, modification) { if (optionals.length <= 0 || !siteId) return let operations = [] optionals.forEach(f => operations.push({ index: { _index: 'op_file' } }, { ...f, siteId }) ) const res = await this.client.bulk({ refresh: true, body: operations }) if (res.errors) signale.error(res) modification.op_files = await this.filterItemsToGetIds(siteId, "op_file") }
function pushToES(data, flag) { client.bulk({ body: data }, function (err, resp) { if (err) { console.error(err); return; } if (resp) { // console.log('Added ' + resp.items.length + ' records.'); if (flag === 'exit') { process.exit(0); } console.log(resp); return; } }); }
/** * Perform operations using bulk API * More info: https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html * @param {Object[]} data - must be in the format specified in the bulk API * @param {Object} [options] * @param {boolean} [options.noRefresh] - avoid refreshing the index on each iteration * @param {Function} [callback] - only in case callback style is used * @return {Promise} */ bulk (data, options, callback) { return this.client.bulk({ index: this.index, type: this.type, body: data, refresh: !(options && options.noRefresh) }).then(result => { if (callback && typeof callback === 'function') { return callback(null, result); } return result; }).catch(err => { if (callback && typeof callback === 'function') { return callback(err); } throw err; }); }
.bulk({ body }) .then(() => { console.log(`${fileName} has been indexed.`);
sendBulkRequest(bulkOp) { if (bulkOp.length === 0) { return; } return this.esClient.bulk({ refresh: false, body: bulkOp }).then(response => { if (!response.errors) { return; } response.items.forEach(item => { let erroredItem; if (item.delete && item.delete.error) { erroredItem = item.delete } else if (item.index && item.index.error) { erroredItem = item.index } if (erroredItem) { logger.error(`Bulk Request Error:`, erroredItem); if (erroredItem.error.type === 'routing_missing_exception') { logger.debug('This is most likely due do to a missing child parent relationship in the config. See default config file for reference.'); } } }); }).catch(err => { logger.error(`Bulk Error:`, err); }); }
var indexAllData = function(madebulk, callback) { client.bulk({ maxRetries: 1, index: 'world', type: 'country', body: madebulk, timeout: '5m' }, function(err, resp, status) { if (err) { console.log(err); } else { callback(resp.items); } }) }
_queueIP(callback) { if (this.recordRelationQueue.length / 2 >= this.batchSizeTorrent) { this.client.bulk({ body: this.recordRelationQueue }, function (err, resp) { }); this.recordRelationQueue = []; this.client.bulk({ body: this.recordIPQueue }, function (err, resp) { }); this.recordIPQueue = []; console.log('Elasticsearch Class: IP and Relation Indexed') this.client.bulk({ body: this.recordTorrentQueue }, function (err, resp) { }); this.recordTorrentQueue = []; console.log('Elasticsearch Class: Peers updated') if (callback != null) callback(); } }
_queueInfohash() { if (this.recordInfohashQueue.length / 2 >= this.batchSizeDHT) { this.client.bulk({ body: this.recordInfohashQueue }, function (err, resp) { }); this.recordInfohashQueue = []; console.log('Elasticsearch Class: Infohash Indexed') } }
/* data = { index: {_index: 'gov', _type: 'constituencies', _id: '1' } }, { "ConstituencyName": "Ipswich", "ConstituencyID": "E14000761", "ConstituencyType": "Borough" ... } */ function insertMany(ind, ty, data) { return client.bulk( { index: ind, type: ty, body: data }) }
async addFeeds(siteId, feeds, modification) { if (feeds.length <= 0 || !siteId) return let operations = [] feeds.forEach(f => operations.push({ index: { _index: 'feed' } }, { ...f, siteId }) ) const res = await this.client.bulk({ refresh: true, body: operations }) if (res.errors) signale.error(res) modification.feeds = await this.filterItemsToGetIds(siteId, "feed") }