/** * Writes the objects to index. * * @param ba The data as a bytes array */ public void writeProcessedToIndex(BytesArray ba) { Assert.notNull(ba, "no data given"); Assert.isTrue(ba.length() > 0, "no data given"); lazyInitWriting(); trivialBytesRef.reset(); trivialBytesRef.add(ba); doWriteToIndex(trivialBytesRef); }
@Override public Writable serialize(Object data, ObjectInspector objInspector) throws SerDeException { lazyInitializeWrite(); // serialize the type directly to json (to avoid converting to Writable and then serializing) scratchPad.reset(); hiveType.setObjectInspector(objInspector); hiveType.setObject(data); // We use the command directly instead of the bulk entry writer since there is no close() method on SerDes. // See FileSinkOperator#process() for more info of how this is used with the output format. command.write(hiveType).copyTo(scratchPad); result.setContent(scratchPad); return result; }
private void processData(BulkProcessor processor) { BytesRef data = new BytesRef(); data.add(renderEntry("A")); processor.add(data); data.reset(); data.add(renderEntry("B")); processor.add(data); data.reset(); data.add(renderEntry("C")); processor.add(data); data.reset(); data.add(renderEntry("D")); processor.add(data); data.reset(); data.add(renderEntry("E")); processor.add(data); data.reset(); }
@Test public void testWriteBulkEntryWithHandlersThatCorrectsData() { BytesRef response = new BytesRef(); response.add("abcdefg".getBytes()); BulkCommand command = Mockito.mock(BulkCommand.class); Mockito.when(command.write(1)).thenThrow(new EsHadoopIllegalStateException("Things broke")); Mockito.when(command.write(10)).thenReturn(response); Settings settings = new TestSettings(); settings.setProperty(SerializationHandlerLoader.ES_WRITE_DATA_ERROR_HANDLERS, "fix"); settings.setProperty(SerializationHandlerLoader.ES_WRITE_DATA_ERROR_HANDLER+".fix", CorrectingHandler.class.getName()); BulkEntryWriter bulkEntryWriter = new BulkEntryWriter(settings, command); BytesRef value = bulkEntryWriter.writeBulkEntry(1); Assert.assertNotNull("Skipped values should be null", value); Assert.assertEquals(7, response.length()); Assert.assertArrayEquals("abcdefg".getBytes(), response.toString().getBytes()); }
public void add(BytesArrayPool baPool) { for (BytesArray pool : baPool.inUse()) { add(pool); } }
if (payload.length() > ba.available()) { if (autoFlush) { flush();
@Test public void testAddAndReset() throws Exception { BytesArrayPool pool = new BytesArrayPool(); pool.get().bytes("Test"); pool.get().bytes("Data"); pool.get().bytes("Rules"); BytesRef ref = new BytesRef(); assertEquals(13, pool.length()); ref.add(pool); assertEquals("TestDataRules", ref.toString()); BytesRef ref2 = new BytesRef(); pool.reset(); pool.get().bytes("New"); pool.get().bytes("Info"); assertEquals(7, pool.length()); ref2.add(pool); assertEquals("NewInfo", ref2.toString()); } }
/** postpone writing initialization since we can do only reading so there's no need to allocate buffers */ private void lazyInitWriting() { if (!writeInitialized) { this.writeInitialized = true; this.bulkProcessor = new BulkProcessor(client, resources.getResourceWrite(), settings); this.trivialBytesRef = new BytesRef(); this.bulkEntryWriter = new BulkEntryWriter(settings, BulkCommands.create(settings, metaExtractor, client.internalVersion)); } }
private void doWriteToIndex(BytesRef payload) { bulkProcessor.add(payload); payload.reset(); }
private void doWriteToIndex(BytesRef payload) { // check space first // ba is the backing array for data if (payload.length() > ba.available()) { if (autoFlush) { flush(); } else { throw new EsHadoopIllegalStateException( String.format("Auto-flush disabled and bulk buffer full; disable manual flush or increase capacity [current size %s]; bailing out", ba.capacity())); } } data.copyFrom(payload); payload.reset(); dataEntries++; if (bufferEntriesThreshold > 0 && dataEntries >= bufferEntriesThreshold) { if (autoFlush) { flush(); } else { // handle the corner case of manual flush that occurs only after the buffer is completely full (think size of 1) if (dataEntries > bufferEntriesThreshold) { throw new EsHadoopIllegalStateException( String.format( "Auto-flush disabled and maximum number of entries surpassed; disable manual flush or increase capacity [current size %s]; bailing out", bufferEntriesThreshold)); } } } }
@Test(expected = EsHadoopIllegalArgumentException.class) public void testIdMandatory() throws Exception { assumeTrue(ConfigurationOptions.ES_OPERATION_UPDATE.equals(operation)); Settings set = settings(); set.setProperty(ConfigurationOptions.ES_MAPPING_ID, ""); create(set).write(data).copyTo(ba); }
private void writeTemplate(Collection<Object> template, Object object) { for (Object item : template) { if (item instanceof BytesArray) { ref.add((BytesArray) item); } else if (item instanceof FieldWriter) { ref.add(((FieldWriter) item).write(object)); } // used in the dynamic case else if (item instanceof DynamicContentRef) { List<Object> dynamicContent = ((DynamicContentRef) item).getDynamicContent(); writeTemplate(dynamicContent, object); } else { throw new EsHadoopIllegalArgumentException(String.format("Unknown object type received [%s][%s]", item, item.getClass())); } } } }
data.copyFrom(newEntry); if (ba.available() < newEntry.length()) { trackingArrayExpanded = true;
/** postpone writing initialization since we can do only reading so there's no need to allocate buffers */ private void lazyInitWriting() { if (!writeInitialized) { this.writeInitialized = true; this.bulkProcessor = new BulkProcessor(client, resources.getResourceWrite(), settings); this.trivialBytesRef = new BytesRef(); this.bulkEntryWriter = new BulkEntryWriter(settings, BulkCommands.create(settings, metaExtractor, client.internalVersion)); } }
private void doWriteToIndex(BytesRef payload) { bulkProcessor.add(payload); payload.reset(); }
@Override public BytesRef write(Object object) { ref.reset(); scratchPad.reset(); Object processed = preProcess(object, scratchPad); // write before object writeTemplate(beforeObject, processed); // write object doWriteObject(processed, scratchPad, valueWriter); ref.add(scratchPad); // writer after object writeTemplate(afterObject, processed); return ref; }
@Test public void testNoHeader() throws Exception { assumeFalse(ConfigurationOptions.ES_OPERATION_UPDATE.equals(operation)); create(settings()).write(data).copyTo(ba); String result = prefix() + "}}" + map(); assertEquals(result, ba.toString()); }