@Override public void set(String v) { pageBuilder.setJson(column, ValueFactory.newString(v)); }
@Override public void set(Timestamp v) { pageBuilder.setJson(column, ValueFactory.newString(timestampFormatter.format(v))); }
public List<TaskReport> run(final TaskSource outputTask) { TransactionalPageOutput tran = runner.open(outputTask, schema, 1); boolean committed = false; try { ImmutableMapValue jsonValue = newMap( newString("_c1"), newBoolean(true), newString("_c2"), newInteger(10), newString("_c3"), newString("embulk"), newString("_c4"), newMap(newString("k"), newString("v")) ); for (Page page : PageTestUtils.buildPage( runtime.getBufferAllocator(), schema, true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), jsonValue, true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), jsonValue)) { tran.add(page); } tran.commit(); committed = true; } finally { if (!committed) { tran.abort(); } tran.close(); } return new ArrayList<TaskReport>(); } });
map = ((Value) record[0]).asMapValue().map(); assertEquals(newBoolean(true), map.get(newString("_c0"))); assertEquals(newInteger(10L), map.get(newString("_c1"))); assertEquals(newString("embulk"), map.get(newString("_c2"))); assertEquals(newMap(newString("k"), newString("v")), map.get(newString("_c3"))); map = ((Value) record[0]).asMapValue().map(); assertEquals(newBoolean(false), map.get(newString("_c0"))); assertEquals(newInteger(-10L), map.get(newString("_c1"))); assertEquals(newString("エンバルク"), map.get(newString("_c2"))); assertEquals(newArray(newString("e0"), newString("e1")), map.get(newString("_c3")));
@Test public void useSkipInvalidEscapeString() throws Exception { ConfigSource config = this.config.deepCopy().set("invalid_string_escapes", "SKIP"); transaction(config, fileInput( "{\"\\a\":\"b\"}\\" )); List<Object[]> records = Pages.toObjects(plugin.newSchema(), output.pages); assertEquals(1, records.size()); Object[] record = records.get(0); Map<Value, Value> map = ((Value) record[0]).asMapValue().map(); assertEquals(newString("b"), map.get(newString(""))); }
@Test public void useUnEscapeInvalidEscapeString() throws Exception { ConfigSource config = this.config.deepCopy().set("invalid_string_escapes", "UNESCAPE"); transaction(config, fileInput( "{\"\\a\":\"b\"}\\" )); List<Object[]> records = Pages.toObjects(plugin.newSchema(), output.pages); assertEquals(1, records.size()); Object[] record = records.get(0); Map<Value, Value> map = ((Value) record[0]).asMapValue().map(); assertEquals(newString("b"), map.get(newString("a"))); }
@Test public void testDoubleWriteJsonsToRow() { MockPageOutput output = new MockPageOutput(); Schema schema = Schema.builder() .add("col0", JSON) .add("col1", JSON) .add("col2", JSON) .build(); builder = new PageBuilder(bufferAllocator, schema, output); builder.setJson(0, newString("v0")); builder.setJson(1, newString("v1")); builder.setNull(2); builder.setJson(0, newString("v2")); // store to page for col0 builder.setNull(1); // null is stored to page for col1 builder.setJson(2, newString("v3")); // store to page for col2 builder.addRecord(); builder.finish(); builder.close(); reader = new PageReader(schema); reader.setPage(output.pages.get(0)); assertTrue(reader.nextRecord()); assertEquals(reader.getJson(0), newString("v2")); assertTrue(reader.isNull(1)); assertEquals(reader.getJson(2), newString("v3")); assertFalse(reader.nextRecord()); reader.close(); }
@Override public ImmutableStringValue immutableValue() { return ValueFactory.newString((byte[]) objectValue); }
public static ImmutableStringValue newString(byte[] b, int off, int len) { return newString(b, off, len, false); }
@Override public Value serialize( Object object, Function<Object, Value> serialize ) { return ValueFactory.newString( object.toString() ); } }
@Override protected String getStringFromValueState( String state, String key ) throws Exception { return MessagePack.newDefaultUnpacker( Base64.getDecoder().decode( state ) ) .unpackValue().asMapValue() .map().get( ValueFactory.newString( key ) ) .asStringValue().asString(); } }
@Test public void visitArray_columnsUsingBracketNotation() { PluginTask task = taskFromYamlString( "type: column", "columns:", " - {name: \"$['json1']['k1'][1]\", src: \"$['json1']['k1'][0]\"}", " - {name: \"$['json1']['k2'][0]\"}", " - {name: \"$['json1']['k3'][0]['k3']\", type: string, default: v}"); Schema inputSchema = Schema.builder() .add("json1", JSON) .build(); JsonVisitor subject = jsonVisitor(task, inputSchema); // {"k1":[{"k1":"v"},"v"],"k2":["v","v"]} Value k1 = ValueFactory.newString("k1"); Value k2 = ValueFactory.newString("k2"); Value v = ValueFactory.newString("v"); Value map = ValueFactory.newMap( k1, ValueFactory.newArray(ValueFactory.newMap(k1, v), v), k2, ValueFactory.newArray(v, v)); MapValue visited = subject.visit("$['json1']", map).asMapValue(); assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k2\":[\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString()); }
@Test public void visit_withDotAndBracket() { PluginTask task = taskFromYamlString( "type: column", "columns:", " - {name: \"$.json1['k_1']\"}", " - {name: \"$.json1['k_1'][0]['k_1']\"}", " - {name: \"$['json1']['k_2']\"}", " - {name: \"$['json1']['k_2']['k_2']\"}"); Schema inputSchema = Schema.builder() .add("json1", JSON) .build(); JsonVisitor subject = jsonVisitor(task, inputSchema); // {"k.1":[{"k.1":"v"}], "k.2":{"k.2":"v"}} Value k1 = ValueFactory.newString("k_1"); Value k2 = ValueFactory.newString("k_2"); Value v = ValueFactory.newString("v"); Value map = ValueFactory.newMap( k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)), k2, ValueFactory.newMap(k2, v)); MapValue visited = subject.visit("$['json1']", map).asMapValue(); assertEquals("{\"k_1\":[{\"k_1\":\"v\"}],\"k_2\":{\"k_2\":\"v\"}}", visited.toString()); }
@Test public void visit_withColumnNameIncludingSingleQuotes() { PluginTask task = taskFromYamlString( "type: column", "columns:", " - {name: \"$['\\\\'json1']['k1']\"}"); Schema inputSchema = Schema.builder() .add("'json1", JSON) .build(); JsonVisitor subject = jsonVisitor(task, inputSchema); // {"k1":"v"} Value k1 = ValueFactory.newString("k1"); Value v = ValueFactory.newString("v"); Value map = ValueFactory.newMap(k1, v); MapValue visited = subject.visit("$['\\'json1']", map).asMapValue(); assertEquals("{\"k1\":\"v\"}", visited.toString()); }
@Test public void visitArray_dropColumnsUsingBracketNotation() { PluginTask task = taskFromYamlString( "type: column", "drop_columns:", " - {name: \"$['json1']['k1'][0]['k1']\"}", " - {name: \"$['json1']['k2'][*]\"}"); // ending with [*] is allowed for drop_columns, but not for others Schema inputSchema = Schema.builder() .add("json1", JSON) .add("json2", JSON) .build(); JsonVisitor subject = jsonVisitor(task, inputSchema); // {"k1":[{"k1":"v"}[,"k2":["v","v"]} Value k1 = ValueFactory.newString("k1"); Value k2 = ValueFactory.newString("k2"); Value v = ValueFactory.newString("v"); Value map = ValueFactory.newMap( k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)), k2, ValueFactory.newArray(v, v)); MapValue visited = subject.visit("$['json1']", map).asMapValue(); assertEquals("{\"k1\":[{}],\"k2\":[]}", visited.toString()); }
@Test public void visitArray_addColumnsUsingBracketNotation() { PluginTask task = taskFromYamlString( "type: column", "add_columns:", " - {name: \"$['json1']['k1'][1]\", src: \"$['json1']['k1'][0]\"}", " - {name: \"$['json1']['k3'][0]['k3']\", type: string, default: v}"); Schema inputSchema = Schema.builder() .add("json1", JSON) .add("json2", JSON) .build(); JsonVisitor subject = jsonVisitor(task, inputSchema); // {"k1":[{"k1":"v"}],"k2":["v","v"]} Value k1 = ValueFactory.newString("k1"); Value k2 = ValueFactory.newString("k2"); Value v = ValueFactory.newString("v"); Value map = ValueFactory.newMap( k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)), k2, ValueFactory.newArray(v, v)); MapValue visited = subject.visit("$['json1']", map).asMapValue(); assertEquals("{\"k1\":[{\"k1\":\"v\"},{\"k1\":\"v\"}],\"k2\":[\"v\",\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString()); }
@Test public void getTailNameValue() { assertEquals(ValueFactory.newString("baz"), new JsonColumn("$['foo'].bar.baz", Types.BOOLEAN).getTailNameValue()); assertEquals(ValueFactory.newString("bar"), new JsonColumn("$.foo.bar", Types.BOOLEAN).getTailNameValue()); assertEquals(ValueFactory.newString("foo"), new JsonColumn("$.foo", Types.BOOLEAN).getTailNameValue()); assertEquals(ValueFactory.newNil(), new JsonColumn("$.foo[0][1]", Types.BOOLEAN).getTailNameValue()); assertEquals(ValueFactory.newNil(), new JsonColumn("$.foo[0]", Types.BOOLEAN).getTailNameValue()); assertEquals(ValueFactory.newNil(), new JsonColumn("$[0]", Types.BOOLEAN).getTailNameValue()); }
@Test public void ingestWithoutAck() throws IOException { Ingester ingester = new FluentdIngester(new FluentdIngester.Config(), fluentdSender); ingester.ingest(TAG, ByteBuffer.wrap(DATA)); verify(fluentdSender, times(1)).send(byteBuffersArgumentCaptor.capture()); List<ByteBuffer> byteBuffers = byteBuffersArgumentCaptor.getAllValues().get(0); byte[] ingested = getIngestedData(byteBuffers); MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(ingested); ImmutableArrayValue arrayValue = unpacker.unpackValue().asArrayValue(); assertEquals(3, arrayValue.size()); assertEquals(TAG, arrayValue.get(0).asStringValue().asString()); assertArrayEquals(DATA, arrayValue.get(1).asRawValue().asByteArray()); Map<Value, Value> options = arrayValue.get(2).asMapValue().map(); assertEquals(1, options.size()); assertEquals(DATA.length, options.get(ValueFactory.newString("size")).asIntegerValue().asInt()); }