token = parser.nextToken(); if (token == JsonToken.END_OBJECT) { return ValueFactory.newMap(map); } else if (token == null) { throw new JsonParseException(
public List<TaskReport> run(final TaskSource outputTask) { TransactionalPageOutput tran = runner.open(outputTask, schema, 1); boolean committed = false; try { ImmutableMapValue jsonValue = newMap( newString("_c1"), newBoolean(true), newString("_c2"), newInteger(10), newString("_c3"), newString("embulk"), newString("_c4"), newMap(newString("k"), newString("v")) ); for (Page page : PageTestUtils.buildPage( runtime.getBufferAllocator(), schema, true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), jsonValue, true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), jsonValue)) { tran.add(page); } tran.commit(); committed = true; } finally { if (!committed) { tran.abort(); } tran.close(); } return new ArrayList<TaskReport>(); } });
assertEquals(newInteger(10L), map.get(newString("_c1"))); assertEquals(newString("embulk"), map.get(newString("_c2"))); assertEquals(newMap(newString("k"), newString("v")), map.get(newString("_c3")));
public MapValue build() { return newMap(map); }
@SafeVarargs public static MapValue newMap(Map.Entry<? extends Value, ? extends Value>... pairs) { Value[] kvs = new Value[pairs.length * 2]; for (int i = 0; i < pairs.length; i += 2) { kvs[i * 2] = pairs[i].getKey(); kvs[i * 2 + 1] = pairs[i].getValue(); } return newMap(kvs, true); }
@Override public ImmutableMapValue immutableValue() { return ValueFactory.newMap(map()); }
public static ArrayList<JsonColumn> getAncestorJsonColumnList(String path) { ArrayList<JsonColumn> ancestorJsonColumnList = new ArrayList<>(); Path compiledPath; try { compiledPath = PathCompiler.compile(path); } catch (InvalidPathException e) { throw new ConfigException(String.format("jsonpath %s, %s", path, e.getMessage())); } StringBuilder partialPath = new StringBuilder("$"); PathToken parts = compiledPath.getRoot(); parts = parts.next(); // skip "$" while (! parts.isLeaf()) { partialPath.append(parts.getPathFragment()); PathToken next = parts.next(); JsonColumn jsonColumn; if (next instanceof ArrayPathToken || next instanceof WildcardPathToken) { jsonColumn = new JsonColumn(partialPath.toString(), Types.JSON, ValueFactory.newArray(new Value[0], false)); } else { jsonColumn = new JsonColumn(partialPath.toString(), Types.JSON, ValueFactory.newMap(new Value[0])); } ancestorJsonColumnList.add(jsonColumn); parts = next; } return ancestorJsonColumnList; }
@Test public void visit_withDotAndBracket() { PluginTask task = taskFromYamlString( "type: column", "columns:", " - {name: \"$.json1['k_1']\"}", " - {name: \"$.json1['k_1'][0]['k_1']\"}", " - {name: \"$['json1']['k_2']\"}", " - {name: \"$['json1']['k_2']['k_2']\"}"); Schema inputSchema = Schema.builder() .add("json1", JSON) .build(); JsonVisitor subject = jsonVisitor(task, inputSchema); // {"k.1":[{"k.1":"v"}], "k.2":{"k.2":"v"}} Value k1 = ValueFactory.newString("k_1"); Value k2 = ValueFactory.newString("k_2"); Value v = ValueFactory.newString("v"); Value map = ValueFactory.newMap( k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)), k2, ValueFactory.newMap(k2, v)); MapValue visited = subject.visit("$['json1']", map).asMapValue(); assertEquals("{\"k_1\":[{\"k_1\":\"v\"}],\"k_2\":{\"k_2\":\"v\"}}", visited.toString()); }
@Test public void visitMap_Columns() { PluginTask task = taskFromYamlString( "type: column", "columns:", " - {name: $.json1.k1}", " - {name: $.json1.k2.k2}", " - {name: $.json1.k3.k3, type: string, default: v}", " - {name: $.json1.k4, src: $.json1.k2}"); Schema inputSchema = Schema.builder() .add("json1", JSON) .add("json2", JSON) .build(); JsonVisitor subject = jsonVisitor(task, inputSchema); // {"k1":{"k1":"v"},"k2":{"k1":"v","k2":"v"}} Value k1 = ValueFactory.newString("k1"); Value k2 = ValueFactory.newString("k2"); Value v = ValueFactory.newString("v"); Value map = ValueFactory.newMap( k1, ValueFactory.newMap(k1, v), k2, ValueFactory.newMap(k2, v)); MapValue visited = subject.visit("$['json1']", map).asMapValue(); assertEquals("{\"k1\":{\"k1\":\"v\"},\"k2\":{\"k2\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString()); }
@Test public void visitMap_AddColumns() { PluginTask task = taskFromYamlString( "type: column", "add_columns:", " - {name: $.json1.k3.k3, type: string, default: v}", " - {name: $.json1.k4, src: $.json1.k2}"); Schema inputSchema = Schema.builder() .add("json1", JSON) .add("json2", JSON) .build(); JsonVisitor subject = jsonVisitor(task, inputSchema); // {"k1":{"k1":"v"},"k2":{"k2":"v"}} Value k1 = ValueFactory.newString("k1"); Value k2 = ValueFactory.newString("k2"); Value v = ValueFactory.newString("v"); Value map = ValueFactory.newMap( k1, ValueFactory.newMap(k1, v), k2, ValueFactory.newMap(k2, v)); MapValue visited = subject.visit("$['json1']", map).asMapValue(); assertEquals("{\"k1\":{\"k1\":\"v\"},\"k2\":{\"k2\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString()); }
@Test public void visitMap_dropColumnsUsingBracketNotation() { PluginTask task = taskFromYamlString( "type: column", "drop_columns:", " - {name: \"$['json1']['k1']['k1']\"}", " - {name: \"$['json1']['k2']\"}"); Schema inputSchema = Schema.builder() .add("json1", JSON) .add("json2", JSON) .build(); JsonVisitor subject = jsonVisitor(task, inputSchema); // {"k1":{"k1":"v"},"k2":{"k2":"v"}} Value k1 = ValueFactory.newString("k1"); Value k2 = ValueFactory.newString("k2"); Value v = ValueFactory.newString("v"); Value map = ValueFactory.newMap( k1, ValueFactory.newMap(k1, v), k2, ValueFactory.newMap(k2, v)); MapValue visited = subject.visit("$['json1']", map).asMapValue(); assertEquals("{\"k1\":{}}", visited.toString()); }
@Test public void visitMap_DropColumns() { PluginTask task = taskFromYamlString( "type: column", "drop_columns:", " - {name: $.json1.k1.k1}", " - {name: $.json1.k2}"); Schema inputSchema = Schema.builder() .add("json1", JSON) .add("json2", JSON) .build(); JsonVisitor subject = jsonVisitor(task, inputSchema); // {"k1":{"k1":"v"},"k2":{"k2":"v"}} Value k1 = ValueFactory.newString("k1"); Value k2 = ValueFactory.newString("k2"); Value v = ValueFactory.newString("v"); Value map = ValueFactory.newMap( k1, ValueFactory.newMap(k1, v), k2, ValueFactory.newMap(k2, v)); MapValue visited = subject.visit("$['json1']", map).asMapValue(); assertEquals("{\"k1\":{}}", visited.toString()); }
@Test public void visitMap_addColumnsUsingBracketNotation() { PluginTask task = taskFromYamlString( "type: column", "add_columns:", " - {name: \"$['json1']['k3']['k3']\", type: string, default: v}", " - {name: \"$['json1']['k4']\", src: \"$['json1']['k2']\"}"); Schema inputSchema = Schema.builder() .add("json1", JSON) .add("json2", JSON) .build(); JsonVisitor subject = jsonVisitor(task, inputSchema); // {"k1":{"k1":"v"},"k2":{"k2":"v"}} Value k1 = ValueFactory.newString("k1"); Value k2 = ValueFactory.newString("k2"); Value v = ValueFactory.newString("v"); Value map = ValueFactory.newMap( k1, ValueFactory.newMap(k1, v), k2, ValueFactory.newMap(k2, v)); MapValue visited = subject.visit("$['json1']", map).asMapValue(); assertEquals("{\"k1\":{\"k1\":\"v\"},\"k2\":{\"k2\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString()); }
@Test public void visit_withSingleQuotesAndDoubleQuotes() { PluginTask task = taskFromYamlString( "type: column", "columns:", " - {name: \"$['json1']['k_1']\", src: \"$['json1']['k.1']\"}", " - {name: '$[\"json1\"][\"k_1\"][0][\"k_1\"]', src: '$[\"json1\"][\"k_1\"][0][\"k.1\"]'}", " - {name: '$[\"json1\"][\"k_2\"]', src: '$[\"json1\"][\"k.2\"]'}", " - {name: '$[\"json1\"][\"k_2\"][\"k_2\"]', src: '$[\"json1\"][\"k_2\"][\"k.2\"]'}"); Schema inputSchema = Schema.builder() .add("json1", JSON) .build(); JsonVisitor subject = jsonVisitor(task, inputSchema); // {"k.1":[{"k.1":"v"}], "k.2":{"k.2":"v"}} Value k1 = ValueFactory.newString("k.1"); Value k2 = ValueFactory.newString("k.2"); Value v = ValueFactory.newString("v"); Value map = ValueFactory.newMap( k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)), k2, ValueFactory.newMap(k2, v)); MapValue visited = subject.visit("$['json1']", map).asMapValue(); assertEquals("{\"k_1\":[{\"k_1\":\"v\"}],\"k_2\":{\"k_2\":\"v\"}}", visited.toString()); }
@Test public void visitArray_AddColumns() { PluginTask task = taskFromYamlString( "type: column", "add_columns:", " - {name: \"$.json1.k1[1]\", src: \"$.json1.k1[0]\"}", " - {name: \"$.json1.k3[*].k2\", type: string, default: v}", " - {name: \"$.json1.k4[*].k1\", type: string, default: v}", " - {name: \"$.json1.k5[0].k1\", type: string, default: v}"); Schema inputSchema = Schema.builder() .add("json1", JSON) .add("json2", JSON) .build(); JsonVisitor subject = jsonVisitor(task, inputSchema); // {"k1":[{"k1":"v"}],"k2":["v","v"],"k3":[{"k1":"v"}]} Value k1 = ValueFactory.newString("k1"); Value k2 = ValueFactory.newString("k2"); Value k3 = ValueFactory.newString("k3"); Value v = ValueFactory.newString("v"); Value map = ValueFactory.newMap( k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)), k2, ValueFactory.newArray(v, v), k3, ValueFactory.newArray(ValueFactory.newMap(k1, v))); MapValue visited = subject.visit("$['json1']", map).asMapValue(); assertEquals("{\"k1\":[{\"k1\":\"v\"},{\"k1\":\"v\"}],\"k2\":[\"v\",\"v\"],\"k3\":[{\"k1\":\"v\",\"k2\":\"v\"}],\"k4\":[],\"k5\":[{\"k1\":\"v\"}]}", visited.toString()); }
@Test public void visitArray_columnsUsingBracketNotation() { PluginTask task = taskFromYamlString( "type: column", "columns:", " - {name: \"$['json1']['k1'][1]\", src: \"$['json1']['k1'][0]\"}", " - {name: \"$['json1']['k2'][0]\"}", " - {name: \"$['json1']['k3'][0]['k3']\", type: string, default: v}"); Schema inputSchema = Schema.builder() .add("json1", JSON) .build(); JsonVisitor subject = jsonVisitor(task, inputSchema); // {"k1":[{"k1":"v"},"v"],"k2":["v","v"]} Value k1 = ValueFactory.newString("k1"); Value k2 = ValueFactory.newString("k2"); Value v = ValueFactory.newString("v"); Value map = ValueFactory.newMap( k1, ValueFactory.newArray(ValueFactory.newMap(k1, v), v), k2, ValueFactory.newArray(v, v)); MapValue visited = subject.visit("$['json1']", map).asMapValue(); assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k2\":[\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString()); }
@Test public void visitArray_addColumnsUsingBracketNotation() { PluginTask task = taskFromYamlString( "type: column", "add_columns:", " - {name: \"$['json1']['k1'][1]\", src: \"$['json1']['k1'][0]\"}", " - {name: \"$['json1']['k3'][0]['k3']\", type: string, default: v}"); Schema inputSchema = Schema.builder() .add("json1", JSON) .add("json2", JSON) .build(); JsonVisitor subject = jsonVisitor(task, inputSchema); // {"k1":[{"k1":"v"}],"k2":["v","v"]} Value k1 = ValueFactory.newString("k1"); Value k2 = ValueFactory.newString("k2"); Value v = ValueFactory.newString("v"); Value map = ValueFactory.newMap( k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)), k2, ValueFactory.newArray(v, v)); MapValue visited = subject.visit("$['json1']", map).asMapValue(); assertEquals("{\"k1\":[{\"k1\":\"v\"},{\"k1\":\"v\"}],\"k2\":[\"v\",\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString()); }
@Test public void visitArray_dropColumnsUsingBracketNotation() { PluginTask task = taskFromYamlString( "type: column", "drop_columns:", " - {name: \"$['json1']['k1'][0]['k1']\"}", " - {name: \"$['json1']['k2'][*]\"}"); // ending with [*] is allowed for drop_columns, but not for others Schema inputSchema = Schema.builder() .add("json1", JSON) .add("json2", JSON) .build(); JsonVisitor subject = jsonVisitor(task, inputSchema); // {"k1":[{"k1":"v"}[,"k2":["v","v"]} Value k1 = ValueFactory.newString("k1"); Value k2 = ValueFactory.newString("k2"); Value v = ValueFactory.newString("v"); Value map = ValueFactory.newMap( k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)), k2, ValueFactory.newArray(v, v)); MapValue visited = subject.visit("$['json1']", map).asMapValue(); assertEquals("{\"k1\":[{}],\"k2\":[]}", visited.toString()); }
@Test public void visit_withColumnNameIncludingSingleQuotes() { PluginTask task = taskFromYamlString( "type: column", "columns:", " - {name: \"$['\\\\'json1']['k1']\"}"); Schema inputSchema = Schema.builder() .add("'json1", JSON) .build(); JsonVisitor subject = jsonVisitor(task, inputSchema); // {"k1":"v"} Value k1 = ValueFactory.newString("k1"); Value v = ValueFactory.newString("v"); Value map = ValueFactory.newMap(k1, v); MapValue visited = subject.visit("$['\\'json1']", map).asMapValue(); assertEquals("{\"k1\":\"v\"}", visited.toString()); }