@Override public Type map(Types.MapType map, Supplier<Type> keyTypeFuture, Supplier<Type> valueTypeFuture) { Preconditions.checkArgument(sourceType.isMapType(), "Not a map: " + sourceType); Types.MapType sourceMap = sourceType.asMapType(); int sourceKeyId = sourceMap.keyId(); int sourceValueId = sourceMap.valueId(); try { this.sourceType = sourceMap.keyType(); Type keyType = keyTypeFuture.get(); this.sourceType = sourceMap.valueType(); Type valueType = valueTypeFuture.get(); if (map.isValueOptional()) { return Types.MapType.ofOptional(sourceKeyId, sourceValueId, keyType, valueType); } else { return Types.MapType.ofRequired(sourceKeyId, sourceValueId, keyType, valueType); } } finally { this.sourceType = sourceMap; } }
return String.format("array<%s>", fromIcebergToHiveType(listType.elementType())); case MAP: final Types.MapType mapType = type.asMapType(); return String.format("map<%s,%s>", fromIcebergToHiveType(mapType.keyType()), fromIcebergToHiveType(mapType.valueType()));
return String.format("array<%s>", fromIcebergToHiveType(listType.elementType())); case MAP: final Types.MapType mapType = type.asMapType(); return String.format("map<%s,%s>", fromIcebergToHiveType(mapType.keyType()), fromIcebergToHiveType(mapType.valueType()));
return format("array<%s>", convert(listType.elementType())); case MAP: final Types.MapType mapType = type.asMapType(); return format("map<%s,%s>", convert(mapType.keyType()), convert(mapType.valueType())); default:
@Override public Type map(Types.MapType map, Supplier<Type> keyTypeFuture, Supplier<Type> valueTypeFuture) { Preconditions.checkArgument(sourceType.isMapType(), "Not a map: " + sourceType); Types.MapType sourceMap = sourceType.asMapType(); try { this.sourceType = sourceMap.keyType(); Type keyType = keyTypeFuture.get(); this.sourceType = sourceMap.valueType(); Type valueType = valueTypeFuture.get(); if (map.keyType() == keyType && map.valueType() == valueType) { return map; } if (map.isValueOptional()) { return Types.MapType.ofOptional(map.keyId(), map.valueId(), keyType, valueType); } else { return Types.MapType.ofRequired(map.keyId(), map.valueId(), keyType, valueType); } } finally { this.sourceType = sourceMap; } }
Types.NestedField valueField = null; if (iType != null) { map = iType.asMapType(); keyField = map.fields().get(0); valueField = map.fields().get(1);
MapType mapType = type.asMapType();
break; case MAP: assertEqualsMaps(prefix + ".elem " + e, childType.asMapType(), expected.getMap(e), toJavaMap((scala.collection.Map<?, ?>) actual.get(e))); break;
null == TypeUtil.find( schema, type -> type.isMapType() && type.asMapType().keyType() != Types.StringType.get()));
protected void writeAndValidate(Schema schema) throws IOException { Assume.assumeTrue("Parquet Avro cannot write non-string map keys", null == TypeUtil.find(schema, type -> type.isMapType() && type.asMapType().keyType() != Types.StringType.get())); List<GenericData.Record> expected = RandomData.generateList(schema, 100, 0L); File testFile = temp.newFile(); Assert.assertTrue("Delete should succeed", testFile.delete()); try (FileAppender<GenericData.Record> writer = Parquet.write(Files.localOutput(testFile)) .schema(schema) .named("test") .build()) { writer.addAll(expected); } try (CloseableIterable<InternalRow> reader = Parquet.read(Files.localInput(testFile)) .project(schema) .createReaderFunc(type -> SparkParquetReaders.buildReader(schema, type)) .build()) { Iterator<InternalRow> rows = reader.iterator(); for (int i = 0; i < expected.size(); i += 1) { Assert.assertTrue("Should have expected number of rows", rows.hasNext()); assertEqualsUnsafe(schema.asStruct(), expected.get(i), rows.next()); } Assert.assertFalse("Should not have extra rows", rows.hasNext()); } } }
break; case MAP: assertEqualsMaps(prefix + ".key=" + expectedKey, valueType.asMapType(), expectedValueArray.getMap(e), toJavaMap((scala.collection.Map<?, ?>) actual.get(expectedKey)));
break; case MAP: assertEqualsMaps(prefix + "." + fieldName, childType.asMapType(), expected.getMap(c), toJavaMap((scala.collection.Map<?, ?>) actual.getMap(c))); break;
Assert.assertTrue("Expected should be a Map", expected instanceof Map); Assert.assertTrue("Actual should be a Map", actual instanceof Map); assertEquals(type.asMapType(), (Map<?, ?>) expected, (Map<?, ?>) actual); break; default:
Assert.assertTrue("Expected should be a Map", expected instanceof Map); Assert.assertTrue("Actual should be a Map", actual instanceof Map); assertEquals(type.asMapType(), (Map<?, ?>) expected, (Map<?, ?>) actual); break; default:
Record l1 = GenericRecord.create(writeSchema.findType("locations").asMapType().valueType().asStructType()); l1.setField("lat", 53.992811f); l1.setField("long", -1.542616f); Record l2 = GenericRecord.create(writeSchema.findType("locations").asMapType().valueType().asStructType()); l2.setField("lat", 52.995143f); l2.setField("long", -1.539054f);