public static List<DimensionSchema> getDefaultSchemas( final List<String> dimNames, final DimensionSchema.MultiValueHandling multiValueHandling ) { return dimNames.stream() .map(input -> new StringDimensionSchema(input, multiValueHandling, true)) .collect(Collectors.toList()); }
@Test(expected = IllegalArgumentException.class) public void testComma() { @SuppressWarnings("unused") // expected exception final ParseSpec spec = new DelimitedParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec( DimensionsSpec.getDefaultSchemas(Arrays.asList("a,", "b")), new ArrayList<>(), new ArrayList<>() ), ",", null, Collections.singletonList("a"), false, 0 ); }
@Test(expected = IllegalArgumentException.class) public void testColumnMissing() { @SuppressWarnings("unused") // expected exception final ParseSpec spec = new CSVParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec( DimensionsSpec.getDefaultSchemas(Arrays.asList("a", "b")), new ArrayList<>(), new ArrayList<>() ), ",", Collections.singletonList("a"), false, 0 ); }
@Test(expected = IllegalArgumentException.class) public void testColumnMissing() { @SuppressWarnings("unused") // expected exception final ParseSpec spec = new DelimitedParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec( DimensionsSpec.getDefaultSchemas(Arrays.asList("a", "b")), new ArrayList<>(), new ArrayList<>() ), ",", " ", Collections.singletonList("a"), false, 0 ); }
@Test(expected = IllegalArgumentException.class) public void testComma() { @SuppressWarnings("unused") // expected exception final ParseSpec spec = new CSVParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec( DimensionsSpec.getDefaultSchemas(Arrays.asList("a,", "b")), new ArrayList<>(), new ArrayList<>() ), ",", Collections.singletonList("a"), false, 0 ); } }
@Test(expected = ParseException.class) public void testDuplicateNames() { @SuppressWarnings("unused") // expected exception final ParseSpec spec = new DelimitedParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec( DimensionsSpec.getDefaultSchemas(Arrays.asList("a", "b", "a")), new ArrayList<>(), new ArrayList<>() ), ",", " ", Arrays.asList("a", "b"), false, 0 ); }
@Test public void testDefaultTimestampSpec() { expectedException.expect(NullPointerException.class); expectedException.expectMessage("parseSpec requires timestampSpec"); @SuppressWarnings("unused") // expected exception final ParseSpec spec = new DelimitedParseSpec( null, new DimensionsSpec( DimensionsSpec.getDefaultSchemas(Collections.singletonList("a")), Lists.newArrayList("B", "B"), new ArrayList<>() ), ",", null, Arrays.asList("a", "B"), false, 0 ); }
@Test(expected = IllegalArgumentException.class) public void testDimAndDimExcluOverlap() { @SuppressWarnings("unused") // expected exception final ParseSpec spec = new DelimitedParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec( DimensionsSpec.getDefaultSchemas(Arrays.asList("a", "B")), Collections.singletonList("B"), new ArrayList<>() ), ",", null, Arrays.asList("a", "B"), false, 0 ); }
@Test public void testDimExclusionDuplicate() { @SuppressWarnings("unused") // expected exception final ParseSpec spec = new DelimitedParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec( DimensionsSpec.getDefaultSchemas(Collections.singletonList("a")), Lists.newArrayList("B", "B"), new ArrayList<>() ), ",", null, Arrays.asList("a", "B"), false, 0 ); }
@Test(expected = IllegalArgumentException.class) public void testDefaultColumnList() { @SuppressWarnings("unused") // expected exception final DelimitedParseSpec spec = new DelimitedParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec( DimensionsSpec.getDefaultSchemas(Arrays.asList("a", "b")), new ArrayList<>(), new ArrayList<>() ), ",", null, null, false, 0 ); } }
@Test public void testLowercasing() { JSONLowercaseParseSpec spec = new JSONLowercaseParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec( DimensionsSpec.getDefaultSchemas(Arrays.asList("A", "B")), new ArrayList<>(), new ArrayList<>() ) ); Parser parser = spec.makeParser(); Map<String, Object> event = parser.parseToMap("{\"timestamp\":\"2015-01-01\",\"A\":\"foo\"}"); Assert.assertEquals("foo", event.get("a")); } }
public static List<InputRow> inputRowsWithDimensions(final List<String> dimensions) { final MapInputRowParser parser = new MapInputRowParser( new TimeAndDimsParseSpec( new TimestampSpec(TIMESTAMP_COLUMN, "iso", null), new DimensionsSpec( DimensionsSpec.getDefaultSchemas(dimensions), dimensions.isEmpty() ? ImmutableList.of("t", "c1", "c2") : null, null ) ) ); return ImmutableList.of( parser.parseBatch(ImmutableMap.of("t", "2000-01-01", "c1", "9", "c2", ImmutableList.of("a"))).get(0), parser.parseBatch(ImmutableMap.of("t", "2000-01-02", "c1", "10.1", "c2", ImmutableList.of())).get(0), parser.parseBatch(ImmutableMap.of("t", "2000-01-03", "c1", "2", "c2", ImmutableList.of(""))).get(0), parser.parseBatch(ImmutableMap.of("t", "2001-01-01", "c1", "1", "c2", ImmutableList.of("a", "c"))).get(0), parser.parseBatch(ImmutableMap.of("t", "2001-01-02", "c1", "4", "c2", ImmutableList.of("abc"))).get(0), parser.parseBatch(ImmutableMap.of("t", "2001-01-03", "c1", "5")).get(0) ); }
private InputRow testCharsetParseHelper(Charset charset) throws Exception { final StringInputRowParser parser = new StringInputRowParser( new JSONParseSpec( new TimestampSpec("timestamp", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("foo", "bar")), null, null), null, null ), charset.name() ); final ByteBufferInputRowParser parser2 = jsonMapper.readValue( jsonMapper.writeValueAsBytes(parser), ByteBufferInputRowParser.class ); final InputRow parsed = parser2.parseBatch( ByteBuffer.wrap( "{\"foo\":\"x\",\"bar\":\"y\",\"qux\":\"z\",\"timestamp\":\"3000\"}".getBytes(charset) ) ).get(0); return parsed; }
@Test public void testMakeParser() { final JavaScriptConfig config = JavaScriptConfig.getEnabledInstance(); JavaScriptParseSpec spec = new JavaScriptParseSpec( new TimestampSpec("abc", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Collections.singletonList("abc")), null, null), "function(str) { var parts = str.split(\"-\"); return { one: parts[0], two: parts[1] } }", config ); final Parser<String, Object> parser = spec.makeParser(); final Map<String, Object> obj = parser.parseToMap("x-y"); Assert.assertEquals(ImmutableMap.of("one", "x", "two", "y"), obj); }
@Test public void testSerde() throws IOException { HashMap<String, Boolean> feature = new HashMap<String, Boolean>(); feature.put("ALLOW_UNQUOTED_CONTROL_CHARS", true); JSONParseSpec spec = new JSONParseSpec( new TimestampSpec("timestamp", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("bar", "foo")), null, null), null, feature ); final JSONParseSpec serde = (JSONParseSpec) jsonMapper.readValue( jsonMapper.writeValueAsString(spec), ParseSpec.class ); Assert.assertEquals("timestamp", serde.getTimestampSpec().getTimestampColumn()); Assert.assertEquals("iso", serde.getTimestampSpec().getTimestampFormat()); Assert.assertEquals(Arrays.asList("bar", "foo"), serde.getDimensionsSpec().getDimensionNames()); Assert.assertEquals(feature, serde.getFeatureSpec()); } }
@Test public void testDisableJavaScript() { final JavaScriptParseSpec parseSpec = new JavaScriptParseSpec( new TimestampSpec("timestamp", "auto", null), new DimensionsSpec( DimensionsSpec.getDefaultSchemas( ImmutableList.of( "dim1", "dim2" ) ), null, null ), "func", new JavaScriptConfig(false) ); final StringInputRowParser parser = new StringInputRowParser(parseSpec, "UTF-8"); expectedException.expect(CoreMatchers.instanceOf(IllegalStateException.class)); expectedException.expectMessage("JavaScript is disabled"); parser.startFileFromBeginning(); }
@Test public void testMakeParserNotAllowed() { final JavaScriptConfig config = new JavaScriptConfig(false); JavaScriptParseSpec spec = new JavaScriptParseSpec( new TimestampSpec("abc", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Collections.singletonList("abc")), null, null), "abc", config ); expectedException.expect(IllegalStateException.class); expectedException.expectMessage("JavaScript is disabled"); spec.makeParser(); } }
@Test public void testSerde() throws IOException { RegexParseSpec spec = new RegexParseSpec( new TimestampSpec("abc", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Collections.singletonList("abc")), null, null), "\u0001", Collections.singletonList("abc"), "abc" ); final RegexParseSpec serde = (RegexParseSpec) jsonMapper.readValue( jsonMapper.writeValueAsString(spec), ParseSpec.class ); Assert.assertEquals("abc", serde.getTimestampSpec().getTimestampColumn()); Assert.assertEquals("iso", serde.getTimestampSpec().getTimestampFormat()); Assert.assertEquals("abc", serde.getPattern()); Assert.assertEquals("\u0001", serde.getListDelimiter()); Assert.assertEquals(Collections.singletonList("abc"), serde.getDimensionsSpec().getDimensionNames()); } }
@Test public void testSerde() throws IOException { DelimitedParseSpec spec = new DelimitedParseSpec( new TimestampSpec("abc", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Collections.singletonList("abc")), null, null), "\u0001", "\u0002", Collections.singletonList("abc"), false, 0 ); final DelimitedParseSpec serde = jsonMapper.readValue( jsonMapper.writeValueAsString(spec), DelimitedParseSpec.class ); Assert.assertEquals("abc", serde.getTimestampSpec().getTimestampColumn()); Assert.assertEquals("iso", serde.getTimestampSpec().getTimestampFormat()); Assert.assertEquals(Collections.singletonList("abc"), serde.getColumns()); Assert.assertEquals("\u0001", serde.getDelimiter()); Assert.assertEquals("\u0002", serde.getListDelimiter()); Assert.assertEquals(Collections.singletonList("abc"), serde.getDimensionsSpec().getDimensionNames()); }
@Test public void testgetDimensions() { final IncrementalIndex<Aggregator> incrementalIndex = new IncrementalIndex.Builder() .setIndexSchema( new IncrementalIndexSchema.Builder() .withMetrics(new CountAggregatorFactory("count")) .withDimensionsSpec( new DimensionsSpec( DimensionsSpec.getDefaultSchemas(Arrays.asList("dim0", "dim1")), null, null ) ) .build() ) .setMaxRowCount(1000000) .buildOnheap(); closerRule.closeLater(incrementalIndex); Assert.assertEquals(Arrays.asList("dim0", "dim1"), incrementalIndex.getDimensionNames()); }