public DimensionsSpec withDimensionExclusions(Set<String> dimExs) { return new DimensionsSpec( dimensions, ImmutableList.copyOf(Sets.union(dimensionExclusions, dimExs)), null ); }
@Deprecated public DimensionsSpec withSpatialDimensions(List<SpatialDimensionSchema> spatials) { return new DimensionsSpec(dimensions, ImmutableList.copyOf(dimensionExclusions), spatials); }
@PublicApi public DimensionsSpec withDimensions(List<DimensionSchema> dims) { return new DimensionsSpec(dims, ImmutableList.copyOf(dimensionExclusions), null); }
@JsonCreator public TimeAndDimsParseSpec( @JsonProperty("timestampSpec") TimestampSpec timestampSpec, @JsonProperty("dimensionsSpec") DimensionsSpec dimensionsSpec ) { super( timestampSpec != null ? timestampSpec : new TimestampSpec(null, null, null), dimensionsSpec != null ? dimensionsSpec : new DimensionsSpec(null, null, null) ); }
@JsonCreator public InfluxParseSpec( @JsonProperty("dimensionsSpec") DimensionsSpec dimensionsSpec, @JsonProperty("whitelistMeasurements") List<String> measurementWhitelist ) { super( new TimestampSpec(InfluxParser.TIMESTAMP_KEY, "millis", null), dimensionsSpec != null ? dimensionsSpec : new DimensionsSpec(null, null, null) ); this.measurementWhitelist = measurementWhitelist; }
public Parser getFlatParser() { JSONParseSpec spec = new JSONParseSpec( new TimestampSpec("ts", "iso", null), new DimensionsSpec(null, null, null), null, null ); return spec.makeParser(); }
public Parser getFieldDiscoveryParser() { List<JSONPathFieldSpec> fields = new ArrayList<>(); JSONPathSpec flattenSpec = new JSONPathSpec(true, fields); JSONParseSpec spec = new JSONParseSpec( new TimestampSpec("ts", "iso", null), new DimensionsSpec(null, null, null), flattenSpec, null ); return spec.makeParser(); }
@Test(expected = IllegalArgumentException.class) public void testColumnMissing() { @SuppressWarnings("unused") // expected exception final ParseSpec spec = new DelimitedParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec( DimensionsSpec.getDefaultSchemas(Arrays.asList("a", "b")), new ArrayList<>(), new ArrayList<>() ), ",", " ", Collections.singletonList("a"), false, 0 ); }
public Builder withDimensionsSpec(InputRowParser parser) { if (parser != null && parser.getParseSpec() != null && parser.getParseSpec().getDimensionsSpec() != null) { this.dimensionsSpec = parser.getParseSpec().getDimensionsSpec(); } else { this.dimensionsSpec = new DimensionsSpec(null, null, null); } return this; }
@Test(expected = IllegalArgumentException.class) public void testDimAndDimExcluOverlap() { @SuppressWarnings("unused") // expected exception final ParseSpec spec = new DelimitedParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec( DimensionsSpec.getDefaultSchemas(Arrays.asList("a", "B")), Collections.singletonList("B"), new ArrayList<>() ), ",", null, Arrays.asList("a", "B"), false, 0 ); }
@Test public void testDimExclusionDuplicate() { @SuppressWarnings("unused") // expected exception final ParseSpec spec = new DelimitedParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec( DimensionsSpec.getDefaultSchemas(Collections.singletonList("a")), Lists.newArrayList("B", "B"), new ArrayList<>() ), ",", null, Arrays.asList("a", "B"), false, 0 ); }
public static IncrementalIndex loadIncrementalIndex( final IncrementalIndex retVal, final CharSource source ) throws IOException { final StringInputRowParser parser = new StringInputRowParser( new DelimitedParseSpec( new TimestampSpec("ts", "iso", null), new DimensionsSpec(DIMENSION_SCHEMAS, null, null), "\t", "\u0001", Arrays.asList(COLUMNS), false, 0 ), "utf8" ); return loadIncrementalIndex(() -> retVal, source, parser); }
private static List<DimensionsSpec> getExpectedDimensionsSpecForAutoGeneration(boolean keepSegmentGranularity) { if (keepSegmentGranularity) { return ImmutableList.of( new DimensionsSpec(getDimensionSchema(new StringDimensionSchema("string_to_double"))), new DimensionsSpec(getDimensionSchema(new StringDimensionSchema("string_to_double"))), new DimensionsSpec(getDimensionSchema(new StringDimensionSchema("string_to_double"))), new DimensionsSpec(getDimensionSchema(new StringDimensionSchema("string_to_double"))), new DimensionsSpec(getDimensionSchema(new DoubleDimensionSchema("string_to_double"))), new DimensionsSpec(getDimensionSchema(new DoubleDimensionSchema("string_to_double"))) ); } else { return Collections.singletonList( new DimensionsSpec(getDimensionSchema(new DoubleDimensionSchema("string_to_double"))) ); } }
@Test public void testMakeParser() { final JavaScriptConfig config = JavaScriptConfig.getEnabledInstance(); JavaScriptParseSpec spec = new JavaScriptParseSpec( new TimestampSpec("abc", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Collections.singletonList("abc")), null, null), "function(str) { var parts = str.split(\"-\"); return { one: parts[0], two: parts[1] } }", config ); final Parser<String, Object> parser = spec.makeParser(); final Map<String, Object> obj = parser.parseToMap("x-y"); Assert.assertEquals(ImmutableMap.of("one", "x", "two", "y"), obj); }
@Test public void testMakeParserNotAllowed() { final JavaScriptConfig config = new JavaScriptConfig(false); JavaScriptParseSpec spec = new JavaScriptParseSpec( new TimestampSpec("abc", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Collections.singletonList("abc")), null, null), "abc", config ); expectedException.expect(IllegalStateException.class); expectedException.expectMessage("JavaScript is disabled"); spec.makeParser(); } }
@Test public void testSerde() throws IOException { RegexParseSpec spec = new RegexParseSpec( new TimestampSpec("abc", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Collections.singletonList("abc")), null, null), "\u0001", Collections.singletonList("abc"), "abc" ); final RegexParseSpec serde = (RegexParseSpec) jsonMapper.readValue( jsonMapper.writeValueAsString(spec), ParseSpec.class ); Assert.assertEquals("abc", serde.getTimestampSpec().getTimestampColumn()); Assert.assertEquals("iso", serde.getTimestampSpec().getTimestampFormat()); Assert.assertEquals("abc", serde.getPattern()); Assert.assertEquals("\u0001", serde.getListDelimiter()); Assert.assertEquals(Collections.singletonList("abc"), serde.getDimensionsSpec().getDimensionNames()); } }
private QueryableIndex persistAndLoad(List<DimensionSchema> schema, InputRow... rows) throws IOException { IncrementalIndex toPersist = IncrementalIndexTest.createIndex(null, new DimensionsSpec(schema, null, null)); for (InputRow row : rows) { toPersist.add(row); } final File tempDir = temporaryFolder.newFolder(); return closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null))); }
private IncrementalIndex getIndexWithDimsFromSchemata(List<DimensionSchema> dims) { IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder() .withDimensionsSpec(new DimensionsSpec(dims, null, null)) .withMetrics(new CountAggregatorFactory("count")) .build(); return new IncrementalIndex.Builder() .setIndexSchema(schema) .setMaxRowCount(1000) .buildOnheap(); }
private IncrementalIndex getIndexWithDims(List<String> dims) { IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder() .withDimensionsSpec(new DimensionsSpec(makeDimensionSchemas(dims), null, null)) .withMetrics(new CountAggregatorFactory("count")) .build(); return new IncrementalIndex.Builder() .setIndexSchema(schema) .setMaxRowCount(1000) .buildOnheap(); }
private static class TestFirehoseFactory implements FirehoseFactory<InputRowParser> { public TestFirehoseFactory() { } @Override @SuppressWarnings("unchecked") public Firehose connect(InputRowParser parser, File temporaryDirectory) throws ParseException { return new TestFirehose(parser); } }