private void assertSameIdentifier(final SegmentIdWithShardSpec one, final SegmentIdWithShardSpec other) { Assert.assertEquals(one, other); Assert.assertEquals(one.getShardSpec().getPartitionNum(), other.getShardSpec().getPartitionNum()); if (one.getShardSpec().getClass() == NumberedShardSpec.class && other.getShardSpec().getClass() == NumberedShardSpec.class) { Assert.assertEquals( ((NumberedShardSpec) one.getShardSpec()).getPartitions(), ((NumberedShardSpec) other.getShardSpec()).getPartitions() ); } else if (one.getShardSpec().getClass() == LinearShardSpec.class && other.getShardSpec().getClass() == LinearShardSpec.class) { // do nothing } else { throw new ISE( "Unexpected shardSpecs [%s] and [%s]", one.getShardSpec().getClass(), other.getShardSpec().getClass() ); } } }
identifier.getInterval(), schema, identifier.getShardSpec(), identifier.getVersion(), tuningConfig.getMaxRowsInMemory(), currSink.getInterval(), currSink.getVersion(), identifier.getShardSpec().createChunk(currSink) );
private void addSegment(final SegmentIdWithShardSpec identifier) { segments.put(identifier.getInterval().getStartMillis(), identifier); try { segmentAnnouncer.announceSegment( new DataSegment( identifier.getDataSource(), identifier.getInterval(), identifier.getVersion(), ImmutableMap.of(), ImmutableList.of(), ImmutableList.of(), identifier.getShardSpec(), null, 0 ) ); } catch (IOException e) { log.makeAlert(e, "Failed to announce new segment[%s]", identifier.getDataSource()) .addData("interval", identifier.getInterval()) .emit(); } }
segmentIdentifier.getInterval(), segmentIdentifier.getVersion(), segmentIdentifier.getShardSpec().getPartitionNum() ), Execs.directExecutor(),
for (PartitionChunk<DataSegment> existing : existingHolder.getObject()) { if (maxId == null || maxId.getShardSpec().getPartitionNum() < existing.getObject().getShardSpec().getPartitionNum()) { maxId = SegmentIdWithShardSpec.fromDataSegment(existing.getObject()); pending.getVersion().compareTo(maxId.getVersion()) > 0 || (pending.getVersion().equals(maxId.getVersion()) && pending.getShardSpec().getPartitionNum() > maxId.getShardSpec().getPartitionNum())) { maxId = pending; ); return null; } else if (maxId.getShardSpec() instanceof LinearShardSpec) { return new SegmentIdWithShardSpec( dataSource, maxId.getInterval(), maxId.getVersion(), new LinearShardSpec(maxId.getShardSpec().getPartitionNum() + 1) ); } else if (maxId.getShardSpec() instanceof NumberedShardSpec) { return new SegmentIdWithShardSpec( dataSource, maxId.getVersion(), new NumberedShardSpec( maxId.getShardSpec().getPartitionNum() + 1, ((NumberedShardSpec) maxId.getShardSpec()).getPartitions() interval,
private Sink getOrCreateSink(final SegmentIdWithShardSpec identifier) { Sink retVal = sinks.get(identifier); if (retVal == null) { retVal = new Sink( identifier.getInterval(), schema, identifier.getShardSpec(), identifier.getVersion(), tuningConfig.getMaxRowsInMemory(), maxBytesTuningConfig, tuningConfig.isReportParseExceptions(), null ); try { segmentAnnouncer.announceSegment(retVal.getSegment()); } catch (IOException e) { log.makeAlert(e, "Failed to announce new segment[%s]", schema.getDataSource()) .addData("interval", retVal.getInterval()) .emit(); } sinks.put(identifier, retVal); metrics.setSinkCount(sinks.size()); sinkTimeline.add(retVal.getInterval(), retVal.getVersion(), identifier.getShardSpec().createChunk(retVal)); } return retVal; }
sink.getInterval(), sink.getVersion(), identifier.getShardSpec().createChunk(sink) ); for (FireHydrant hydrant : sink) {
ImmutableList.of(), ImmutableList.of(), id.getShardSpec(), 0,
@Test public void testSerde() throws Exception { final ObjectMapper objectMapper = new DefaultObjectMapper(); objectMapper.registerSubtypes(NumberedShardSpec.class); final SegmentIdWithShardSpec id2 = objectMapper.readValue( objectMapper.writeValueAsBytes(ID_1), SegmentIdWithShardSpec.class ); Assert.assertEquals(ID_1, id2); Assert.assertEquals(DATA_SOURCE, id2.getDataSource()); Assert.assertEquals(INTERVAL, id2.getInterval()); Assert.assertEquals(VERSION, id2.getVersion()); Assert.assertEquals(SHARD_SPEC_1.getPartitionNum(), id2.getShardSpec().getPartitionNum()); Assert.assertEquals(SHARD_SPEC_1.getPartitions(), ((NumberedShardSpec) id2.getShardSpec()).getPartitions()); }
IDENTIFIERS.get(2).getInterval(), IDENTIFIERS.get(2).getVersion(), IDENTIFIERS.get(2).getShardSpec().getPartitionNum() Intervals.of("2001/PT1H"), IDENTIFIERS.get(2).getVersion(), IDENTIFIERS.get(2).getShardSpec().getPartitionNum() Intervals.of("2001/PT1H"), IDENTIFIERS.get(2).getVersion(), IDENTIFIERS.get(2).getShardSpec().getPartitionNum() ), new SegmentDescriptor( Intervals.of("2001T03/PT1H"), IDENTIFIERS.get(2).getVersion(), IDENTIFIERS.get(2).getShardSpec().getPartitionNum()