@Override public void close() throws Exception { if ( indexReader != null ) { indexReader.close(); } }
@Override public Iterator<Long> iterator() { Iterator<Document> iterator = documents.iterator(); return new Iterator<Long>() { @Override public boolean hasNext() { return iterator.hasNext(); } @Override public Long next() { return entityIdReader.applyAsLong( iterator.next() ); } }; }
@Override public long maxCount() { return documents.maxCount(); }
@Override protected void scan() { long recordsPerCPU = RecordDistributor.calculateRecordsPerCpu( store.maxCount(), numberOfThreads ); cacheAccess.prepareForProcessingOfSingleStore( recordsPerCPU ); QueueDistributor<RECORD> distributor = distribution.distributor( recordsPerCPU, numberOfThreads ); distributeRecords( numberOfThreads, getClass().getSimpleName() + "-" + name, DEFAULT_QUEUE_SIZE, store.iterator(), progress, processor, distributor ); } }
private static BoundedIterable<Long> mockedAllEntriesReader( boolean knownMaxCount, List<Long> entries ) { BoundedIterable<Long> mockedAllEntriesReader = mock( BoundedIterable.class ); when( mockedAllEntriesReader.maxCount() ).thenReturn( knownMaxCount ? entries.size() : BoundedIterable.UNKNOWN_MAX_COUNT ); when( mockedAllEntriesReader.iterator() ).thenReturn( entries.iterator() ); return mockedAllEntriesReader; }
@Override public long maxCount() { long sum = 0L; for ( BoundedIterable<Long> part : allEntriesReader ) { long partMaxCount = part.maxCount(); if ( partMaxCount == UNKNOWN_MAX_COUNT ) { return UNKNOWN_MAX_COUNT; } sum += partMaxCount; } return sum; }
@Override public Iterator<Long> iterator() { if ( indexReader == null ) { indexReader = indexAccessor.newAllEntriesReader(); } return indexReader.iterator(); } }
@Override public void close() throws Exception { documents.close(); }
@Override public long maxCount() { long sum = 0; for ( BoundedIterable entry : entries ) { long maxCount = entry.maxCount(); if ( maxCount == UNKNOWN_MAX_COUNT ) { return UNKNOWN_MAX_COUNT; } sum += maxCount; } return sum; }
@Test void shouldProcessRecordsSequentiallyAndUpdateProgress() throws Exception { // given ProgressMonitorFactory.MultiPartBuilder progressBuilder = mock( ProgressMonitorFactory.MultiPartBuilder.class ); ProgressListener progressListener = mock( ProgressListener.class ); when( progressBuilder.progressForPart( anyString(), anyLong() ) ).thenReturn( progressListener ); @SuppressWarnings( "unchecked" ) BoundedIterable<Integer> store = mock( BoundedIterable.class ); when( store.iterator() ).thenReturn( asList( 42, 75, 192 ).iterator() ); @SuppressWarnings( "unchecked" ) RecordProcessor<Integer> recordProcessor = mock( RecordProcessor.class ); RecordScanner<Integer> scanner = new SequentialRecordScanner<>( "our test task", Statistics.NONE, 1, store, progressBuilder, recordProcessor ); // when scanner.run(); // then verifyProcessCloseAndDone( recordProcessor, store, progressListener ); }
@Test public void allEntriesReaderMustCloseAll() throws Exception { // given BoundedIterable<Long>[] allEntriesReaders = Arrays.stream( aliveAccessors ) .map( accessor -> mockSingleAllEntriesReader( accessor, Arrays.asList() ) ) .toArray( BoundedIterable[]::new ); // when fusionIndexAccessor.newAllEntriesReader().close(); // then for ( BoundedIterable<Long> allEntriesReader : allEntriesReaders ) { verify( allEntriesReader, times( 1 ) ).close(); } }
@Override public long maxCount() { long sum = 0L; for ( BoundedIterable<Long> part : allEntriesReader ) { long partMaxCount = part.maxCount(); if ( partMaxCount == UNKNOWN_MAX_COUNT ) { return UNKNOWN_MAX_COUNT; } sum += partMaxCount; } return sum; }
@Test void shouldProcessRecordsParallelAndUpdateProgress() throws Exception { // given ProgressMonitorFactory.MultiPartBuilder progressBuilder = mock( ProgressMonitorFactory.MultiPartBuilder.class ); ProgressListener progressListener = mock( ProgressListener.class ); when( progressBuilder.progressForPart( anyString(), anyLong() ) ).thenReturn( progressListener ); @SuppressWarnings( "unchecked" ) BoundedIterable<Integer> store = mock( BoundedIterable.class ); when( store.iterator() ).thenReturn( asList( 42, 75, 192 ).iterator() ); @SuppressWarnings( "unchecked" ) RecordProcessor<Integer> recordProcessor = mock( RecordProcessor.class ); RecordScanner<Integer> scanner = new ParallelRecordScanner<>( "our test task", Statistics.NONE, 1, store, progressBuilder, recordProcessor, CacheAccess.EMPTY, QueueDistribution.ROUND_ROBIN ); // when scanner.run(); // then verifyProcessCloseAndDone( recordProcessor, store, progressListener ); }
private static void verifyProcessCloseAndDone( RecordProcessor<Integer> recordProcessor, BoundedIterable<Integer> store, ProgressListener progressListener ) throws Exception { verify( recordProcessor ).process( 42 ); verify( recordProcessor ).process( 75 ); verify( recordProcessor ).process( 192 ); verify( recordProcessor ).close(); verify( store ).close(); verify( progressListener, times( 3 ) ).add( 1 ); verify( progressListener ).done(); } }
@Override public long maxCount() { try ( BoundedIterable<Long> reader = indexAccessor.newAllEntriesReader() ) { return reader.maxCount(); } catch ( Exception e ) { throw new RuntimeException( e ); } }
@Test public void shouldScanMultipleRanges() { // GIVEN int labelId1 = 1; int labelId2 = 2; long nodeId1 = 10; long nodeId2 = 1280; start( asList( labelChanges( nodeId1, NO_LABELS, new long[]{labelId1} ), labelChanges( nodeId2, NO_LABELS, new long[]{labelId1, labelId2} ) ) ); // WHEN BoundedIterable<NodeLabelRange> reader = store.allNodeLabelRanges(); Iterator<NodeLabelRange> iterator = reader.iterator(); NodeLabelRange range1 = iterator.next(); NodeLabelRange range2 = iterator.next(); assertFalse( iterator.hasNext() ); // THEN assertArrayEquals( new long[]{nodeId1}, reducedNodes( range1 ) ); assertArrayEquals( new long[]{nodeId2}, reducedNodes( range2 ) ); assertArrayEquals( new long[]{labelId1}, sorted( range1.labels( nodeId1 ) ) ); assertArrayEquals( new long[]{labelId1, labelId2}, sorted( range2.labels( nodeId2 ) ) ); }
store.close();
RecordScanner( String name, Statistics statistics, int threads, BoundedIterable<RECORD> store, ProgressMonitorFactory.MultiPartBuilder builder, RecordProcessor<RECORD> processor, IterableStore... warmUpStores ) { super( name, statistics, threads ); this.store = store; this.processor = processor; long maxCount = store.maxCount(); this.progress = maxCount == -1 ? builder.progressForUnknownPart( name ) : builder.progressForPart( name, maxCount ); this.warmUpStores = warmUpStores; }
@Test public void shouldScanSingleRange() { // GIVEN int labelId1 = 1; int labelId2 = 2; long nodeId1 = 10; long nodeId2 = 11; start( asList( labelChanges( nodeId1, NO_LABELS, new long[]{labelId1} ), labelChanges( nodeId2, NO_LABELS, new long[]{labelId1, labelId2} ) ) ); // WHEN BoundedIterable<NodeLabelRange> reader = store.allNodeLabelRanges(); NodeLabelRange range = single( reader.iterator() ); // THEN assertArrayEquals( new long[]{nodeId1, nodeId2}, reducedNodes( range ) ); assertArrayEquals( new long[]{labelId1}, sorted( range.labels( nodeId1 ) ) ); assertArrayEquals( new long[]{labelId1, labelId2}, sorted( range.labels( nodeId2 ) ) ); }
@Test public void allEntriesReaderMustReportFusionMaxCountOfAll() { long lastId = 0; for ( IndexAccessor accessor : aliveAccessors ) { mockSingleAllEntriesReader( accessor, Arrays.asList( lastId++, lastId++ ) ); } // then BoundedIterable<Long> fusionAllEntriesReader = fusionIndexAccessor.newAllEntriesReader(); assertThat( fusionAllEntriesReader.maxCount(), is( lastId ) ); }