@Override public int maxNumberOfProcessors() { return defaults.maxNumberOfProcessors(); }
public DynamicProcessorAssigner( Configuration config ) { super( 1, SECONDS ); this.config = config; this.availableProcessors = config.maxNumberOfProcessors(); }
@Override public int maxNumberOfProcessors() { return processors != null ? processors.intValue() : DEFAULT.maxNumberOfProcessors(); }
@Override public void start( int orderingGuarantees ) { super.start( orderingGuarantees ); this.executor = new DynamicTaskExecutor<>( 1, maxProcessors, config.maxNumberOfProcessors(), PARK, name(), Sender::new ); }
@Test void shouldOverrideMaxNumberOfProcessors() { shouldOverride( expected -> new Configuration() { @Override public int batchSize() { return expected; } }, Configuration::maxNumberOfProcessors, DEFAULT.maxNumberOfProcessors() + 1, DEFAULT.maxNumberOfProcessors() + 10 ); }
protected ForkedProcessorStep( StageControl control, String name, Configuration config, StatsProvider... statsProviders ) { super( control, name, config, statsProviders ); this.maxProcessors = config.maxNumberOfProcessors(); this.forkedProcessors = new Object[this.maxProcessors]; stripingLock = new StampedLock(); Unit noop = new Unit( -1, null, 0 ); head = new AtomicReference<>( noop ); tail = new AtomicReference<>( noop ); stripingLock.unlock( applyProcessorCount( stripingLock.readLock() ) ); downstreamSender = new CompletedBatchesSender( name + " [CompletedBatchSender]" ); maxQueueLength = 200 + maxProcessors; }
/** * Keeping all counts for all combinations of label/reltype can require a lot of memory if there are lots of those tokens. * Each processor will allocate such a data structure and so in extreme cases the number of processors will have to * be limited to not surpass the available memory limits. * * @param config {@link Configuration} holding things like max number of processors and max memory. * @param cache {@link NodeLabelsCache} which is the only other data structure occupying memory at this point. * @param highLabelId high label id for this store. * @param highRelationshipTypeId high relationship type id for this store. * @return number of processors suitable for this step. In most cases this will be 0, which is the typical value used * when just allowing the importer to grab up to {@link Configuration#maxNumberOfProcessors()}. The returned value * will at least be 1. */ private static int numberOfProcessors( Configuration config, NodeLabelsCache cache, int highLabelId, int highRelationshipTypeId ) { GatheringMemoryStatsVisitor memVisitor = new GatheringMemoryStatsVisitor(); cache.acceptMemoryStatsVisitor( memVisitor ); long availableMem = config.maxMemoryUsage() - memVisitor.getTotalUsage(); long threadMem = RelationshipCountsProcessor.calculateMemoryUsage( highLabelId, highRelationshipTypeId ); long possibleThreads = availableMem / threadMem; return possibleThreads >= config.maxNumberOfProcessors() ? 0 : toIntExact( max( 1, possibleThreads ) ); }
private ProcessRelationshipCountsDataStep instantiateStep( int highLabelId, int highRelationshipTypeId, long labelCacheSize, int maxProcessors, long maxMemory ) { StageControl control = new SimpleStageControl(); NodeLabelsCache cache = nodeLabelsCache( labelCacheSize ); Configuration config = mock( Configuration.class ); when( config.maxNumberOfProcessors() ).thenReturn( maxProcessors ); when( config.maxMemoryUsage() ).thenReturn( maxMemory ); return new ProcessRelationshipCountsDataStep( control, cache, config, highLabelId, highRelationshipTypeId, mock( CountsAccessor.Updater.class ), OFF_HEAP, INSTANCE ); }
@Override public int maxNumberOfProcessors() { return args.getNumber( ImportTool.Options.PROCESSORS.key(), DEFAULT.maxNumberOfProcessors() ).intValue(); }
/** * Imports nodes w/ their properties and labels from {@link Input#nodes()}. This will as a side-effect populate the {@link IdMapper}, * to later be used for looking up ID --> nodeId in {@link #importRelationships()}. After a completed node import, * {@link #prepareIdMapper()} must be called. * * @throws IOException on I/O error. */ public void importNodes() throws IOException { // Import nodes, properties, labels neoStore.startFlushingPageCache(); DataImporter.importNodes( config.maxNumberOfProcessors(), input, neoStore, idMapper, executionMonitor, storeUpdateMonitor ); neoStore.stopFlushingPageCache(); updatePeakMemoryUsage(); }
static void printOverview( File storeDir, Collection<Option<File[]>> nodesFiles, Collection<Option<File[]>> relationshipsFiles, org.neo4j.unsafe.impl.batchimport.Configuration configuration, PrintStream out ) { out.println( "Neo4j version: " + Version.getNeo4jVersion() ); out.println( "Importing the contents of these files into " + storeDir + ":" ); printInputFiles( "Nodes", nodesFiles, out ); printInputFiles( "Relationships", relationshipsFiles, out ); out.println(); out.println( "Available resources:" ); printIndented( "Total machine memory: " + bytes( OsBeanUtil.getTotalPhysicalMemory() ), out ); printIndented( "Free machine memory: " + bytes( OsBeanUtil.getFreePhysicalMemory() ), out ); printIndented( "Max heap memory : " + bytes( Runtime.getRuntime().maxMemory() ), out ); printIndented( "Processors: " + configuration.maxNumberOfProcessors(), out ); printIndented( "Configured max memory: " + bytes( configuration.maxMemoryUsage() ), out ); printIndented( "High-IO: " + configuration.highIO(), out ); out.println(); }
/** * Uses {@link IdMapper} as lookup for ID --> nodeId and imports all relationships from {@link Input#relationships()} * and writes them into the {@link RelationshipStore}. No linking between relationships is done in this method, * it's done later in {@link #linkRelationships(int)}. * * @throws IOException on I/O error. */ public void importRelationships() throws IOException { // Import relationships (unlinked), properties neoStore.startFlushingPageCache(); DataStatistics typeDistribution = DataImporter.importRelationships( config.maxNumberOfProcessors(), input, neoStore, idMapper, badCollector, executionMonitor, storeUpdateMonitor, !badCollector.isCollectingBadRelationships() ); neoStore.stopFlushingPageCache(); updatePeakMemoryUsage(); idMapper.close(); idMapper = null; putState( typeDistribution ); }
ExecutionMonitor processorAssigner = eagerRandomSaturation( config.maxNumberOfProcessors() ); DatabaseLayout databaseLayout = directory.databaseLayout( "dir" + random.nextAlphaNumericString( 8, 8 ) );
public DynamicProcessorAssigner( Configuration config ) { super( 1, SECONDS ); this.config = config; this.availableProcessors = config.maxNumberOfProcessors(); }
@Override public int maxNumberOfProcessors() { return defaults.maxNumberOfProcessors(); }
@Override public int maxNumberOfProcessors() { return processors != null ? processors.intValue() : DEFAULT.maxNumberOfProcessors(); }
@Override public void start( int orderingGuarantees ) { super.start( orderingGuarantees ); this.executor = new DynamicTaskExecutor<>( 1, maxProcessors, config.maxNumberOfProcessors(), PARK, name(), Sender::new ); }
protected ForkedProcessorStep( StageControl control, String name, Configuration config, StatsProvider... statsProviders ) { super( control, name, config, statsProviders ); this.maxProcessors = config.maxNumberOfProcessors(); this.forkedProcessors = new Object[this.maxProcessors]; stripingLock = new StampedLock(); Unit noop = new Unit( -1, null, 0 ); head = new AtomicReference<>( noop ); tail = new AtomicReference<>( noop ); stripingLock.unlock( applyProcessorCount( stripingLock.readLock() ) ); downstreamSender = new CompletedBatchesSender( name + " [CompletedBatchSender]" ); maxQueueLength = 200 + maxProcessors; }
/** * Imports nodes w/ their properties and labels from {@link Input#nodes()}. This will as a side-effect populate the {@link IdMapper}, * to later be used for looking up ID --> nodeId in {@link #importRelationships()}. After a completed node import, * {@link #prepareIdMapper()} must be called. * * @throws IOException on I/O error. */ public void importNodes() throws IOException { // Import nodes, properties, labels neoStore.startFlushingPageCache(); DataImporter.importNodes( config.maxNumberOfProcessors(), input, neoStore, idMapper, executionMonitor, storeUpdateMonitor ); neoStore.stopFlushingPageCache(); updatePeakMemoryUsage(); }
/** * Uses {@link IdMapper} as lookup for ID --> nodeId and imports all relationships from {@link Input#relationships()} * and writes them into the {@link RelationshipStore}. No linking between relationships is done in this method, * it's done later in {@link #linkRelationships(int)}. * * @throws IOException on I/O error. */ public void importRelationships() throws IOException { // Import relationships (unlinked), properties neoStore.startFlushingPageCache(); DataStatistics typeDistribution = DataImporter.importRelationships( config.maxNumberOfProcessors(), input, neoStore, idMapper, badCollector, executionMonitor, storeUpdateMonitor, !badCollector.isCollectingBadRelationships() ); neoStore.stopFlushingPageCache(); updatePeakMemoryUsage(); idMapper.close(); idMapper = null; putState( typeDistribution ); }