@Override public int maxNumberOfProcessors() { return defaults.maxNumberOfProcessors(); }
/** * Keeping all counts for all combinations of label/reltype can require a lot of memory if there are lots of those tokens. * Each processor will allocate such a data structure and so in extreme cases the number of processors will have to * be limited to not surpass the available memory limits. * * @param config {@link Configuration} holding things like max number of processors and max memory. * @param cache {@link NodeLabelsCache} which is the only other data structure occupying memory at this point. * @param highLabelId high label id for this store. * @param highRelationshipTypeId high relationship type id for this store. * @return number of processors suitable for this step. In most cases this will be 0, which is the typical value used * when just allowing the importer to grab up to {@link Configuration#maxNumberOfProcessors()}. The returned value * will at least be 1. */ private static int numberOfProcessors( Configuration config, NodeLabelsCache cache, int highLabelId, int highRelationshipTypeId ) { GatheringMemoryStatsVisitor memVisitor = new GatheringMemoryStatsVisitor(); cache.acceptMemoryStatsVisitor( memVisitor ); long availableMem = config.maxMemoryUsage() - memVisitor.getTotalUsage(); long threadMem = RelationshipCountsProcessor.calculateMemoryUsage( highLabelId, highRelationshipTypeId ); long possibleThreads = availableMem / threadMem; return possibleThreads >= config.maxNumberOfProcessors() ? 0 : toIntExact( max( 1, possibleThreads ) ); }
public Backwards( long lowIncluded, long highExcluded, Configuration config ) { this.lowIncluded = lowIncluded; this.highExcluded = highExcluded; this.batchSize = config.batchSize(); this.endId = highExcluded; }
private static Config getNeo4jConfig( Configuration config, Config dbConfig ) { dbConfig.augment( stringMap( dense_node_threshold.name(), valueOf( config.denseNodeThreshold() ), pagecache_memory.name(), valueOf( config.pageCacheMemory() ) ) ); return dbConfig; }
static Long parseMaxMemory( String maxMemoryString ) { if ( maxMemoryString != null ) { maxMemoryString = maxMemoryString.trim(); if ( maxMemoryString.endsWith( "%" ) ) { int percent = Integer.parseInt( maxMemoryString.substring( 0, maxMemoryString.length() - 1 ) ); long result = calculateMaxMemoryFromPercent( percent ); if ( !canDetectFreeMemory() ) { System.err.println( "WARNING: amount of free memory couldn't be detected so defaults to " + bytes( result ) + ". For optimal performance instead explicitly specify amount of " + "memory that importer is allowed to use using " + Options.MAX_MEMORY.argument() ); } return result; } return Settings.parseLongWithUnit( maxMemoryString ); } return null; }
static void printOverview( File storeDir, Collection<Option<File[]>> nodesFiles, Collection<Option<File[]>> relationshipsFiles, org.neo4j.unsafe.impl.batchimport.Configuration configuration, PrintStream out ) { out.println( "Neo4j version: " + Version.getNeo4jVersion() ); out.println( "Importing the contents of these files into " + storeDir + ":" ); printInputFiles( "Nodes", nodesFiles, out ); printInputFiles( "Relationships", relationshipsFiles, out ); out.println(); out.println( "Available resources:" ); printIndented( "Total machine memory: " + bytes( OsBeanUtil.getTotalPhysicalMemory() ), out ); printIndented( "Free machine memory: " + bytes( OsBeanUtil.getFreePhysicalMemory() ), out ); printIndented( "Max heap memory : " + bytes( Runtime.getRuntime().maxMemory() ), out ); printIndented( "Processors: " + configuration.maxNumberOfProcessors(), out ); printIndented( "Configured max memory: " + bytes( configuration.maxMemoryUsage() ), out ); printIndented( "High-IO: " + configuration.highIO(), out ); out.println(); }
ExecutionMonitor processorAssigner = eagerRandomSaturation( config.maxNumberOfProcessors() ); DatabaseLayout databaseLayout = directory.databaseLayout( "dir" + random.nextAlphaNumericString( 8, 8 ) ); nodes( nodeRandomSeed, NODE_COUNT, config.batchSize(), inputIdGenerator, groupDistribution ), relationships( relationshipRandomSeed, RELATIONSHIP_COUNT, config.batchSize(), inputIdGenerator, groupDistribution ), idMapper.apply( groups ),
@Override public long maxMemoryUsage() { return defaults.maxMemoryUsage(); }
@Override public long pageCacheMemory() { return defaultSettingsSuitableForTests ? mebiBytes( 8 ) : DEFAULT.pageCacheMemory(); }
public void initialize( Input input ) throws IOException { log.info( "Import starting" ); startTime = currentTimeMillis(); this.input = input; PageCacheArrayFactoryMonitor numberArrayFactoryMonitor = new PageCacheArrayFactoryMonitor(); numberArrayFactory = auto( neoStore.getPageCache(), storeDir, config.allowCacheAllocationOnHeap(), numberArrayFactoryMonitor ); badCollector = input.badCollector(); // Some temporary caches and indexes in the import idMapper = input.idMapper( numberArrayFactory ); nodeRelationshipCache = new NodeRelationshipCache( numberArrayFactory, config.denseNodeThreshold() ); Estimates inputEstimates = input.calculateEstimates( neoStore.getPropertyStore().newValueEncodedSizeCalculator() ); // Sanity checking against estimates new EstimationSanityChecker( recordFormats, monitor ).sanityCheck( inputEstimates ); new HeapSizeSanityChecker( monitor ).sanityCheck( inputEstimates, recordFormats, neoStore, nodeRelationshipCache.memoryEstimation( inputEstimates.numberOfNodes() ), idMapper.memoryEstimation( inputEstimates.numberOfNodes() ) ); dependencies.satisfyDependencies( inputEstimates, idMapper, neoStore, nodeRelationshipCache, numberArrayFactoryMonitor ); if ( neoStore.determineDoubleRelationshipRecordUnits( inputEstimates ) ) { monitor.doubleRelationshipRecordUnitsEnabled(); } executionMonitor.initialize( dependencies ); }
/** * @return max memory to use for import cache data structures while importing. * This should exclude the memory acquired by this JVM. By default this returns total physical * memory on the machine it's running on minus the max memory of this JVM. * {@value #DEFAULT_MAX_MEMORY_PERCENT}% of that figure. * @throws UnsupportedOperationException if available memory couldn't be determined. */ default long maxMemoryUsage() { return calculateMaxMemoryFromPercent( DEFAULT_MAX_MEMORY_PERCENT ); }
@Override public int maxNumberOfProcessors() { return Configuration.allAvailableProcessors(); } }
@Override public boolean allowCacheAllocationOnHeap() { return defaults.allowCacheAllocationOnHeap(); } }
@Override public int denseNodeThreshold() { return config.getRaw().containsKey( dense_node_threshold.name() ) ? config.get( dense_node_threshold ) : defaults.denseNodeThreshold(); }
static void printOverview( File storeDir, Collection<Option<File[]>> nodesFiles, Collection<Option<File[]>> relationshipsFiles, org.neo4j.unsafe.impl.batchimport.Configuration configuration, PrintStream out ) { out.println( "Neo4j version: " + Version.getNeo4jVersion() ); out.println( "Importing the contents of these files into " + storeDir + ":" ); printInputFiles( "Nodes", nodesFiles, out ); printInputFiles( "Relationships", relationshipsFiles, out ); out.println(); out.println( "Available resources:" ); printIndented( "Total machine memory: " + bytes( OsBeanUtil.getTotalPhysicalMemory() ), out ); printIndented( "Free machine memory: " + bytes( OsBeanUtil.getFreePhysicalMemory() ), out ); printIndented( "Max heap memory : " + bytes( Runtime.getRuntime().maxMemory() ), out ); printIndented( "Processors: " + configuration.maxNumberOfProcessors(), out ); printIndented( "Configured max memory: " + bytes( configuration.maxMemoryUsage() ), out ); printIndented( "High-IO: " + configuration.highIO(), out ); out.println(); }
@Override public long maxMemoryUsage() { return maxMemory != null ? maxMemory : DEFAULT.maxMemoryUsage(); }
private static Config getNeo4jConfig( Configuration config, Config dbConfig ) { dbConfig.augment( stringMap( dense_node_threshold.name(), valueOf( config.denseNodeThreshold() ), pagecache_memory.name(), valueOf( config.pageCacheMemory() ) ) ); return dbConfig; }
@Test public void shouldOverrideSmallPageCacheMemorySettingContainingUnit() { // GIVEN long overridden = parseLongWithUnit( "10m" ); Config dbConfig = Config.defaults( pagecache_memory, valueOf( overridden ) ); Configuration config = new Configuration.Overridden( dbConfig ); // WHEN long memory = config.pageCacheMemory(); // THEN assertEquals( overridden, memory ); }
static Long parseMaxMemory( String maxMemoryString ) { if ( maxMemoryString != null ) { maxMemoryString = maxMemoryString.trim(); if ( maxMemoryString.endsWith( "%" ) ) { int percent = Integer.parseInt( maxMemoryString.substring( 0, maxMemoryString.length() - 1 ) ); long result = calculateMaxMemoryFromPercent( percent ); if ( !canDetectFreeMemory() ) { System.err.println( "WARNING: amount of free memory couldn't be detected so defaults to " + bytes( result ) + ". For optimal performance instead explicitly specify amount of " + "memory that importer is allowed to use using " + Options.MAX_MEMORY.argument() ); } return result; } return Settings.parseLongWithUnit( maxMemoryString ); } return null; }
public void initialize( Input input ) throws IOException { log.info( "Import starting" ); startTime = currentTimeMillis(); this.input = input; PageCacheArrayFactoryMonitor numberArrayFactoryMonitor = new PageCacheArrayFactoryMonitor(); numberArrayFactory = auto( neoStore.getPageCache(), storeDir, config.allowCacheAllocationOnHeap(), numberArrayFactoryMonitor ); badCollector = input.badCollector(); // Some temporary caches and indexes in the import idMapper = input.idMapper( numberArrayFactory ); nodeRelationshipCache = new NodeRelationshipCache( numberArrayFactory, config.denseNodeThreshold() ); Estimates inputEstimates = input.calculateEstimates( neoStore.getPropertyStore().newValueEncodedSizeCalculator() ); // Sanity checking against estimates new EstimationSanityChecker( recordFormats, monitor ).sanityCheck( inputEstimates ); new HeapSizeSanityChecker( monitor ).sanityCheck( inputEstimates, recordFormats, neoStore, nodeRelationshipCache.memoryEstimation( inputEstimates.numberOfNodes() ), idMapper.memoryEstimation( inputEstimates.numberOfNodes() ) ); dependencies.satisfyDependencies( inputEstimates, idMapper, neoStore, nodeRelationshipCache, numberArrayFactoryMonitor ); if ( neoStore.determineDoubleRelationshipRecordUnits( inputEstimates ) ) { monitor.doubleRelationshipRecordUnitsEnabled(); } executionMonitor.initialize( dependencies ); }