@Override public long maxMemoryUsage() { return defaults.maxMemoryUsage(); }
@Override public long maxMemoryUsage() { return maxMemory != null ? maxMemory : DEFAULT.maxMemoryUsage(); }
/** * @param storeDir directory which the db will be created in. * @param fileSystem {@link FileSystemAbstraction} that the {@code storeDir} lives in. * @param neoStore {@link BatchingNeoStores} to import into. * @param config import-specific {@link Configuration}. * @param logService {@link LogService} to use. * @param executionMonitor {@link ExecutionMonitor} to follow progress as the import proceeds. * @param recordFormats which {@link RecordFormats record format} to use for the created db. * @param monitor {@link Monitor} for some events. */ public ImportLogic( File storeDir, FileSystemAbstraction fileSystem, BatchingNeoStores neoStore, Configuration config, LogService logService, ExecutionMonitor executionMonitor, RecordFormats recordFormats, Monitor monitor ) { this.storeDir = storeDir; this.fileSystem = fileSystem; this.neoStore = neoStore; this.config = config; this.recordFormats = recordFormats; this.monitor = monitor; this.log = logService.getInternalLogProvider().getLog( getClass() ); this.executionMonitor = ExecutionSupervisors.withDynamicProcessorAssignment( executionMonitor, config ); this.maxMemory = config.maxMemoryUsage(); }
/** * Keeping all counts for all combinations of label/reltype can require a lot of memory if there are lots of those tokens. * Each processor will allocate such a data structure and so in extreme cases the number of processors will have to * be limited to not surpass the available memory limits. * * @param config {@link Configuration} holding things like max number of processors and max memory. * @param cache {@link NodeLabelsCache} which is the only other data structure occupying memory at this point. * @param highLabelId high label id for this store. * @param highRelationshipTypeId high relationship type id for this store. * @return number of processors suitable for this step. In most cases this will be 0, which is the typical value used * when just allowing the importer to grab up to {@link Configuration#maxNumberOfProcessors()}. The returned value * will at least be 1. */ private static int numberOfProcessors( Configuration config, NodeLabelsCache cache, int highLabelId, int highRelationshipTypeId ) { GatheringMemoryStatsVisitor memVisitor = new GatheringMemoryStatsVisitor(); cache.acceptMemoryStatsVisitor( memVisitor ); long availableMem = config.maxMemoryUsage() - memVisitor.getTotalUsage(); long threadMem = RelationshipCountsProcessor.calculateMemoryUsage( highLabelId, highRelationshipTypeId ); long possibleThreads = availableMem / threadMem; return possibleThreads >= config.maxNumberOfProcessors() ? 0 : toIntExact( max( 1, possibleThreads ) ); }
private ProcessRelationshipCountsDataStep instantiateStep( int highLabelId, int highRelationshipTypeId, long labelCacheSize, int maxProcessors, long maxMemory ) { StageControl control = new SimpleStageControl(); NodeLabelsCache cache = nodeLabelsCache( labelCacheSize ); Configuration config = mock( Configuration.class ); when( config.maxNumberOfProcessors() ).thenReturn( maxProcessors ); when( config.maxMemoryUsage() ).thenReturn( maxMemory ); return new ProcessRelationshipCountsDataStep( control, cache, config, highLabelId, highRelationshipTypeId, mock( CountsAccessor.Updater.class ), OFF_HEAP, INSTANCE ); }
static void printOverview( File storeDir, Collection<Option<File[]>> nodesFiles, Collection<Option<File[]>> relationshipsFiles, org.neo4j.unsafe.impl.batchimport.Configuration configuration, PrintStream out ) { out.println( "Neo4j version: " + Version.getNeo4jVersion() ); out.println( "Importing the contents of these files into " + storeDir + ":" ); printInputFiles( "Nodes", nodesFiles, out ); printInputFiles( "Relationships", relationshipsFiles, out ); out.println(); out.println( "Available resources:" ); printIndented( "Total machine memory: " + bytes( OsBeanUtil.getTotalPhysicalMemory() ), out ); printIndented( "Free machine memory: " + bytes( OsBeanUtil.getFreePhysicalMemory() ), out ); printIndented( "Max heap memory : " + bytes( Runtime.getRuntime().maxMemory() ), out ); printIndented( "Processors: " + configuration.maxNumberOfProcessors(), out ); printIndented( "Configured max memory: " + bytes( configuration.maxMemoryUsage() ), out ); printIndented( "High-IO: " + configuration.highIO(), out ); out.println(); }
@Override public long maxMemoryUsage() { String custom = args.get( ImportTool.Options.MAX_MEMORY.key(), (String) ImportTool.Options.MAX_MEMORY.defaultValue() ); return custom != null ? ImportTool.parseMaxMemory( custom ) : DEFAULT.maxMemoryUsage(); } };
@Test public void shouldCalculateCorrectMaxMemorySetting() throws Exception { long totalMachineMemory = OsBeanUtil.getTotalPhysicalMemory(); assumeTrue( totalMachineMemory != VALUE_UNAVAILABLE ); // given int percent = 70; Configuration config = new Configuration() { @Override public long maxMemoryUsage() { return Configuration.calculateMaxMemoryFromPercent( percent ); } }; // when long memory = config.maxMemoryUsage(); // then long expected = (long) ((totalMachineMemory - Runtime.getRuntime().maxMemory()) * (percent / 100D)); long diff = abs( expected - memory ); assertThat( diff, lessThan( (long)(expected / 10D) ) ); }
@Override public long maxMemoryUsage() { return maxMemory != null ? maxMemory : DEFAULT.maxMemoryUsage(); }
@Override public long maxMemoryUsage() { return defaults.maxMemoryUsage(); }
/** * @param storeDir directory which the db will be created in. * @param fileSystem {@link FileSystemAbstraction} that the {@code storeDir} lives in. * @param neoStore {@link BatchingNeoStores} to import into. * @param config import-specific {@link Configuration}. * @param logService {@link LogService} to use. * @param executionMonitor {@link ExecutionMonitor} to follow progress as the import proceeds. * @param recordFormats which {@link RecordFormats record format} to use for the created db. * @param monitor {@link Monitor} for some events. */ public ImportLogic( File storeDir, FileSystemAbstraction fileSystem, BatchingNeoStores neoStore, Configuration config, LogService logService, ExecutionMonitor executionMonitor, RecordFormats recordFormats, Monitor monitor ) { this.storeDir = storeDir; this.fileSystem = fileSystem; this.neoStore = neoStore; this.config = config; this.recordFormats = recordFormats; this.monitor = monitor; this.log = logService.getInternalLogProvider().getLog( getClass() ); this.executionMonitor = ExecutionSupervisors.withDynamicProcessorAssignment( executionMonitor, config ); this.maxMemory = config.maxMemoryUsage(); }
/** * Keeping all counts for all combinations of label/reltype can require a lot of memory if there are lots of those tokens. * Each processor will allocate such a data structure and so in extreme cases the number of processors will have to * be limited to not surpass the available memory limits. * * @param config {@link Configuration} holding things like max number of processors and max memory. * @param cache {@link NodeLabelsCache} which is the only other data structure occupying memory at this point. * @param highLabelId high label id for this store. * @param highRelationshipTypeId high relationship type id for this store. * @return number of processors suitable for this step. In most cases this will be 0, which is the typical value used * when just allowing the importer to grab up to {@link Configuration#maxNumberOfProcessors()}. The returned value * will at least be 1. */ private static int numberOfProcessors( Configuration config, NodeLabelsCache cache, int highLabelId, int highRelationshipTypeId ) { GatheringMemoryStatsVisitor memVisitor = new GatheringMemoryStatsVisitor(); cache.acceptMemoryStatsVisitor( memVisitor ); long availableMem = config.maxMemoryUsage() - memVisitor.getTotalUsage(); long threadMem = RelationshipCountsProcessor.calculateMemoryUsage( highLabelId, highRelationshipTypeId ); long possibleThreads = availableMem / threadMem; return possibleThreads >= config.maxNumberOfProcessors() ? 0 : toIntExact( max( 1, possibleThreads ) ); }
static void printOverview( File storeDir, Collection<Option<File[]>> nodesFiles, Collection<Option<File[]>> relationshipsFiles, org.neo4j.unsafe.impl.batchimport.Configuration configuration, PrintStream out ) { out.println( "Neo4j version: " + Version.getNeo4jVersion() ); out.println( "Importing the contents of these files into " + storeDir + ":" ); printInputFiles( "Nodes", nodesFiles, out ); printInputFiles( "Relationships", relationshipsFiles, out ); out.println(); out.println( "Available resources:" ); printIndented( "Total machine memory: " + bytes( OsBeanUtil.getTotalPhysicalMemory() ), out ); printIndented( "Free machine memory: " + bytes( OsBeanUtil.getFreePhysicalMemory() ), out ); printIndented( "Max heap memory : " + bytes( Runtime.getRuntime().maxMemory() ), out ); printIndented( "Processors: " + configuration.maxNumberOfProcessors(), out ); printIndented( "Configured max memory: " + bytes( configuration.maxMemoryUsage() ), out ); printIndented( "High-IO: " + configuration.highIO(), out ); out.println(); }