public static void importNodes( int numRunners, Input input, BatchingNeoStores stores, IdMapper idMapper, ExecutionMonitor executionMonitor, Monitor monitor ) throws IOException { Supplier<EntityImporter> importers = () -> new NodeImporter( stores, idMapper, monitor ); importData( NODE_IMPORT_NAME, numRunners, input.nodes(), stores, importers, executionMonitor, new MemoryUsageStatsProvider( stores, idMapper ) ); }
public static DataStatistics importRelationships( int numRunners, Input input, BatchingNeoStores stores, IdMapper idMapper, Collector badCollector, ExecutionMonitor executionMonitor, Monitor monitor, boolean validateRelationshipData ) throws IOException { DataStatistics typeDistribution = new DataStatistics( monitor.nodes.sum(), monitor.properties.sum(), new RelationshipTypeCount[0] ); Supplier<EntityImporter> importers = () -> new RelationshipImporter( stores, idMapper, typeDistribution, monitor, badCollector, validateRelationshipData, stores.usesDoubleRelationshipRecordUnits() ); importData( RELATIONSHIP_IMPORT_NAME, numRunners, input.relationships(), stores, importers, executionMonitor, new MemoryUsageStatsProvider( stores, idMapper ) ); return typeDistribution; }
/** * Populates {@link NodeRelationshipCache} with node degrees, which is required to know how to physically layout each * relationship chain. This is required before running {@link #linkRelationships(int)}. */ public void calculateNodeDegrees() { Configuration relationshipConfig = configWithRecordsPerPageBasedBatchSize( config, neoStore.getRelationshipStore() ); nodeRelationshipCache.setNodeCount( neoStore.getNodeStore().getHighId() ); MemoryUsageStatsProvider memoryUsageStats = new MemoryUsageStatsProvider( neoStore, nodeRelationshipCache ); NodeDegreeCountStage nodeDegreeStage = new NodeDegreeCountStage( relationshipConfig, neoStore.getRelationshipStore(), nodeRelationshipCache, memoryUsageStats ); executeStage( nodeDegreeStage ); nodeRelationshipCache.countingCompleted(); availableMemoryForLinking = maxMemory - totalMemoryUsageOf( nodeRelationshipCache, neoStore ); }
/** * Prepares {@link IdMapper} to be queried for ID --> nodeId lookups. This is required for running {@link #importRelationships()}. */ public void prepareIdMapper() { if ( idMapper.needsPreparation() ) { MemoryUsageStatsProvider memoryUsageStats = new MemoryUsageStatsProvider( neoStore, idMapper ); LongFunction<Object> inputIdLookup = new NodeInputIdPropertyLookup( neoStore.getTemporaryPropertyStore() ); executeStage( new IdMapperPreparationStage( config, idMapper, inputIdLookup, badCollector, memoryUsageStats ) ); final LongIterator duplicateNodeIds = idMapper.leftOverDuplicateNodesIds(); if ( duplicateNodeIds.hasNext() ) { executeStage( new DeleteDuplicateNodesStage( config, duplicateNodeIds, neoStore, storeUpdateMonitor ) ); } updatePeakMemoryUsage(); } }
/** * Builds the counts store. Requires that {@link #importNodes()} and {@link #importRelationships()} has run. */ public void buildCountsStore() { // Count nodes per label and labels per node try ( CountsAccessor.Updater countsUpdater = neoStore.getCountsStore().reset( neoStore.getLastCommittedTransactionId() ) ) { MigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); nodeLabelsCache = new NodeLabelsCache( numberArrayFactory, neoStore.getLabelRepository().getHighId() ); MemoryUsageStatsProvider memoryUsageStats = new MemoryUsageStatsProvider( neoStore, nodeLabelsCache ); executeStage( new NodeCountsAndLabelIndexBuildStage( config, nodeLabelsCache, neoStore.getNodeStore(), neoStore.getLabelRepository().getHighId(), countsUpdater, progressMonitor.startSection( "Nodes" ), neoStore.getLabelScanStore(), memoryUsageStats ) ); // Count label-[type]->label executeStage( new RelationshipCountsStage( config, nodeLabelsCache, neoStore.getRelationshipStore(), neoStore.getLabelRepository().getHighId(), neoStore.getRelationshipTypeRepository().getHighId(), countsUpdater, numberArrayFactory, progressMonitor.startSection( "Relationships" ) ) ); } }
StatsProvider memoryUsage = new MemoryUsageStatsProvider( neoStore, groupCache ); executeStage( new CountGroupsStage( groupConfig, fromStore, groupCache, memoryUsage ) ); long fromNodeId = 0;
MemoryUsageStatsProvider memoryUsageStats = new MemoryUsageStatsProvider( neoStore, nodeRelationshipCache );
public static void importNodes( int numRunners, Input input, BatchingNeoStores stores, IdMapper idMapper, ExecutionMonitor executionMonitor, Monitor monitor ) throws IOException { Supplier<EntityImporter> importers = () -> new NodeImporter( stores, idMapper, monitor ); importData( NODE_IMPORT_NAME, numRunners, input.nodes(), stores, importers, executionMonitor, new MemoryUsageStatsProvider( stores, idMapper ) ); }
public static DataStatistics importRelationships( int numRunners, Input input, BatchingNeoStores stores, IdMapper idMapper, Collector badCollector, ExecutionMonitor executionMonitor, Monitor monitor, boolean validateRelationshipData ) throws IOException { DataStatistics typeDistribution = new DataStatistics( monitor.nodes.sum(), monitor.properties.sum(), new RelationshipTypeCount[0] ); Supplier<EntityImporter> importers = () -> new RelationshipImporter( stores, idMapper, typeDistribution, monitor, badCollector, validateRelationshipData, stores.usesDoubleRelationshipRecordUnits() ); importData( RELATIONSHIP_IMPORT_NAME, numRunners, input.relationships(), stores, importers, executionMonitor, new MemoryUsageStatsProvider( stores, idMapper ) ); return typeDistribution; }
/** * Populates {@link NodeRelationshipCache} with node degrees, which is required to know how to physically layout each * relationship chain. This is required before running {@link #linkRelationships(int)}. */ public void calculateNodeDegrees() { Configuration relationshipConfig = configWithRecordsPerPageBasedBatchSize( config, neoStore.getRelationshipStore() ); nodeRelationshipCache.setNodeCount( neoStore.getNodeStore().getHighId() ); MemoryUsageStatsProvider memoryUsageStats = new MemoryUsageStatsProvider( neoStore, nodeRelationshipCache ); NodeDegreeCountStage nodeDegreeStage = new NodeDegreeCountStage( relationshipConfig, neoStore.getRelationshipStore(), nodeRelationshipCache, memoryUsageStats ); executeStage( nodeDegreeStage ); nodeRelationshipCache.countingCompleted(); availableMemoryForLinking = maxMemory - totalMemoryUsageOf( nodeRelationshipCache, neoStore ); }
/** * Prepares {@link IdMapper} to be queried for ID --> nodeId lookups. This is required for running {@link #importRelationships()}. */ public void prepareIdMapper() { if ( idMapper.needsPreparation() ) { MemoryUsageStatsProvider memoryUsageStats = new MemoryUsageStatsProvider( neoStore, idMapper ); LongFunction<Object> inputIdLookup = new NodeInputIdPropertyLookup( neoStore.getTemporaryPropertyStore() ); executeStage( new IdMapperPreparationStage( config, idMapper, inputIdLookup, badCollector, memoryUsageStats ) ); final LongIterator duplicateNodeIds = idMapper.leftOverDuplicateNodesIds(); if ( duplicateNodeIds.hasNext() ) { executeStage( new DeleteDuplicateNodesStage( config, duplicateNodeIds, neoStore, storeUpdateMonitor ) ); } updatePeakMemoryUsage(); } }
/** * Builds the counts store. Requires that {@link #importNodes()} and {@link #importRelationships()} has run. */ public void buildCountsStore() { // Count nodes per label and labels per node try ( CountsAccessor.Updater countsUpdater = neoStore.getCountsStore().reset( neoStore.getLastCommittedTransactionId() ) ) { MigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); nodeLabelsCache = new NodeLabelsCache( numberArrayFactory, neoStore.getLabelRepository().getHighId() ); MemoryUsageStatsProvider memoryUsageStats = new MemoryUsageStatsProvider( neoStore, nodeLabelsCache ); executeStage( new NodeCountsAndLabelIndexBuildStage( config, nodeLabelsCache, neoStore.getNodeStore(), neoStore.getLabelRepository().getHighId(), countsUpdater, progressMonitor.startSection( "Nodes" ), neoStore.getLabelScanStore(), memoryUsageStats ) ); // Count label-[type]->label executeStage( new RelationshipCountsStage( config, nodeLabelsCache, neoStore.getRelationshipStore(), neoStore.getLabelRepository().getHighId(), neoStore.getRelationshipTypeRepository().getHighId(), countsUpdater, numberArrayFactory, progressMonitor.startSection( "Relationships" ) ) ); } }
StatsProvider memoryUsage = new MemoryUsageStatsProvider( neoStore, groupCache ); executeStage( new CountGroupsStage( groupConfig, fromStore, groupCache, memoryUsage ) ); long fromNodeId = 0;
MemoryUsageStatsProvider memoryUsageStats = new MemoryUsageStatsProvider( neoStore, nodeRelationshipCache );