@Override public GraphManager getGraphManager( ApplicationScope appScope ) { return gmf.createEdgeManager( appScope ); }
@Override public void invalidate() { ecmf.invalidate(); eif.invalidate(); gmf.invalidate(); mmf.invalidate(); } }
@Override public Observable<EdgeScope> getEdgesToEntities( final Observable<ApplicationScope> appScopes, final Optional<String> edgeType, final Optional<Edge> lastEdge) { return appScopes.flatMap( applicationScope -> { final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope ); return edgesObservable.edgesFromSourceDescending( gm, applicationScope.getApplication(), edgeType, lastEdge ) .map( edge -> new EdgeScope(applicationScope, edge )); } ); } }
@Override public Set<String> getConnectionsAsSource( final EntityRef entityRef ) { Preconditions.checkNotNull(entityRef, "entityRef cannot be null"); final GraphManager graphManager = graphManagerFactory.createEdgeManager( applicationScope ); final SearchEdgeType searchByEdgeType = createConnectionTypeSearch( entityRef.asId() ); return graphManager.getEdgeTypesFromSource( searchByEdgeType ).map( edgeName -> getConnectionNameFromEdgeName( edgeName ) ) .collect( () -> new HashSet<String>(), ( r, s ) -> r.add( s ) ).toBlocking().last(); }
@Override public Set<String> getConnectionsAsTarget( final EntityRef entityRef ) { Preconditions.checkNotNull( entityRef, "entityRef cannot be null" ); final GraphManager graphManager = graphManagerFactory.createEdgeManager( applicationScope ); final SearchEdgeType searchByEdgeType = createConnectionTypeSearch( entityRef.asId() ); return graphManager.getEdgeTypesToTarget(searchByEdgeType).map( edgeName -> getConnectionNameFromEdgeName( edgeName ) ) .collect( () -> new HashSet<String>( ), ( r, s ) -> r.add(s) ).toBlocking().last(); }
@Override public Observable<EntityIdScope> getEntities( final Observable<ApplicationScope> appScopes ) { return appScopes.flatMap( applicationScope -> { final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope ); final Id applicationId = applicationScope.getApplication(); //load all nodes that are targets of our application node. I.E. // entities that have been saved final Observable<Id> entityNodes = targetIdObservable.getTargetNodes( gm, applicationId ); //create our application node to emit since it's an entity as well final Observable<Id> applicationNode = Observable.just( applicationId ); //merge both the specified application node and the entity node // so they all get used return Observable.merge( applicationNode, entityNodes ). map( id -> new EntityIdScope( applicationScope, id ) ); } ); }
@Override public Observable<FilterResult<Entity>> call( final Observable<FilterResult<Id>> filterResultObservable ) { final ApplicationScope applicationScope = pipelineContext.getApplicationScope(); final EntityCollectionManager entityCollectionManager = entityCollectionManagerFactory.createCollectionManager( applicationScope ); //it's more efficient to make 1 network hop to get everything, then drop our results if required final Observable<FilterResult<Entity>> entityObservable = filterResultObservable.buffer( pipelineContext.getLimit() ).flatMap( bufferedIds -> { if (logger.isTraceEnabled()) { logger.trace("Attempting to batch load ids {}", bufferedIds); } final Observable<EntitySet> entitySetObservable = Observable.from( bufferedIds ).map( filterResultId -> filterResultId.getValue() ).toList() .flatMap( ids -> entityCollectionManager.load( ids ) ); //now we have a collection, validate our candidate set is correct. GraphManager graphManager = graphManagerFactory.createEdgeManager(applicationScope); return entitySetObservable.map( entitySet -> new EntityVerifier( applicationScope, graphManager, entitySet, bufferedIds, readRepairFig ) ) .doOnNext( entityCollector -> entityCollector.merge() ).flatMap( entityCollector -> Observable.from( entityCollector.getResults() ) ); } ); return entityObservable; }
@Override public Observable<FilterResult<Id>> call( final Observable<FilterResult<Id>> filterValueObservable ) { final GraphManager gm = graphManagerFactory.createEdgeManager( pipelineContext.getApplicationScope() ); return filterValueObservable.flatMap( filterValue -> { final String edgeTypeName = getEdgeName(); final Id id = filterValue.getValue(); //create our search final SearchByEdge searchByEdge = new SimpleSearchByEdge( id, edgeTypeName, targetId, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.absent() ); //load the versions of the edge, take the first since that's all we need to validate existence, then emit the target node return gm.loadEdgeVersions( searchByEdge ).take( 1 ).map( edge -> edge.getTargetNode() ).map( targetId -> new FilterResult<>(targetId, filterValue.getPath())); } ); }
@Override public long getApplicationSize(ApplicationScope applicationScope) { final IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope); EntityIndex entityIndex = entityIndexFactory.createEntityIndex(indexLocationStrategy); GraphManager graphManager = graphManagerFactory.createEdgeManager(applicationScope); Long sum = ObservableTimer.time( MathObservable.sumLong( graphManager.getEdgeTypesFromSource(new SimpleSearchEdgeType(applicationScope.getApplication(), CpNamingUtils.EDGE_COLL_PREFIX, Optional.<String>absent())) .map(type -> CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), type)) .map(edge -> entityIndex.getTotalEntitySizeInBytes(edge)) ), sumTimer).toBlocking().last(); return sum.longValue(); }
public Observable<MarkedEdge> getApplicationInfoEdges( final UUID applicationId ) { final ApplicationScope managementAppScope = getApplicationScope( CpNamingUtils.MANAGEMENT_APPLICATION_ID ); final GraphManager gm = graphManagerFactory.createEdgeManager( managementAppScope ); String edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( CpNamingUtils.APPLICATION_INFOS ); final SimpleSearchByEdge simpleSearchByEdgeType = new SimpleSearchByEdge( CpNamingUtils.generateApplicationId( CpNamingUtils.MANAGEMENT_APPLICATION_ID ), edgeType, CpNamingUtils.generateApplicationId( applicationId ), Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.absent() ); return gm.loadEdgeVersions( simpleSearchByEdgeType ); } }
@Override public Observable<FilterResult<Id>> call( final Observable<FilterResult<Id>> filterResultObservable ) { //get the graph manager final GraphManager graphManager = graphManagerFactory.createEdgeManager( pipelineContext.getApplicationScope() ); final String edgeName = getEdgeTypeFromConnectionType( connectionName ); //return all ids that are emitted from this edge return filterResultObservable.flatMap( idFilterResult -> { //set our our constant state final Optional<MarkedEdge> startFromCursor = getSeekValue(); final Id id = idFilterResult.getValue(); final Optional<Edge> typeWrapper = Optional.fromNullable(startFromCursor.orNull()); final SimpleSearchByIdType search = new SimpleSearchByIdType( id, edgeName, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, entityType, typeWrapper ); return graphManager.loadEdgesFromSourceByType( search ).map( edge -> createFilterResult( edge.getTargetNode(), edge, idFilterResult.getPath() )); } ); }
@Test( expected = NullPointerException.class ) public void invalidEdgeTypesWrite() { final GraphManager em = emf.createEdgeManager( scope ); em.writeEdge( null ); }
final Id entityId = entityIdScope.getId(); final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
final GraphManager gm = graphManagerFactory.createEdgeManager( graphNode.applicationScope );
@Test( expected = NullPointerException.class ) public void invalidEdgeTypesDelete() { final GraphManager em = emf.createEdgeManager( scope ); em.markEdge( null ); }
/** * Use Graph to get old appinfos from the old and deprecated System App. */ public Observable<org.apache.usergrid.persistence.model.entity.Entity> getOldAppInfos() { final ApplicationScope systemAppScope = getApplicationScope( SYSTEM_APP_ID ); final EntityCollectionManager systemCollectionManager = entityCollectionManagerFactory.createCollectionManager( systemAppScope ); final GraphManager gm = graphManagerFactory.createEdgeManager( systemAppScope ); String edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( "appinfos" ); Id rootAppId = systemAppScope.getApplication(); final SimpleSearchByEdgeType simpleSearchByEdgeType = new SimpleSearchByEdgeType( rootAppId, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.absent() ); Observable<org.apache.usergrid.persistence.model.entity.Entity> entityObs = gm.loadEdgesFromSource( simpleSearchByEdgeType ).flatMap( edge -> { final Id appInfoId = edge.getTargetNode(); return systemCollectionManager.load( appInfoId ).filter( entity -> ( entity != null ) ); } ); return entityObs; }
EntityCollectionManager entityCollectionManager = entityCollectionManagerFactory.createCollectionManager(applicationScope); GraphManager graphManager = graphManagerFactory.createEdgeManager(applicationScope); MapManager mapManager = getMapManagerForTypes(applicationScope);
@Override public Long call() throws Exception { GraphManager gm = factory.createEdgeManager( scope ); while ( !Thread.currentThread().isInterrupted() ) { //do a read to eventually trigger our group compaction. Take 2 pages of columns final long returnedEdgeCount = generator.doSearch( gm ) .doOnNext( edge -> readMeter.mark() ) .countLong().toBlocking().last(); logger.info( "Completed reading {} edges", returnedEdgeCount ); if ( writeCount != returnedEdgeCount ) { logger.warn( "Unexpected edge count returned!!! Expected {} but was {}", writeCount, returnedEdgeCount ); } assertEquals( "Expected to read same edge count", writeCount, returnedEdgeCount ); } return 0L; } }
@Override public Map<String, Long> getEachCollectionSize(ApplicationScope applicationScope) { final IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope); EntityIndex entityIndex = entityIndexFactory.createEntityIndex(indexLocationStrategy); GraphManager graphManager = graphManagerFactory.createEdgeManager(applicationScope); Map<String,Long> sumMap = ObservableTimer.time( graphManager.getEdgeTypesFromSource(new SimpleSearchEdgeType(applicationScope.getApplication(), CpNamingUtils.EDGE_COLL_PREFIX, Optional.<String>absent())) .collect(() -> new HashMap<String,Long>(), ((map, type) -> { SearchEdge edge = CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), type); final String collectionName = CpNamingUtils.getCollectionNameFromEdgeName(type); long sumType = entityIndex.getTotalEntitySizeInBytes(edge); map.put(collectionName,sumType); }) ) , sumTimer).toBlocking().last(); return sumMap; }
@Override public Observable<IndexOperationMessage> indexEntity( final ApplicationScope applicationScope, final Entity entity ) { //bootstrap the lower modules from their caches final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope ); final EntityIndex ei = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope)); final Id entityId = entity.getId(); //we always index in the target scope final Observable<Edge> edgesToTarget = edgesObservable.edgesToTarget( gm, entityId, true); //we may have to index we're indexing from source->target here final Observable<IndexEdge> sourceEdgesToIndex = edgesToTarget.map( edge -> generateScopeFromSource( edge ) ); //do our observable for batching //try to send a whole batch if we can final Observable<IndexOperationMessage> batches = sourceEdgesToIndex .buffer(indexFig.getIndexBatchSize() ) //map into batches based on our buffer size .flatMap( buffer -> Observable.from( buffer ) //collect results into a single batch .collect( () -> ei.createBatch(), ( batch, indexEdge ) -> { if (logger.isDebugEnabled()) { logger.debug("adding edge {} to batch for entity {}", indexEdge, entity); } final Optional<Set<String>> fieldsToIndex = getFilteredStringObjectMap( indexEdge ); batch.index( indexEdge, entity ,fieldsToIndex); } ) //return the future from the batch execution .map( batch -> batch.build() ) ); return ObservableTimer.time( batches, indexTimer ); }