@Override public boolean markDiscarded(DatabaseDocument<T> d, String stage) { DatabaseDocument<T> cached = cache.getDocumentById(d.getID()); d.putAll(cached); cache.remove(d.getID()); if (writer.markDiscarded(d, stage)) { return true; } return false; }
@Override public boolean markPending(DatabaseDocument<T> d, String stage) { DatabaseDocument<T> cached = cache.getDocumentById(d.getID()); d.putAll(cached); cache.remove(d.getID()); return writer.markPending(d, stage); }
public NodeMaster(CoreConfiguration conf, CachingDocumentNIO<T> documentNIO, Pipeline pipeline, ShutdownHandler shutdownHandler) { this.conf = conf; this.shutdownHandler = shutdownHandler; sm = StageManager.getStageManager(); this.pipeline = pipeline; this.pollingInterval = conf.getPollingInterval(); this.port = conf.getRestPort(); this.namespace = conf.getNamespace(); this.documentNIO = documentNIO; this.dbc = documentNIO.getDatabaseConnector(); } /**
private void addGroup(Pipeline p, StageGroup g) { if(p.hasGroup(g.getName())) { p.getGroup(g.getName()).setProperties(g.toPropertiesMap()); p.getGroup(g.getName()).setPropertiesModifiedDate(g.getPropertiesModifiedDate()); } else { p.addGroup(g); } }
@Override public boolean markProcessed(DatabaseDocument<T> d, String stage) { DatabaseDocument<T> cached = cache.getDocumentById(d.getID()); if (cached != null) { d.putAll(cached); cache.remove(d.getID()); } if (writer.markProcessed(d, stage)) { return true; } return false; }
@Override public boolean markFailed(DatabaseDocument<T> d, String stage) { DatabaseDocument<T> cached = cache.getDocumentById(d.getID()); d.putAll(cached); cache.remove(d.getID()); if (writer.markFailed(d, stage)) { return true; } return false; }
public StageGroup getGroupForStage(String name) { for (StageGroup g : getStageGroups()) { if (g.hasStage(name)) { return g; } } return null; }
public CachingDocumentNIO(DatabaseConnector<T> backing, Cache<T> cache, boolean startMonitorThread, int cacheTimeout) { writer = backing.getDocumentWriter(); reader = backing.getDocumentReader(); this.getDatabaseConnector(backing); this.cache = cache; this.cacheTimeout = cacheTimeout; monitor = new CacheMonitor(); if (startMonitorThread) { monitor.start(); } }
private boolean attachFiles(StageGroup group) { Set<DatabaseFile> files = group.getDatabaseFiles(); if(files == null) { return false; } for(DatabaseFile file : group.getDatabaseFiles()) { file.attach(dbc.getPipelineReader().getStream(file)); } return true; }
private void startStopped() throws IOException { for(StageRunner runner : sm.getRunners()) { if(!runner.isAlive() && !runner.isStarted()) { runner.prepare(); runner.start(); } } }
@Override public void connect() throws IOException { backing.connect(); documentio = new CachingDocumentNIO<BackingType>(backing, cache); documentio.prepare(); }
public void saveStatus() { connector.getStatusWriter().increment(getAndClearProcessed(), getAndClearFailed(), getAndClearDiscarded()); } }
@Override public DatabaseDocument<T> getDocumentById(DocumentID<T> id, boolean includeInactive) { DatabaseDocument<T> doc = getDocumentById(id); if (doc == null && includeInactive) { return reader.getDocumentById(id, includeInactive); } return cache.getDocumentById(id); }
@Override public DatabaseDocument<T> getDocument(DatabaseQuery<T> q) { DatabaseDocument<T> doc = cache.getDocument(q); if (doc == null) { doc = reader.getDocument(q); cache.add(doc); } return getCopy(doc); }
@Override public DatabaseDocument<T> getDocumentById(DocumentID<T> id) { DatabaseDocument<T> doc = cache.getDocumentById(id); if (doc == null) { doc = reader.getDocumentById(id, false); if(doc != null) { cache.add(doc); } else { return null; } } return getCopy(doc); }
/** * Flushes the cache back to the database */ public void flush() { Collection<DatabaseDocument<T>> docs = cache.removeAll(); for (DatabaseDocument<T> d : docs) { d.removeFetchedBy(CACHE_TAG); writer.update(d); } }
/** * Since skip becomes impossible to calculate in a cached scenario, this * will simply query the underlying reader. */ @Override public List<DatabaseDocument<T>> getDocuments(DatabaseQuery<T> q, int limit, int skip) { List<DatabaseDocument<T>> docs = reader.getDocuments(q, limit, skip); cache.add(docs); return docs; }
public void findAndDestroy(String groupName) { StageRunner sw = removeRunner(groupName); if(sw!=null) { sw.destroy(); } }