/** {@inheritDoc} */ @Override public void onKernalStop(boolean cancel) { for (IgfsContext igfsCtx : igfsCache.values()) { if (log.isDebugEnabled()) log.debug("Stopping igfs: " + igfsCtx.configuration().getName()); List<IgfsManager> mgrs = igfsCtx.managers(); for (ListIterator<IgfsManager> it = mgrs.listIterator(mgrs.size()); it.hasPrevious();) { IgfsManager mgr = it.previous(); mgr.onKernalStop(cancel); } } if (log.isDebugEnabled()) log.debug("Finished executing IGFS processor onKernalStop() callback."); }
/** {@inheritDoc} */ @Override public void stop(boolean cancel) { // Stop IGFS instances. for (IgfsContext igfsCtx : igfsCache.values()) { if (log.isDebugEnabled()) log.debug("Stopping igfs: " + igfsCtx.configuration().getName()); List<IgfsManager> mgrs = igfsCtx.managers(); for (ListIterator<IgfsManager> it = mgrs.listIterator(mgrs.size()); it.hasPrevious();) { IgfsManager mgr = it.previous(); mgr.stop(cancel); } igfsCtx.igfs().stop(cancel); } igfsCache.clear(); if (log.isDebugEnabled()) log.debug("IGFS processor stopped."); }
/** {@inheritDoc} */ @Override protected void onKernalStart0() throws IgniteCheckedException { if (igfsCtx.configuration().isFragmentizerEnabled()) { // Check at startup if this node is a fragmentizer coordinator. DiscoveryEvent locJoinEvt = igfsCtx.kernalContext().discovery().localJoinEvent(); checkLaunchCoordinator(locJoinEvt); } }
while (fragmentingFiles.size() < igfsCtx.configuration().getFragmentizerConcurrentFiles()) { IgfsEntryInfo fileInfo = fileForFragmentizer(fragmentingFiles.keySet());
/** {@inheritDoc} */ @Override protected void start0() throws IgniteCheckedException { metaCacheStartLatch = new CountDownLatch(1); cfg = igfsCtx.configuration(); evts = igfsCtx.kernalContext().event(); sampling = new IgfsSamplingKey(cfg.getName()); log = igfsCtx.kernalContext().log(IgfsMetaManager.class); metaCacheName = cfg.getMetaCacheConfiguration().getName(); }
/** * Constructs IGFS IPC handler. * * @param igfsCtx Context. * @param endpointCfg Endpoint configuration. * @param mgmt Management flag. */ IgfsIpcHandler(IgfsContext igfsCtx, IgfsIpcEndpointConfiguration endpointCfg, boolean mgmt) { assert igfsCtx != null; ctx = igfsCtx.kernalContext(); igfs = igfsCtx.igfs(); // Keep buffer size multiple of block size so no extra byte array copies is performed. bufSize = igfsCtx.configuration().getBlockSize() * 2; // Create thread pool for request handling. int threadCnt = endpointCfg.getThreadCount(); String prefix = "igfs-" + igfsCtx.igfs().name() + (mgmt ? "mgmt-" : "") + "-ipc"; pool = new IgniteThreadPoolExecutor(prefix, igfsCtx.kernalContext().igniteInstanceName(), threadCnt, threadCnt, Long.MAX_VALUE, new LinkedBlockingQueue<Runnable>()); log = ctx.log(IgfsIpcHandler.class); }
/** * Flushes this output stream and forces any buffered output bytes to be written out. * * @throws IOException if an I/O error occurs. */ @Override public void flush() throws IOException { synchronized (mux) { checkClosed(null, 0); sendBufferIfNotEmpty(); flushRemainder(); awaitAcks(); // Update file length if needed. if (igfsCtx.configuration().isUpdateFileLengthOnFlush() && space > 0) { try { IgfsEntryInfo fileInfo0 = igfsCtx.meta().reserveSpace(fileInfo.id(), space, streamRange); if (fileInfo0 == null) throw new IOException("File was concurrently deleted: " + path); else fileInfo = fileInfo0; streamRange = initialStreamRange(fileInfo); space = 0; } catch (IgniteCheckedException e) { throw new IOException("Failed to update file length data [path=" + path + ", space=" + space + ']', e); } } } }
/** * Gets initial affinity range. This range will have 0 length and will start from first * non-occupied file block. * * @param fileInfo File info to build initial range for. * @return Affinity range. */ private IgfsFileAffinityRange initialStreamRange(IgfsEntryInfo fileInfo) { if (!igfsCtx.configuration().isFragmentizerEnabled()) return null; if (!Boolean.parseBoolean(fileInfo.properties().get(IgfsUtils.PROP_PREFER_LOCAL_WRITES))) return null; int blockSize = fileInfo.blockSize(); // Find first non-occupied block offset. long off = ((fileInfo.length() + blockSize - 1) / blockSize) * blockSize; // Need to get last affinity key and reuse it if we are on the same node. long lastBlockOff = off - fileInfo.blockSize(); if (lastBlockOff < 0) lastBlockOff = 0; IgfsFileMap map = fileInfo.fileMap(); IgniteUuid prevAffKey = map == null ? null : map.affinityKey(lastBlockOff, false); IgniteUuid affKey = igfsCtx.data().nextAffinityKey(prevAffKey); return affKey == null ? null : new IgfsFileAffinityRange(off, off, affKey); }
/** {@inheritDoc} */ @Override protected void start0() throws IgniteCheckedException { if (!igfsCtx.configuration().isFragmentizerEnabled()) return; // We care only about node leave and fail events. igfsCtx.kernalContext().event().addLocalEventListener(new GridLocalEventListener() { @Override public void onEvent(Event evt) { assert evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED; DiscoveryEvent discoEvt = (DiscoveryEvent)evt; checkLaunchCoordinator(discoEvt); } }, EVT_NODE_LEFT, EVT_NODE_FAILED); fragmentizerWorker = new FragmentizerWorker(); String igfsName = igfsCtx.configuration().getName(); topic = F.isEmpty(igfsName) ? TOPIC_IGFS : TOPIC_IGFS.topic(igfsName); igfsCtx.kernalContext().io().addMessageListener(topic, fragmentizerWorker); new IgniteThread(fragmentizerWorker).start(); }
/** {@inheritDoc} */ @Override protected void start0() throws IgniteCheckedException { dataCacheStartLatch = new CountDownLatch(1); String igfsName = igfsCtx.configuration().getName(); topic = F.isEmpty(igfsName) ? TOPIC_IGFS : TOPIC_IGFS.topic(igfsName); igfsCtx.kernalContext().io().addMessageListener(topic, new GridMessageListener() { @Override public void onMessage(UUID nodeId, Object msg, byte plc) { if (msg instanceof IgfsBlocksMessage) processBlocksMessage(nodeId, (IgfsBlocksMessage)msg); else if (msg instanceof IgfsAckMessage) processAckMessage(nodeId, (IgfsAckMessage)msg); } }); igfsCtx.kernalContext().event().addLocalEventListener(new GridLocalEventListener() { @Override public void onEvent(Event evt) { assert evt.type() == EVT_NODE_FAILED || evt.type() == EVT_NODE_LEFT; DiscoveryEvent discoEvt = (DiscoveryEvent)evt; if (igfsCtx.igfsNode(discoEvt.eventNode())) { for (WriteCompletionFuture future : pendingWrites.values()) { future.onError(discoEvt.eventNode().id(), new ClusterTopologyCheckedException("Node left grid before write completed: " + evt.node().id())); } } } }, EVT_NODE_LEFT, EVT_NODE_FAILED); delWorker = new AsyncDeleteWorker(igfsCtx.kernalContext().igniteInstanceName(), "igfs-" + igfsName + "-delete-worker", log); dataCacheName = igfsCtx.configuration().getDataCacheConfiguration().getName(); }
cfg = igfsCtx.configuration(); log = igfsCtx.kernalContext().log(IgfsImpl.class); evts = igfsCtx.kernalContext().event(); String dataCacheName = igfsCtx.configuration().getDataCacheConfiguration().getName();
/** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override protected void onKernalStart0() throws IgniteCheckedException { dataCachePrj = igfsCtx.kernalContext().cache().getOrStartCache(dataCacheName); assert dataCachePrj != null; dataCache = (IgniteInternalCache)dataCachePrj; AffinityKeyMapper mapper = igfsCtx.kernalContext().cache() .internalCache(dataCacheName).configuration().getAffinityMapper(); grpSize = mapper instanceof IgfsGroupDataBlocksKeyMapper ? ((IgfsGroupDataBlocksKeyMapper)mapper).getGroupSize() : 1; grpBlockSize = igfsCtx.configuration().getBlockSize() * (long)grpSize; assert grpBlockSize != 0; igfsCtx.kernalContext().cache().internalCache(dataCacheName).preloader() .startFuture().listen(new CI1<IgniteInternalFuture<Object>>() { @Override public void apply(IgniteInternalFuture<Object> f) { dataCacheStartLatch.countDown(); } }); new Thread(delWorker).start(); }
IgfsUtils.createFile( IgniteUuid.randomUuid(), igfsCtx.configuration().getBlockSize(), status.length(), null,
/** * Creates new instance of explicit data streamer. * * @return New instance of data streamer. */ private IgniteDataStreamer<IgfsBlockKey, byte[]> dataStreamer() { IgniteDataStreamer<IgfsBlockKey, byte[]> ldr = igfsCtx.kernalContext().<IgfsBlockKey, byte[]>dataStream().dataStreamer(dataCachePrj.name()); FileSystemConfiguration cfg = igfsCtx.configuration(); if (cfg.getPerNodeBatchSize() > 0) ldr.perNodeBufferSize(cfg.getPerNodeBatchSize()); if (cfg.getPerNodeParallelBatchCount() > 0) ldr.perNodeParallelOperations(cfg.getPerNodeParallelBatchCount()); ldr.receiver(DataStreamerCacheUpdaters.<IgfsBlockKey, byte[]>batchedSorted()); return ldr; }
/** {@inheritDoc} */ @Override protected void start0() throws IgniteCheckedException { FileSystemConfiguration igfsCfg = igfsCtx.configuration(); if (igfsCfg.isIpcEndpointEnabled()) { IgfsIpcEndpointConfiguration ipcCfg = igfsCfg.getIpcEndpointConfiguration(); if (ipcCfg == null) ipcCfg = new IgfsIpcEndpointConfiguration(); bind(ipcCfg, /*management*/false); } if (igfsCfg.getManagementPort() >= 0) { IgfsIpcEndpointConfiguration mgmtIpcCfg = new IgfsIpcEndpointConfiguration(); mgmtIpcCfg.setType(TCP); mgmtIpcCfg.setPort(igfsCfg.getManagementPort()); bind(mgmtIpcCfg, /*management*/true); } if (bindWorker != null) new IgniteThread(bindWorker).start(); }
if (bytesProcessed >= igfsCtx.configuration().getFragmentizerThrottlingBlockLength()) { ldr.flush(); U.sleep(igfsCtx.configuration().getFragmentizerThrottlingDelay());
igfsCtx.configuration().getBlockSize(), status.length(), affKey,
igfsPrimary.context().configuration().getDataCacheConfiguration().getName());
/** {@inheritDoc} */ @Override protected void onKernalStart0() throws IgniteCheckedException { if (igfsCtx.configuration().isFragmentizerEnabled()) { // Check at startup if this node is a fragmentizer coordinator. DiscoveryEvent locJoinEvt = igfsCtx.kernalContext().discovery().localJoinEvent(); checkLaunchCoordinator(locJoinEvt); } }
/** {@inheritDoc} */ @Override protected void start0() throws IgniteCheckedException { metaCacheStartLatch = new CountDownLatch(1); cfg = igfsCtx.configuration(); evts = igfsCtx.kernalContext().event(); sampling = new IgfsSamplingKey(cfg.getName()); log = igfsCtx.kernalContext().log(IgfsMetaManager.class); metaCacheName = cfg.getMetaCacheConfiguration().getName(); }