blockSize = cfg.getBlockSize(); bufSize = cfg.getStreamBufferSize(); dataCacheName = cfg.getDataCacheName(); dfltMode = cfg.getDefaultMode(); dualModeMaxPendingPutsSize = cfg.getDualModeMaxPendingPutsSize(); dualModePutExec = cfg.getDualModePutExecutorService(); dualModePutExecShutdown = cfg.getDualModePutExecutorServiceShutdown(); fragmentizerConcurrentFiles = cfg.getFragmentizerConcurrentFiles(); fragmentizerLocWritesRatio = cfg.getFragmentizerLocalWritesRatio(); fragmentizerEnabled = cfg.isFragmentizerEnabled(); fragmentizerThrottlingBlockLen = cfg.getFragmentizerThrottlingBlockLength(); fragmentizerThrottlingDelay = cfg.getFragmentizerThrottlingDelay(); secondaryFs = cfg.getSecondaryFileSystem(); ipcEndpointCfg = cfg.getIpcEndpointConfiguration(); ipcEndpointEnabled = cfg.isIpcEndpointEnabled(); maxSpace = cfg.getMaxSpaceSize(); maxTaskRangeLen = cfg.getMaximumTaskRangeLength(); metaCacheName = cfg.getMetaCacheName(); mgmtPort = cfg.getManagementPort(); name = cfg.getName(); pathModes = cfg.getPathModes(); perNodeBatchSize = cfg.getPerNodeBatchSize(); perNodeParallelBatchCnt = cfg.getPerNodeParallelBatchCount(); prefetchBlocks = cfg.getPrefetchBlocks(); seqReadsBeforePrefetch = cfg.getSequentialReadsBeforePrefetch(); trashPurgeTimeout = cfg.getTrashPurgeTimeout();
/** {@inheritDoc} */ @Override public boolean isProxy(URI path) { GridGgfsMode mode = F.isEmpty(cfg.getPathModes()) ? cfg.getDefaultMode() : modeRslvr.resolveMode(new GridGgfsPath(path)); return mode == PROXY; } }
/** * @param cfg Grid configuration. * @param cacheName Cache name. * @return {@code True} in this is GGFS data or meta cache. */ public static boolean isGgfsCache(GridConfiguration cfg, @Nullable String cacheName) { GridGgfsConfiguration[] ggfsCfgs = cfg.getGgfsConfiguration(); if (ggfsCfgs != null) { for (GridGgfsConfiguration ggfsCfg : ggfsCfgs) { // GGFS config probably has not been validated yet => possible NPE, so we check for null. if (ggfsCfg != null && (F.eq(cacheName, ggfsCfg.getDataCacheName()) || F.eq(cacheName, ggfsCfg.getMetaCacheName()))) return true; } } return false; }
ggfs = ggfsCtx.ggfs(); dataCachePrj = ggfsCtx.kernalContext().cache().internalCache(ggfsCtx.configuration().getDataCacheName()); dataCache = ggfsCtx.kernalContext().cache().internalCache(ggfsCtx.configuration().getDataCacheName()); dataCacheStartFut = ggfsCtx.kernalContext().cache().internalCache(ggfsCtx.configuration().getDataCacheName()) .preloader().startFuture(); ggfsCtx.configuration().getDataCacheName()); .internalCache(ggfsCtx.configuration().getDataCacheName()).configuration().getAffinityMapper(); grpBlockSize = ggfsCtx.configuration().getBlockSize() * grpSize; String ggfsName = ggfsCtx.configuration().getName(); trashPurgeTimeout = ggfsCtx.configuration().getTrashPurgeTimeout(); putExecSvc = ggfsCtx.configuration().getDualModePutExecutorService(); putExecSvcShutdown = ggfsCtx.configuration().getDualModePutExecutorServiceShutdown(); else { int coresCnt = Runtime.getRuntime().availableProcessors(); maxPendingPuts = ggfsCtx.configuration().getDualModeMaxPendingPutsSize();
GridCacheConfiguration cacheCfg = cacheCfgs.get(ggfsCfg.getDataCacheName()); ggfsCfg.getName(), ggfsCfg.getBlockSize(), ((GridGgfsGroupDataBlocksKeyMapper)affMapper).groupSize(), ggfsCfg.getMetaCacheName(), ggfsCfg.getDataCacheName(), ggfsCfg.getDefaultMode(), ggfsCfg.getPathModes(), ggfsCfg.isFragmentizerEnabled()));
String name = cfg.getName(); "assign unique name to each): " + name); GridCacheAdapter<Object, Object> dataCache = ctx.cache().internalCache(cfg.getDataCacheName()); GridCache<Object, Object> metaCache = ctx.cache().cache(cfg.getMetaCacheName()); if (F.eq(cfg.getDataCacheName(), cfg.getMetaCacheName())) throw new GridException("Cannot use same cache as both data and meta cache: " + cfg.getName()); long maxSpaceSize = cfg.getMaxSpaceSize(); "to 0 and restart the grid): " + cfg.getDataCacheName()); if (cfg.getMaxSpaceSize() == 0 && dataCache.configuration().getMemoryMode() == OFFHEAP_VALUES) U.warn(log, "GGFS max space size is not specified but data cache values are stored off-heap (max " + "space will be limited to 80% of max JVM heap size): " + cfg.getName()); boolean secondary = cfg.getDefaultMode() == PROXY; if (cfg.getPathModes() != null) { for (Map.Entry<String, GridGgfsMode> mode : cfg.getPathModes().entrySet()) { if (mode.getValue() == PROXY) secondary = true; assertParameter(cfg.getSecondaryFileSystem() != null, "secondaryFileSystem cannot be null when mode is SECONDARY");
meta = ggfsCtx.meta(); data = ggfsCtx.data(); secondaryFs = cfg.getSecondaryFileSystem(); if (cfg.getDefaultMode() == PROXY) throw new GridException("Mode cannot be PROXY if secondary file system hasn't been defined."); dfltMode = cfg.getDefaultMode(); if (ggfsCtx.configuration().getPathModes() != null) { for (Map.Entry<String, GridGgfsMode> e : ggfsCtx.configuration().getPathModes().entrySet()) { if (!dfltModes.containsKey(e.getKey())) cfgModes.put(e.getKey(), e.getValue()); String dataCacheName = ggfsCtx.configuration().getDataCacheName();
bufSize == 0 ? cfg.getStreamBufferSize() : bufSize, mode, batch); throw new GridGgfsInvalidPathException("Failed to resolve parent directory: " + path); info = new GridGgfsFileInfo(cfg.getBlockSize(), /**affinity key*/null, evictExclude(path, mode == PRIMARY), props); cfg.getStreamBufferSize() : bufSize, mode, batch);
if (bytesProcessed >= ggfsCtx.configuration().getFragmentizerThrottlingBlockLength()) { ldr.flush(); U.sleep(ggfsCtx.configuration().getFragmentizerThrottlingDelay());
GridGgfsContext ggfsCtx = new GridGgfsContext( ctx, new GridGgfsConfiguration(cfg), new GridGgfsMetaManager(), new GridGgfsDataManager(), mgr.start(ggfsCtx); ggfsCache.put(maskName(cfg.getName()), ggfsCtx);
/** * @param ggfs Source GGFS. * @return Data transfer object for given GGFS. * @throws GridException */ public static VisorGgfs from(GridGgfs ggfs) throws GridException { assert ggfs != null; return new VisorGgfs( ggfs.name(), ggfs.configuration().getDefaultMode(), VisorGgfsMetrics.from(ggfs.metrics()), ggfs.configuration().getSecondaryFileSystem() != null ); }
/** * Constructs GGFS IPC handler. */ GridGgfsIpcHandler(GridGgfsContext ggfsCtx, boolean mgmt) { assert ggfsCtx != null; this.mgmt = mgmt; ctx = ggfsCtx.kernalContext(); ggfs = ggfsCtx.ggfs(); // Keep buffer size multiple of block size so no extra byte array copies is performed. bufSize = ggfsCtx.configuration().getBlockSize() * 2; log = ctx.log(GridGgfsIpcHandler.class); }
if (F.eq(ctx.name(), ggfsCfg.getDataCacheName())) { if (!ctx.isNear()) { ggfsDataCache = true; ggfsDataCacheSize = new LongAdder(); ggfsDataSpaceMax = ggfsCfg.getMaxSpaceSize();
ggfsCtx.configuration().getFragmentizerLocalWritesRatio()) {
while (fragmentingFiles.size() < ggfsCtx.configuration().getFragmentizerConcurrentFiles()) { GridGgfsFileInfo fileInfo = fileForFragmentizer(fragmentingFiles.keySet());
clone[i] = new GridGgfsConfiguration(ggfsCfgs[i]);
ggfs = ggfsCtx.ggfs(); dataCachePrj = ggfsCtx.kernalContext().cache().internalCache(ggfsCtx.configuration().getDataCacheName()); dataCache = ggfsCtx.kernalContext().cache().internalCache(ggfsCtx.configuration().getDataCacheName()); ggfsCtx.configuration().getDataCacheName()); .internalCache(ggfsCtx.configuration().getDataCacheName()).configuration().getAffinityMapper(); grpBlockSize = ggfsCtx.configuration().getBlockSize() * grpSize; String ggfsName = ggfsCtx.configuration().getName(); trashPurgeTimeout = ggfsCtx.configuration().getTrashPurgeTimeout(); putExecSvc = ggfsCtx.configuration().getDualModePutExecutorService(); putExecSvcShutdown = ggfsCtx.configuration().getDualModePutExecutorServiceShutdown(); else { int coresCnt = Runtime.getRuntime().availableProcessors(); maxPendingPuts = ggfsCtx.configuration().getDualModeMaxPendingPutsSize();
GridCacheConfiguration cacheCfg = cacheCfgs.get(ggfsCfg.getDataCacheName()); ggfsCfg.getName(), ggfsCfg.getBlockSize(), ((GridGgfsGroupDataBlocksKeyMapper)affMapper).groupSize(), ggfsCfg.getMetaCacheName(), ggfsCfg.getDataCacheName(), ggfsCfg.getDefaultMode(), ggfsCfg.getPathModes(), ggfsCfg.isFragmentizerEnabled()));
String name = cfg.getName(); "assign unique name to each): " + name); GridCacheAdapter<Object, Object> dataCache = ctx.cache().internalCache(cfg.getDataCacheName()); GridCache<Object, Object> metaCache = ctx.cache().cache(cfg.getMetaCacheName()); if (F.eq(cfg.getDataCacheName(), cfg.getMetaCacheName())) throw new GridException("Cannot use same cache as both data and meta cache: " + cfg.getName()); long maxSpaceSize = cfg.getMaxSpaceSize(); "to 0 and restart the grid): " + cfg.getDataCacheName()); if (cfg.getMaxSpaceSize() == 0 && dataCache.configuration().getMemoryMode() == OFFHEAP_VALUES) U.warn(log, "GGFS max space size is not specified but data cache values are stored off-heap (max " + "space will be limited to 80% of max JVM heap size): " + cfg.getName()); boolean secondary = cfg.getDefaultMode() == PROXY; if (cfg.getPathModes() != null) { for (Map.Entry<String, GridGgfsMode> mode : cfg.getPathModes().entrySet()) { if (mode.getValue() == PROXY) secondary = true; assertParameter(cfg.getSecondaryFileSystem() != null, "secondaryFileSystem cannot be null when mode is SECONDARY");
meta = ggfsCtx.meta(); data = ggfsCtx.data(); secondaryFs = cfg.getSecondaryFileSystem(); if (cfg.getDefaultMode() == PROXY) throw new GridException("Mode cannot be PROXY if secondary file system hasn't been defined."); dfltMode = cfg.getDefaultMode(); if (ggfsCtx.configuration().getPathModes() != null) { for (Map.Entry<String, GridGgfsMode> e : ggfsCtx.configuration().getPathModes().entrySet()) { if (!dfltModes.containsKey(e.getKey())) cfgModes.put(e.getKey(), e.getValue()); String dataCacheName = ggfsCtx.configuration().getDataCacheName();