static void init(Settings settings, Log log) { InitializationUtils.checkIdForOperation(settings); InitializationUtils.setFieldExtractorIfNotSet(settings, HiveFieldExtractor.class, log); InitializationUtils.discoverEsVersion(settings, log); }
static void initialDiscovery(Settings settings, Log log) { InitializationUtils.discoverEsVersion(settings, log); InitializationUtils.discoverNodesIfNeeded(settings, log); InitializationUtils.filterNonClientNodesIfNeeded(settings, log); InitializationUtils.filterNonDataNodesIfNeeded(settings, log); InitializationUtils.filterNonIngestNodesIfNeeded(settings, log); }
private void init(Configuration cfg) throws IOException { Settings settings = HadoopSettingsManager.loadFrom(cfg); Assert.hasText(settings.getResourceWrite(), String.format("No resource ['%s'] (index/query/location) specified", ES_RESOURCE)); // Need to discover the ESVersion before checking if index exists. InitializationUtils.discoverEsVersion(settings, log); InitializationUtils.checkIdForOperation(settings); InitializationUtils.checkIndexExistence(settings); if (HadoopCfgUtils.getReduceTasks(cfg) != null) { if (HadoopCfgUtils.getSpeculativeReduce(cfg)) { log.warn("Speculative execution enabled for reducer - consider disabling it to prevent data corruption"); } } else { if (HadoopCfgUtils.getSpeculativeMap(cfg)) { log.warn("Speculative execution enabled for mapper - consider disabling it to prevent data corruption"); } } //log.info(String.format("Starting to write/index to [%s][%s]", settings.getTargetUri(), settings.getTargetResource())); } }
@Override public void initialize(Configuration conf, Properties tbl, Properties partitionProperties) throws SerDeException { inspector = HiveUtils.structObjectInspector(tbl); structTypeInfo = HiveUtils.typeInfo(inspector); cfg = conf; settings = (cfg != null ? HadoopSettingsManager.loadFrom(cfg).merge(tbl) : HadoopSettingsManager.loadFrom(tbl)); alias = HiveUtils.alias(settings); version = InitializationUtils.discoverEsVersion(settings, log); HiveUtils.fixHive13InvalidComments(settings, tbl); this.tableProperties = tbl; trace = log.isTraceEnabled(); outputJSON = settings.getOutputAsJson(); if (outputJSON) { jsonFieldName = new Text(HiveUtils.discoverJsonFieldName(settings, alias)); } }
EsMajorVersion version = InitializationUtils.discoverEsVersion(settings, log); InitializationUtils.validateSettingsForReading(settings); List<NodeInfo> nodes = InitializationUtils.discoverNodesIfNeeded(settings, log);
InitializationUtils.discoverEsVersion(settings, log);
InitializationUtils.discoverEsVersion(clientSettings, LOG); Resource resource = new Resource(clientSettings, false); IndexExtractor iformat = ObjectUtils.instantiate(clientSettings.getMappingIndexExtractorClassName(), handlerSettings);
EsMajorVersion version = InitializationUtils.discoverEsVersion(settings, log); ValueReader reader = ObjectUtils.instantiate(settings.getSerializerValueReaderClassName(), settings);
static void init(Settings settings, Log log) { InitializationUtils.checkIdForOperation(settings); InitializationUtils.setFieldExtractorIfNotSet(settings, HiveFieldExtractor.class, log); InitializationUtils.discoverEsVersion(settings, log); }
static void initialDiscovery(Settings settings, Log log) { InitializationUtils.discoverEsVersion(settings, log); InitializationUtils.discoverNodesIfNeeded(settings, log); InitializationUtils.filterNonClientNodesIfNeeded(settings, log); InitializationUtils.filterNonDataNodesIfNeeded(settings, log); InitializationUtils.filterNonIngestNodesIfNeeded(settings, log); }
private void init(Configuration cfg) throws IOException { Settings settings = HadoopSettingsManager.loadFrom(cfg); Assert.hasText(settings.getResourceWrite(), String.format("No resource ['%s'] (index/query/location) specified", ES_RESOURCE)); // Need to discover the ESVersion before checking if index exists. InitializationUtils.discoverEsVersion(settings, log); InitializationUtils.checkIdForOperation(settings); InitializationUtils.checkIndexExistence(settings); if (HadoopCfgUtils.getReduceTasks(cfg) != null) { if (HadoopCfgUtils.getSpeculativeReduce(cfg)) { log.warn("Speculative execution enabled for reducer - consider disabling it to prevent data corruption"); } } else { if (HadoopCfgUtils.getSpeculativeMap(cfg)) { log.warn("Speculative execution enabled for mapper - consider disabling it to prevent data corruption"); } } //log.info(String.format("Starting to write/index to [%s][%s]", settings.getTargetUri(), settings.getTargetResource())); } }
private void init(Configuration cfg) throws IOException { Settings settings = HadoopSettingsManager.loadFrom(cfg); Assert.hasText(settings.getResourceWrite(), String.format("No resource ['%s'] (index/query/location) specified", ES_RESOURCE)); // Need to discover the ESVersion before checking if index exists. InitializationUtils.discoverEsVersion(settings, log); InitializationUtils.checkIdForOperation(settings); InitializationUtils.checkIndexExistence(settings); if (HadoopCfgUtils.getReduceTasks(cfg) != null) { if (HadoopCfgUtils.getSpeculativeReduce(cfg)) { log.warn("Speculative execution enabled for reducer - consider disabling it to prevent data corruption"); } } else { if (HadoopCfgUtils.getSpeculativeMap(cfg)) { log.warn("Speculative execution enabled for mapper - consider disabling it to prevent data corruption"); } } //log.info(String.format("Starting to write/index to [%s][%s]", settings.getTargetUri(), settings.getTargetResource())); } }
private void init(Configuration cfg) throws IOException { Settings settings = HadoopSettingsManager.loadFrom(cfg); Assert.hasText(settings.getResourceWrite(), String.format("No resource ['%s'] (index/query/location) specified", ES_RESOURCE)); // Need to discover the ESVersion before checking if index exists. InitializationUtils.discoverEsVersion(settings, log); InitializationUtils.checkIdForOperation(settings); InitializationUtils.checkIndexExistence(settings); if (HadoopCfgUtils.getReduceTasks(cfg) != null) { if (HadoopCfgUtils.getSpeculativeReduce(cfg)) { log.warn("Speculative execution enabled for reducer - consider disabling it to prevent data corruption"); } } else { if (HadoopCfgUtils.getSpeculativeMap(cfg)) { log.warn("Speculative execution enabled for mapper - consider disabling it to prevent data corruption"); } } //log.info(String.format("Starting to write/index to [%s][%s]", settings.getTargetUri(), settings.getTargetResource())); } }
public static PartitionWriter createWriter(Settings settings, int currentSplit, int totalSplits, Log log) { Version.logVersion(); InitializationUtils.validateSettings(settings); InitializationUtils.discoverEsVersion(settings, log); InitializationUtils.discoverNodesIfNeeded(settings, log); InitializationUtils.filterNonClientNodesIfNeeded(settings, log); InitializationUtils.filterNonDataNodesIfNeeded(settings, log); List<String> nodes = SettingsUtils.discoveredOrDeclaredNodes(settings); // check invalid splits (applicable when running in non-MR environments) - in this case fall back to Random.. int selectedNode = (currentSplit < 0) ? new Random().nextInt(nodes.size()) : currentSplit % nodes.size(); // select the appropriate nodes first, to spread the load before-hand SettingsUtils.pinNode(settings, nodes.get(selectedNode)); Resource resource = new Resource(settings, false); log.info(String.format("Writing to [%s]", resource)); // single index vs multi indices IndexExtractor iformat = ObjectUtils.instantiate(settings.getMappingIndexExtractorClassName(), settings); iformat.compile(resource.toString()); RestRepository repository = (iformat.hasPattern() ? initMultiIndices(settings, currentSplit, resource, log) : initSingleIndex(settings, currentSplit, resource, log)); return new PartitionWriter(settings, currentSplit, totalSplits, repository); }
@Override public void initialize(Configuration conf, Properties tbl, Properties partitionProperties) throws SerDeException { inspector = HiveUtils.structObjectInspector(tbl); structTypeInfo = HiveUtils.typeInfo(inspector); cfg = conf; settings = (cfg != null ? HadoopSettingsManager.loadFrom(cfg).merge(tbl) : HadoopSettingsManager.loadFrom(tbl)); alias = HiveUtils.alias(settings); version = InitializationUtils.discoverEsVersion(settings, log); HiveUtils.fixHive13InvalidComments(settings, tbl); this.tableProperties = tbl; trace = log.isTraceEnabled(); outputJSON = settings.getOutputAsJson(); if (outputJSON) { jsonFieldName = new Text(HiveUtils.discoverJsonFieldName(settings, alias)); } }
InitializationUtils.discoverEsVersion(settings, log); InitializationUtils.discoverNodesIfNeeded(settings, log); InitializationUtils.filterNonClientNodesIfNeeded(settings, log);
InitializationUtils.validateSettingsForReading(settings); EsMajorVersion version = InitializationUtils.discoverEsVersion(settings, log); List<NodeInfo> nodes = InitializationUtils.discoverNodesIfNeeded(settings, log); InitializationUtils.filterNonClientNodesIfNeeded(settings, log);
InitializationUtils.discoverEsVersion(settings, log);
InitializationUtils.discoverEsVersion(settings, log);
EsMajorVersion version = InitializationUtils.discoverEsVersion(settings, log); ValueReader reader = ObjectUtils.instantiate(settings.getSerializerValueReaderClassName(), settings);