public static KylinConfig loadKylinConfigFromHdfs(SerializableConfiguration conf, String uri) { HadoopUtil.setCurrentConfiguration(conf.get()); KylinConfig config = loadKylinConfigFromHdfs(uri); // This is a bad example where the thread local KylinConfig cannot be auto-closed due to // limitation of MR API. It works because MR task runs its own process. Do not copy. @SuppressWarnings("unused") SetAndUnsetThreadLocalConfig shouldAutoClose = KylinConfig.setAndUnsetThreadLocalConfig(config); return config; }
@Override public Tuple2<org.apache.hadoop.io.Text, org.apache.hadoop.io.Text> call( Tuple2<ByteArray, Object[]> tuple2) throws Exception { if (initialized == false) { synchronized (SparkCubingByLayer.class) { if (initialized == false) { KylinConfig kylinConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(sConf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kylinConfig)) { CubeDesc desc = CubeDescManager.getInstance(kylinConfig).getCubeDesc(cubeName); codec = new BufferedMeasureCodec(desc.getMeasures()); initialized = true; } } } } ByteBuffer valueBuf = codec.encode(tuple2._2()); org.apache.hadoop.io.Text textResult = new org.apache.hadoop.io.Text(); textResult.set(valueBuf.array(), 0, valueBuf.position()); return new Tuple2<>(new org.apache.hadoop.io.Text(tuple2._1().array()), textResult); } }).saveAsNewAPIHadoopDataset(job.getConfiguration());
@Override public Tuple2<org.apache.hadoop.io.Text, org.apache.hadoop.io.Text> call(Tuple2<Text, Object[]> tuple2) throws Exception { if (initialized == false) { synchronized (SparkCubingMerge.class) { if (initialized == false) { synchronized (SparkCubingMerge.class) { if (initialized == false) { KylinConfig kylinConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(sConf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kylinConfig)) { CubeDesc desc = CubeDescManager.getInstance(kylinConfig).getCubeDesc(cubeName); codec = new BufferedMeasureCodec(desc.getMeasures()); initialized = true; } } } } } } ByteBuffer valueBuf = codec.encode(tuple2._2()); byte[] encodedBytes = new byte[valueBuf.position()]; System.arraycopy(valueBuf.array(), 0, encodedBytes, 0, valueBuf.position()); return new Tuple2<>(tuple2._1(), new org.apache.hadoop.io.Text(encodedBytes)); } };
private void init() { KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kConfig)) { CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName); reducerMapping = new FactDistinctColumnsReducerMapping(cubeInstance); initialized = true; } }
public void init() { KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kConfig)) { CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName); cubeDesc = cubeInstance.getDescriptor(); aggregators = new MeasureAggregators(cubeDesc.getMeasures()); measureNum = cubeDesc.getMeasures().size(); } }
private void init() { kylinConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kylinConfig)) { CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).getCube(cubeName); dictMgr = DictionaryManager.getInstance(kylinConfig); mergingSegments = getMergingSegments(cubeInstance, segmentIds); } }
List<DictionaryInfo> dictInfos = Lists.newArrayList(); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kylinConfig)) { for (CubeSegment segment : mergingSegments) { if (segment.getDictResPath(col) != null) { .setAndUnsetThreadLocalConfig(kylinConfig)) { CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).getCube(cubeName); CubeSegment newSegment = cubeInstance.getSegmentById(segmentId);
@Test public void TestGetJobWorkingDir() throws IOException { FileSystem fileSystem = FileSystem.get(new Configuration()); Path jobWorkDirPath = null; KylinConfig kylinConfig = mock(KylinConfig.class); try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(kylinConfig)) { when(kylinConfig.getHiveTableDirCreateFirst()).thenReturn(true); when(kylinConfig.getHdfsWorkingDirectory()).thenReturn("/tmp/kylin/"); DefaultChainedExecutable defaultChainedExecutable = mock(DefaultChainedExecutable.class); defaultChainedExecutable.setId(RandomUtil.randomUUID().toString()); String jobWorkingDir = HiveInputBase.getJobWorkingDir(defaultChainedExecutable, KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()); jobWorkDirPath = new Path(jobWorkingDir); Assert.assertTrue(fileSystem.exists(jobWorkDirPath)); } finally { if (jobWorkDirPath != null) fileSystem.deleteOnExit(jobWorkDirPath); } }
public void init() { KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kConfig)) { CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName); this.cubeSegment = cubeInstance.getSegmentById(segmentId); this.cubeDesc = cubeInstance.getDescriptor(); this.ndCuboidBuilder = new NDCuboidBuilder(cubeSegment, new RowKeyEncoderProvider(cubeSegment)); this.rowKeySplitter = new RowKeySplitter(cubeSegment); } }
KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kConfig)) { CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName); cubeDesc = cubeInstance.getDescriptor();
try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(kylinConfig); SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
@Test public void testThreadLocalOverride() throws InterruptedException { final String metadata1 = "meta1"; final String metadata2 = "meta2"; // set system KylinConfig KylinConfig sysConfig = KylinConfig.getInstanceFromEnv(); sysConfig.setMetadataUrl(metadata1); assertEquals(metadata1, KylinConfig.getInstanceFromEnv().getMetadataUrl().toString()); // test thread-local override KylinConfig threadConfig = KylinConfig.createKylinConfig(new Properties()); threadConfig.setMetadataUrl(metadata2); try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(threadConfig)) { assertEquals(metadata2, KylinConfig.getInstanceFromEnv().getMetadataUrl().toString()); // other threads still use system KylinConfig final String[] holder = new String[1]; Thread child = new Thread(new Runnable() { @Override public void run() { holder[0] = KylinConfig.getInstanceFromEnv().getMetadataUrl().toString(); } }); child.start(); child.join(); assertEquals(metadata1, holder[0]); } }
@Override public Tuple2<ByteArray, Object[]> call(String[] rowArray) throws Exception { if (initialized == false) { synchronized (SparkCubingByLayer.class) { if (initialized == false) { KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kConfig)) { CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName); CubeDesc cubeDesc = cubeInstance.getDescriptor(); CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId); CubeJoinedFlatTableEnrich interDesc = new CubeJoinedFlatTableEnrich( EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc); long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId); baseCuboidBuilder = new BaseCuboidBuilder(kConfig, cubeDesc, cubeSegment, interDesc, AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid), MeasureIngester.create(cubeDesc.getMeasures()), cubeSegment.buildDictionaryMap()); initialized = true; } } } } baseCuboidBuilder.resetAggrs(); byte[] rowKey = baseCuboidBuilder.buildKey(rowArray); Object[] result = baseCuboidBuilder.buildValueObjects(rowArray); return new Tuple2<>(new ByteArray(rowKey), result); } }
private void init() { KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kConfig)) { CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName); CubeDesc cubeDesc = cubeInstance.getDescriptor(); CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId); CubeJoinedFlatTableEnrich intermediateTableDesc = new CubeJoinedFlatTableEnrich( EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc); keyValueBuilder = new KeyValueBuilder(intermediateTableDesc); reducerMapping = new FactDistinctColumnsReducerMapping(cubeInstance); tmpbuf = ByteBuffer.allocate(4096); int[] rokeyColumnIndexes = intermediateTableDesc.getRowKeyColumnIndexes(); Long[] cuboidIds = getCuboidIds(cubeSegment); Integer[][] cuboidsBitSet = CuboidUtil.getCuboidBitSet(cuboidIds, rokeyColumnIndexes.length); boolean isNewAlgorithm = isUsePutRowKeyToHllNewAlgorithm(cubeDesc); HLLCounter[] cuboidsHLL = getInitCuboidsHLL(cuboidIds.length, cubeDesc.getConfig().getCubeStatsHLLPrecision()); cuboidStatCalculator = new CuboidStatCalculator(rokeyColumnIndexes, cuboidIds, cuboidsBitSet, isNewAlgorithm, cuboidsHLL); allCols = reducerMapping.getAllDimDictCols(); initDictColDeduper(cubeDesc); initColumnIndex(intermediateTableDesc); initialized = true; } }
public static KylinConfig loadKylinConfigFromHdfs(SerializableConfiguration conf, String uri) { HadoopUtil.setCurrentConfiguration(conf.get()); KylinConfig config = loadKylinConfigFromHdfs(uri); // This is a bad example where the thread local KylinConfig cannot be auto-closed due to // limitation of MR API. It works because MR task runs its own process. Do not copy. @SuppressWarnings("unused") SetAndUnsetThreadLocalConfig shouldAutoClose = KylinConfig.setAndUnsetThreadLocalConfig(config); return config; }
@Override public Tuple2<org.apache.hadoop.io.Text, org.apache.hadoop.io.Text> call( Tuple2<ByteArray, Object[]> tuple2) throws Exception { if (initialized == false) { synchronized (SparkCubingByLayer.class) { if (initialized == false) { KylinConfig kylinConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(sConf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kylinConfig)) { CubeDesc desc = CubeDescManager.getInstance(kylinConfig).getCubeDesc(cubeName); codec = new BufferedMeasureCodec(desc.getMeasures()); initialized = true; } } } } ByteBuffer valueBuf = codec.encode(tuple2._2()); org.apache.hadoop.io.Text textResult = new org.apache.hadoop.io.Text(); textResult.set(valueBuf.array(), 0, valueBuf.position()); return new Tuple2<>(new org.apache.hadoop.io.Text(tuple2._1().array()), textResult); } }).saveAsNewAPIHadoopDataset(job.getConfiguration());
private void init() { KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kConfig)) { CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName); reducerMapping = new FactDistinctColumnsReducerMapping(cubeInstance); initialized = true; } }
public void init() { KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kConfig)) { CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName); cubeDesc = cubeInstance.getDescriptor(); aggregators = new MeasureAggregators(cubeDesc.getMeasures()); measureNum = cubeDesc.getMeasures().size(); } }
private void init() { kylinConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kylinConfig)) { CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).getCube(cubeName); dictMgr = DictionaryManager.getInstance(kylinConfig); mergingSegments = getMergingSegments(cubeInstance, segmentIds); } }
public void init() { KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kConfig)) { CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName); this.cubeSegment = cubeInstance.getSegmentById(segmentId); this.cubeDesc = cubeInstance.getDescriptor(); this.ndCuboidBuilder = new NDCuboidBuilder(cubeSegment, new RowKeyEncoderProvider(cubeSegment)); this.rowKeySplitter = new RowKeySplitter(cubeSegment); } }