@Override public void validateSQL(String query) throws Exception { final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); hiveCmdBuilder.addStatement(query); Pair<Integer, String> response = KylinConfig.getInstanceFromEnv().getCliCommandExecutor() .execute(hiveCmdBuilder.toString()); if (response.getFirst() != 0) { throw new IllegalArgumentException(response.getSecond()); } }
/** * only used by Deploy Util * @throws IOException */ @Override public void executeHQL(String hql) throws IOException { final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); hiveCmdBuilder.addStatement(hql); Pair<Integer, String> response = KylinConfig.getInstanceFromEnv().getCliCommandExecutor() .execute(hiveCmdBuilder.toString()); if (response.getFirst() != 0) { throw new IllegalArgumentException("Failed to execute hql [" + hql + "], error message is: " + response.getSecond()); } }
protected void createFlatHiveTable(KylinConfig config) throws IOException { final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); hiveCmdBuilder.overwriteHiveProps(config.getHiveConfigOverride()); hiveCmdBuilder.addStatement(getInitStatement()); hiveCmdBuilder.addStatement(getCreateTableStatement()); final String cmd = hiveCmdBuilder.toString(); stepLogger.log("Create and distribute table, cmd: "); stepLogger.log(cmd); Pair<Integer, String> response = config.getCliCommandExecutor().execute(cmd, stepLogger); Map<String, String> info = stepLogger.getInfo(); //get the flat Hive table size Matcher matcher = HDFS_LOCATION.matcher(cmd); if (matcher.find()) { String hiveFlatTableHdfsUrl = matcher.group(1); long size = getFileSize(hiveFlatTableHdfsUrl); info.put(ExecutableConstants.HDFS_BYTES_WRITTEN, "" + size); logger.info("HDFS_Bytes_Writen: " + size); } getManager().addJobInfo(getId(), info); if (response.getFirst() != 0) { throw new RuntimeException("Failed to create flat hive table, error code " + response.getFirst()); } }
protected void createFlatHiveTable(KylinConfig config) throws IOException { final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); hiveCmdBuilder.overwriteHiveProps(config.getHiveConfigOverride()); hiveCmdBuilder.addStatement(getCmd()); final String cmd = hiveCmdBuilder.toString(); stepLogger.log("cmd: "); stepLogger.log(cmd); Pair<Integer, String> response = config.getCliCommandExecutor().execute(cmd, stepLogger); getManager().addJobInfo(getId(), stepLogger.getInfo()); if (response.getFirst() != 0) { throw new RuntimeException("Failed to create flat hive table, error code " + response.getFirst()); } }
private void redistributeTable(KylinConfig config, int numReducers) throws IOException { final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); hiveCmdBuilder.overwriteHiveProps(config.getHiveConfigOverride()); hiveCmdBuilder.addStatement(getInitStatement()); hiveCmdBuilder.addStatement("set mapreduce.job.reduces=" + numReducers + ";\n"); hiveCmdBuilder.addStatement("set hive.merge.mapredfiles=false;\n"); hiveCmdBuilder.addStatement(getRedistributeDataStatement()); final String cmd = hiveCmdBuilder.toString(); stepLogger.log("Redistribute table, cmd: "); stepLogger.log(cmd); Pair<Integer, String> response = config.getCliCommandExecutor().execute(cmd, stepLogger); getManager().addJobInfo(getId(), stepLogger.getInfo()); if (response.getFirst() != 0) { throw new RuntimeException("Failed to redistribute flat hive table"); } }
/** * only used by Deploy Util * @throws IOException */ @Override public void executeHQL(String hql) throws IOException { final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); hiveCmdBuilder.addStatement(hql); Pair<Integer, String> response = KylinConfig.getInstanceFromEnv().getCliCommandExecutor() .execute(hiveCmdBuilder.toString()); if (response.getFirst() != 0) { throw new IllegalArgumentException("Failed to execute hql [" + hql + "], error message is: " + response.getSecond()); } }
@Override public void validateSQL(String query) throws Exception { final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); hiveCmdBuilder.addStatement(query); Pair<Integer, String> response = KylinConfig.getInstanceFromEnv().getCliCommandExecutor() .execute(hiveCmdBuilder.toString()); if (response.getFirst() != 0) { throw new IllegalArgumentException(response.getSecond()); } }
protected void createFlatHiveTable(KylinConfig config) throws IOException { final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); hiveCmdBuilder.overwriteHiveProps(config.getHiveConfigOverride()); hiveCmdBuilder.addStatement(getInitStatement()); hiveCmdBuilder.addStatement(getCreateTableStatement()); final String cmd = hiveCmdBuilder.toString(); stepLogger.log("Create and distribute table, cmd: "); stepLogger.log(cmd); Pair<Integer, String> response = config.getCliCommandExecutor().execute(cmd, stepLogger); Map<String, String> info = stepLogger.getInfo(); //get the flat Hive table size Matcher matcher = HDFS_LOCATION.matcher(cmd); if (matcher.find()) { String hiveFlatTableHdfsUrl = matcher.group(1); long size = getFileSize(hiveFlatTableHdfsUrl); info.put(ExecutableConstants.HDFS_BYTES_WRITTEN, "" + size); logger.info("HDFS_Bytes_Writen: " + size); } getManager().addJobInfo(getId(), info); if (response.getFirst() != 0) { throw new RuntimeException("Failed to create flat hive table, error code " + response.getFirst()); } }
private void redistributeTable(KylinConfig config, int numReducers) throws IOException { final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); hiveCmdBuilder.overwriteHiveProps(config.getHiveConfigOverride()); hiveCmdBuilder.addStatement(getInitStatement()); hiveCmdBuilder.addStatement("set mapreduce.job.reduces=" + numReducers + ";\n"); hiveCmdBuilder.addStatement("set hive.merge.mapredfiles=false;\n"); hiveCmdBuilder.addStatement(getRedistributeDataStatement()); final String cmd = hiveCmdBuilder.toString(); stepLogger.log("Redistribute table, cmd: "); stepLogger.log(cmd); Pair<Integer, String> response = config.getCliCommandExecutor().execute(cmd, stepLogger); getManager().addJobInfo(getId(), stepLogger.getInfo()); if (response.getFirst() != 0) { throw new RuntimeException("Failed to redistribute flat hive table"); } }