@Override public void configureTableJobProperties(TableDesc tableDesc, Map<String, String> jobProperties) { configureInputJobProperties(tableDesc, jobProperties); configureOutputJobProperties(tableDesc, jobProperties); }
@Override public void configureJobConf(TableDesc tableDesc, JobConf jobConf) { Map<String, String> properties = new HashMap<>(); configureInputJobProperties(tableDesc, properties); configureOutputJobProperties(tableDesc, properties); properties.forEach(jobConf::set); try { KafkaUtils.copyDependencyJars(jobConf, KafkaStorageHandler.class); } catch (IOException e) { throw new RuntimeException(e); } }
@Test public void configureJobPropertiesWithDefaultValues() throws MetaException { KafkaStorageHandler kafkaStorageHandler = new KafkaStorageHandler(); TableDesc tableDesc = Mockito.mock(TableDesc.class); Properties properties = new Properties(); Table preCreateTable = new Table(); preCreateTable.putToParameters(KafkaTableProperties.HIVE_KAFKA_TOPIC.getName(), TEST_TOPIC); preCreateTable.putToParameters(KafkaTableProperties.HIVE_KAFKA_BOOTSTRAP_SERVERS.getName(), LOCALHOST_9291); preCreateTable.setTableType(TableType.EXTERNAL_TABLE.toString()); kafkaStorageHandler.preCreateTable(preCreateTable); preCreateTable.getParameters().forEach(properties::setProperty); Mockito.when(tableDesc.getProperties()).thenReturn(properties); Map<String, String> jobProperties = new HashMap<>(); kafkaStorageHandler.configureInputJobProperties(tableDesc, jobProperties); kafkaStorageHandler.configureOutputJobProperties(tableDesc, jobProperties); Assert.assertEquals(jobProperties.get(KafkaTableProperties.HIVE_KAFKA_TOPIC.getName()), TEST_TOPIC); Assert.assertEquals(jobProperties.get(KafkaTableProperties.HIVE_KAFKA_BOOTSTRAP_SERVERS.getName()), LOCALHOST_9291); Arrays.stream(KafkaTableProperties.values()) .filter(key -> !key.isMandatory()) .forEach((key) -> Assert.assertEquals("Wrong match for key " + key.getName(), key.getDefaultValue(), jobProperties.get(key.getName()))); }
Mockito.when(tableDesc.getProperties()).thenReturn(properties); Map<String, String> jobProperties = new HashMap<>(); kafkaStorageHandler.configureOutputJobProperties(tableDesc, jobProperties); Assert.assertEquals("false", jobProperties.get(KafkaTableProperties.HIVE_KAFKA_OPTIMISTIC_COMMIT.getName())); Assert.assertEquals(jobProperties.get(KafkaTableProperties.HIVE_KAFKA_TOPIC.getName()), TEST_TOPIC);
@Override public void configureTableJobProperties(TableDesc tableDesc, Map<String, String> jobProperties) { configureInputJobProperties(tableDesc, jobProperties); configureOutputJobProperties(tableDesc, jobProperties); }
@Override public void configureJobConf(TableDesc tableDesc, JobConf jobConf) { Map<String, String> properties = new HashMap<>(); configureInputJobProperties(tableDesc, properties); configureOutputJobProperties(tableDesc, properties); properties.forEach(jobConf::set); try { KafkaUtils.copyDependencyJars(jobConf, KafkaStorageHandler.class); } catch (IOException e) { throw new RuntimeException(e); } }