@Override public boolean haveSameSchema(HiveRegistrationUnit unit1, HiveRegistrationUnit unit2) { if (unit1.getSerDeProps().contains(HiveAvroSerDeManager.SCHEMA_LITERAL) && unit2.getSerDeProps().contains(HiveAvroSerDeManager.SCHEMA_LITERAL)) { return unit1.getSerDeProps().getProp(HiveAvroSerDeManager.SCHEMA_LITERAL) .equals(unit2.getSerDeProps().getProp(HiveAvroSerDeManager.SCHEMA_LITERAL)); } else if (unit1.getSerDeProps().contains(HiveAvroSerDeManager.SCHEMA_URL) && unit2.getSerDeProps().contains(HiveAvroSerDeManager.SCHEMA_URL)) { return unit1.getSerDeProps().getProp(HiveAvroSerDeManager.SCHEMA_URL) .equals(unit2.getSerDeProps().getProp(HiveAvroSerDeManager.SCHEMA_URL)); } return false; }
@Override public void updateSchema(HiveRegistrationUnit existingUnit, HiveRegistrationUnit newUnit) throws IOException { Preconditions.checkArgument( newUnit.getSerDeProps().contains(SCHEMA_LITERAL) || newUnit.getSerDeProps().contains(SCHEMA_URL)); if (newUnit.getSerDeProps().contains(SCHEMA_LITERAL)) { existingUnit.setSerDeProp(SCHEMA_LITERAL, newUnit.getSerDeProps().getProp(SCHEMA_LITERAL)); } else { existingUnit.setSerDeProp(SCHEMA_URL, newUnit.getSerDeProps().getProp(SCHEMA_URL)); } }
@Override public void addSerDeProperties(HiveRegistrationUnit source, HiveRegistrationUnit target) throws IOException { if (source.getSerDeType().isPresent()) { target.setSerDeType(source.getSerDeType().get()); } if (source.getInputFormat().isPresent()) { target.setInputFormat(source.getInputFormat().get()); } if (source.getOutputFormat().isPresent()) { target.setOutputFormat(source.getOutputFormat().get()); } if (source.getSerDeProps().contains(SCHEMA_LITERAL)) { target.setSerDeProp(SCHEMA_LITERAL, source.getSerDeProps().getProp(SCHEMA_LITERAL)); } if (source.getSerDeProps().contains(SCHEMA_URL)) { target.setSerDeProp(SCHEMA_URL, source.getSerDeProps().getProp(SCHEMA_URL)); } }
@SuppressWarnings("unchecked") public T compareParameters() { if (!this.result) { checkExistingIsSuperstate(this.existingUnit.getProps(), this.newUnit.getProps()); checkExistingIsSuperstate(this.existingUnit.getStorageProps(), this.newUnit.getStorageProps()); checkExistingIsSuperstate(this.existingUnit.getSerDeProps(), this.newUnit.getSerDeProps()); } return (T) this; }
private static SerDeInfo getSerDeInfo(HiveRegistrationUnit unit) { State props = unit.getSerDeProps(); SerDeInfo si = new SerDeInfo(); si.setParameters(getParameters(props)); si.setName(unit.getTableName()); if (unit.getSerDeType().isPresent()) { si.setSerializationLib(unit.getSerDeType().get()); } return si; }
props.putAll(unit.getProps().getProperties()); props.putAll(unit.getStorageProps().getProperties()); props.putAll(unit.getSerDeProps().getProperties());
private void validateSchemaUrl(State state, String targetSchemaFileName, boolean createConflictingFile) throws IOException { HiveAvroSerDeManager manager = new HiveAvroSerDeManager(state); HiveRegistrationUnit registrationUnit = (new HiveTable.Builder()).withDbName(TEST_DB).withTableName(TEST_TABLE).build(); // Clean up existing file String targetPathStr = new Path(this.testBasePath, targetSchemaFileName).toString(); File targetFile = new File(targetPathStr); targetFile.delete(); // create a conflicting file if (createConflictingFile) { targetFile.createNewFile(); } manager.addSerDeProperties(this.testBasePath, registrationUnit); Assert.assertNull(registrationUnit.getSerDeProps().getProp(HiveAvroSerDeManager.SCHEMA_LITERAL)); String schemaUrl = registrationUnit.getSerDeProps().getProp(HiveAvroSerDeManager.SCHEMA_URL); Assert.assertEquals(schemaUrl, targetPathStr); Assert.assertTrue(IOUtils.contentEquals(this.getClass().getResourceAsStream("/test-hive-table/hive-test.avsc"), new FileInputStream(schemaUrl))); }
/** * Test that the schema is written to the schema literal */ @Test public void testSchemaLiteral() throws IOException { State state = new State(); HiveAvroSerDeManager manager = new HiveAvroSerDeManager(state); HiveRegistrationUnit registrationUnit = (new HiveTable.Builder()).withDbName(TEST_DB).withTableName(TEST_TABLE).build(); manager.addSerDeProperties(this.testBasePath, registrationUnit); Assert.assertTrue(registrationUnit.getSerDeProps().getProp(HiveAvroSerDeManager.SCHEMA_LITERAL).contains("example.avro")); }
@Override public boolean haveSameSchema(HiveRegistrationUnit unit1, HiveRegistrationUnit unit2) { if (unit1.getSerDeProps().contains(HiveAvroSerDeManager.SCHEMA_LITERAL) && unit2.getSerDeProps().contains(HiveAvroSerDeManager.SCHEMA_LITERAL)) { return unit1.getSerDeProps().getProp(HiveAvroSerDeManager.SCHEMA_LITERAL) .equals(unit2.getSerDeProps().getProp(HiveAvroSerDeManager.SCHEMA_LITERAL)); } else if (unit1.getSerDeProps().contains(HiveAvroSerDeManager.SCHEMA_URL) && unit2.getSerDeProps().contains(HiveAvroSerDeManager.SCHEMA_URL)) { return unit1.getSerDeProps().getProp(HiveAvroSerDeManager.SCHEMA_URL) .equals(unit2.getSerDeProps().getProp(HiveAvroSerDeManager.SCHEMA_URL)); } return false; }
@Override public void updateSchema(HiveRegistrationUnit existingUnit, HiveRegistrationUnit newUnit) throws IOException { Preconditions.checkArgument( newUnit.getSerDeProps().contains(SCHEMA_LITERAL) || newUnit.getSerDeProps().contains(SCHEMA_URL)); if (newUnit.getSerDeProps().contains(SCHEMA_LITERAL)) { existingUnit.setSerDeProp(SCHEMA_LITERAL, newUnit.getSerDeProps().getProp(SCHEMA_LITERAL)); } else { existingUnit.setSerDeProp(SCHEMA_URL, newUnit.getSerDeProps().getProp(SCHEMA_URL)); } }
@Override public void addSerDeProperties(HiveRegistrationUnit source, HiveRegistrationUnit target) throws IOException { if (source.getSerDeType().isPresent()) { target.setSerDeType(source.getSerDeType().get()); } if (source.getInputFormat().isPresent()) { target.setInputFormat(source.getInputFormat().get()); } if (source.getOutputFormat().isPresent()) { target.setOutputFormat(source.getOutputFormat().get()); } if (source.getSerDeProps().contains(SCHEMA_LITERAL)) { target.setSerDeProp(SCHEMA_LITERAL, source.getSerDeProps().getProp(SCHEMA_LITERAL)); } if (source.getSerDeProps().contains(SCHEMA_URL)) { target.setSerDeProp(SCHEMA_URL, source.getSerDeProps().getProp(SCHEMA_URL)); } }
@SuppressWarnings("unchecked") public T compareParameters() { if (!this.result) { checkExistingIsSuperstate(this.existingUnit.getProps(), this.newUnit.getProps()); checkExistingIsSuperstate(this.existingUnit.getStorageProps(), this.newUnit.getStorageProps()); checkExistingIsSuperstate(this.existingUnit.getSerDeProps(), this.newUnit.getSerDeProps()); } return (T) this; }
private static SerDeInfo getSerDeInfo(HiveRegistrationUnit unit) { State props = unit.getSerDeProps(); SerDeInfo si = new SerDeInfo(); si.setParameters(getParameters(props)); si.setName(unit.getTableName()); if (unit.getSerDeType().isPresent()) { si.setSerializationLib(unit.getSerDeType().get()); } return si; }
props.putAll(unit.getProps().getProperties()); props.putAll(unit.getStorageProps().getProperties()); props.putAll(unit.getSerDeProps().getProperties());