/** * Setter for SerDe class name. */ public HCatTable serdeLib(String serde) { getSerDeInfo().setSerializationLib(serde); return this; }
public void setSerializationLib(String lib) { getSerdeInfo().setSerializationLib(lib); }
public void setSerializationLib(String lib) { getSerdeInfo().setSerializationLib(lib); }
/** * Setter for StorageHandler class. */ public HCatTable storageHandler(String storageHandler) throws HCatException { this.tblProps.put( org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE, storageHandler); LOG.warn("HiveStorageHandlers can't be reliably instantiated on the client-side. " + "Attempting to derive Input/OutputFormat settings from StorageHandler, on best effort: "); try { HiveStorageHandler sh = HiveUtils.getStorageHandler(getConf(), storageHandler); this.sd.setInputFormat(sh.getInputFormatClass().getName()); this.sd.setOutputFormat(sh.getOutputFormatClass().getName()); this.sd.getSerdeInfo().setSerializationLib(sh.getSerDeClass().getName()); } catch (HiveException e) { LOG.warn("Could not derive Input/OutputFormat and SerDe settings from storageHandler. " + "These values need to be set explicitly.", e); } return this; }
public HCatTable(String dbName, String tableName) { this.dbName = StringUtils.isBlank(dbName)? Warehouse.DEFAULT_DATABASE_NAME : dbName; this.tableName = tableName; this.sd = new StorageDescriptor(); this.sd.setInputFormat(DEFAULT_INPUT_FORMAT_CLASS); this.sd.setOutputFormat(DEFAULT_OUTPUT_FORMAT_CLASS); this.sd.setSerdeInfo(new SerDeInfo()); this.sd.getSerdeInfo().setSerializationLib(DEFAULT_SERDE_CLASS); this.sd.getSerdeInfo().setParameters(new HashMap<String, String>()); this.sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); // Default serialization format. }
private static StorageDescriptor makeStorageDescriptor(String tableName, List<Column> columns, Storage storage) { if (storage.isSkewed()) { throw new IllegalArgumentException("Writing to skewed table/partition is not supported"); } SerDeInfo serdeInfo = new SerDeInfo(); serdeInfo.setName(tableName); serdeInfo.setSerializationLib(storage.getStorageFormat().getSerDeNullable()); serdeInfo.setParameters(storage.getSerdeParameters()); StorageDescriptor sd = new StorageDescriptor(); sd.setLocation(emptyToNull(storage.getLocation())); sd.setCols(columns.stream() .map(ThriftMetastoreUtil::toMetastoreApiFieldSchema) .collect(toList())); sd.setSerdeInfo(serdeInfo); sd.setInputFormat(storage.getStorageFormat().getInputFormatNullable()); sd.setOutputFormat(storage.getStorageFormat().getOutputFormatNullable()); sd.setParameters(ImmutableMap.of()); Optional<HiveBucketProperty> bucketProperty = storage.getBucketProperty(); if (bucketProperty.isPresent()) { sd.setNumBuckets(bucketProperty.get().getBucketCount()); sd.setBucketCols(bucketProperty.get().getBucketedBy()); if (!bucketProperty.get().getSortedBy().isEmpty()) { sd.setSortCols(bucketProperty.get().getSortedBy().stream() .map(column -> new Order(column.getColumnName(), column.getOrder().getHiveOrder())) .collect(toList())); } } return sd; }
private static SerDeInfo getSerDeInfo(HiveRegistrationUnit unit) { State props = unit.getSerDeProps(); SerDeInfo si = new SerDeInfo(); si.setParameters(getParameters(props)); si.setName(unit.getTableName()); if (unit.getSerDeType().isPresent()) { si.setSerializationLib(unit.getSerDeType().get()); } return si; }
protected Table getDummyTable() throws SemanticException { Path dummyPath = createDummyFile(); Table desc = new Table(DUMMY_DATABASE, DUMMY_TABLE); desc.getTTable().getSd().setLocation(dummyPath.toString()); desc.getTTable().getSd().getSerdeInfo().setSerializationLib(NullStructSerDe.class.getName()); desc.setInputFormatClass(NullRowsInputFormat.class); desc.setOutputFormatClass(HiveIgnoreKeyTextOutputFormat.class); return desc; }
private Table getDummyTable() throws SemanticException { Path dummyPath = createDummyFile(); Table desc = new Table(DUMMY_DATABASE, DUMMY_TABLE); desc.getTTable().getSd().setLocation(dummyPath.toString()); desc.getTTable().getSd().getSerdeInfo().setSerializationLib(NullStructSerDe.class.getName()); desc.setInputFormatClass(NullRowsInputFormat.class); desc.setOutputFormatClass(HiveIgnoreKeyTextOutputFormat.class); return desc; }
public TableBuilder(Database database) { this.database = database; partitions = new ArrayList<>(); columnNames = new ArrayList<>(); columnTypes = new ArrayList<>(); partitionKeys = Collections.emptyList(); table = new Table(); table.setDbName(database.getName()); table.setTableType(TableType.MANAGED_TABLE.toString()); Map<String, String> tableParams = new HashMap<String, String>(); tableParams.put("transactional", Boolean.TRUE.toString()); table.setParameters(tableParams); sd = new StorageDescriptor(); sd.setInputFormat(OrcInputFormat.class.getName()); sd.setOutputFormat(OrcOutputFormat.class.getName()); sd.setNumBuckets(1); table.setSd(sd); serDeInfo = new SerDeInfo(); serDeInfo.setParameters(new HashMap<String, String>()); serDeInfo.getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); serDeInfo.setSerializationLib(OrcSerde.class.getName()); sd.setSerdeInfo(serDeInfo); }
private StorageDescriptor newStorageDescriptor(String location, List<Order> sortCols) { StorageDescriptor sd = new StorageDescriptor(); List<FieldSchema> cols = new ArrayList<FieldSchema>(2); cols.add(new FieldSchema("a", "varchar(25)", "still no comment")); cols.add(new FieldSchema("b", "int", "comment")); sd.setCols(cols); sd.setLocation(location); sd.setInputFormat(MockInputFormat.class.getName()); sd.setOutputFormat(MockOutputFormat.class.getName()); sd.setNumBuckets(1); SerDeInfo serde = new SerDeInfo(); serde.setSerializationLib(LazySimpleSerDe.class.getName()); sd.setSerdeInfo(serde); List<String> bucketCols = new ArrayList<String>(1); bucketCols.add("a"); sd.setBucketCols(bucketCols); if (sortCols != null) { sd.setSortCols(sortCols); } return sd; }
public Table createTestAvroTable(String dbName, String tableName, String tableSdLoc, Optional<String> partitionFieldName, boolean ignoreDbCreation) throws Exception { if (!ignoreDbCreation) { createTestDb(dbName); } Table tbl = org.apache.hadoop.hive.ql.metadata.Table.getEmptyTable(dbName, tableName); tbl.getSd().setLocation(tableSdLoc); tbl.getSd().getSerdeInfo().setSerializationLib(AvroSerDe.class.getName()); tbl.getSd().getSerdeInfo().setParameters(ImmutableMap.of(HiveAvroSerDeManager.SCHEMA_URL, "/tmp/dummy")); if (partitionFieldName.isPresent()) { tbl.addToPartitionKeys(new FieldSchema(partitionFieldName.get(), "string", "some comment")); } this.localMetastoreClient.createTable(tbl); return tbl; }
private void addSd(ArrayList<FieldSchema> cols, Table tbl) { StorageDescriptor sd = new StorageDescriptor(); sd.setCols(cols); sd.setCompressed(false); sd.setNumBuckets(1); sd.setParameters(new HashMap<String, String>()); sd.setBucketCols(new ArrayList<String>()); sd.setSerdeInfo(new SerDeInfo()); sd.getSerdeInfo().setName(tbl.getTableName()); sd.getSerdeInfo().setParameters(new HashMap<String, String>()); sd.getSerdeInfo().getParameters() .put(serdeConstants.SERIALIZATION_FORMAT, "1"); sd.setSortCols(new ArrayList<Order>()); sd.getSerdeInfo().setSerializationLib(LazySimpleSerDe.class.getName()); sd.setInputFormat(HiveInputFormat.class.getName()); sd.setOutputFormat(HiveOutputFormat.class.getName()); tbl.setSd(sd); }
private static void createTable(String tableName, String tablePerm) throws Exception { Table tbl = new Table(); tbl.setDbName(DATABASE); tbl.setTableName(tableName); StorageDescriptor sd = new StorageDescriptor(); sd.setCols(ColumnHolder.colMapping.get(tableName)); tbl.setSd(sd); sd.setParameters(new HashMap<String, String>()); sd.setSerdeInfo(new SerDeInfo()); sd.getSerdeInfo().setName(tbl.getTableName()); sd.getSerdeInfo().setParameters(new HashMap<String, String>()); sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName()); sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName()); sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); sd.getSerdeInfo().setSerializationLib( org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class.getName()); tbl.setPartitionKeys(ColumnHolder.partitionCols); hmsc.createTable(tbl); Path path = new Path(warehousedir, tableName); FileSystem fs = path.getFileSystem(hiveConf); fs.setPermission(path, new FsPermission(tablePerm)); }
Table build() { StorageDescriptor sd = new StorageDescriptor(); if (columns == null) { sd.setCols(Collections.emptyList()); } else { sd.setCols(columns); } SerDeInfo serdeInfo = new SerDeInfo(); serdeInfo.setSerializationLib(serde); serdeInfo.setName(tableName); sd.setSerdeInfo(serdeInfo); sd.setInputFormat(inputFormat); sd.setOutputFormat(outputFormat); if (location != null) { sd.setLocation(location); } Table table = new Table(); table.setDbName(dbName); table.setTableName(tableName); table.setSd(sd); table.setParameters(parameters); table.setOwner(owner); if (partitionKeys != null) { table.setPartitionKeys(partitionKeys); } table.setTableType(tableType.toString()); return table; } }
private void alterPartitionSpecInMemory(Table tbl, Map<String, String> partSpec, org.apache.hadoop.hive.metastore.api.Partition tpart, boolean inheritTableSpecs, String partPath) throws HiveException, InvalidOperationException { LOG.debug("altering partition for table " + tbl.getTableName() + " with partition spec : " + partSpec); if (inheritTableSpecs) { tpart.getSd().setOutputFormat(tbl.getTTable().getSd().getOutputFormat()); tpart.getSd().setInputFormat(tbl.getTTable().getSd().getInputFormat()); tpart.getSd().getSerdeInfo().setSerializationLib(tbl.getSerializationLib()); tpart.getSd().getSerdeInfo().setParameters( tbl.getTTable().getSd().getSerdeInfo().getParameters()); tpart.getSd().setBucketCols(tbl.getBucketCols()); tpart.getSd().setNumBuckets(tbl.getNumBuckets()); tpart.getSd().setSortCols(tbl.getSortCols()); } if (partPath == null || partPath.trim().equals("")) { throw new HiveException("new partition path should not be null or empty."); } tpart.getSd().setLocation(partPath); }
private void alterPartitionSpecInMemory(Table tbl, Map<String, String> partSpec, org.apache.hadoop.hive.metastore.api.Partition tpart, boolean inheritTableSpecs, String partPath) throws HiveException, InvalidOperationException { LOG.debug("altering partition for table " + tbl.getTableName() + " with partition spec : " + partSpec); if (inheritTableSpecs) { tpart.getSd().setOutputFormat(tbl.getTTable().getSd().getOutputFormat()); tpart.getSd().setInputFormat(tbl.getTTable().getSd().getInputFormat()); tpart.getSd().getSerdeInfo().setSerializationLib(tbl.getSerializationLib()); tpart.getSd().getSerdeInfo().setParameters( tbl.getTTable().getSd().getSerdeInfo().getParameters()); tpart.getSd().setBucketCols(tbl.getBucketCols()); tpart.getSd().setNumBuckets(tbl.getNumBuckets()); tpart.getSd().setSortCols(tbl.getSortCols()); } if (partPath == null || partPath.trim().equals("")) { throw new HiveException("new partition path should not be null or empty."); } tpart.getSd().setLocation(partPath); }
private void createTable(String dbName, String tableName) throws Exception { String databaseName = (dbName == null) ? Warehouse.DEFAULT_DATABASE_NAME : dbName; try { msc.dropTable(databaseName, tableName); } catch (Exception e) { } // can fail with NoSuchObjectException Table tbl = new Table(); tbl.setDbName(databaseName); tbl.setTableName(tableName); tbl.setTableType("MANAGED_TABLE"); StorageDescriptor sd = new StorageDescriptor(); sd.setCols(getTableColumns()); tbl.setPartitionKeys(getPartitionKeys()); tbl.setSd(sd); sd.setBucketCols(new ArrayList<String>(2)); sd.setSerdeInfo(new SerDeInfo()); sd.getSerdeInfo().setName(tbl.getTableName()); sd.getSerdeInfo().setParameters(new HashMap<String, String>()); sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); sd.getSerdeInfo().setSerializationLib(ColumnarSerDe.class.getName()); sd.setInputFormat(RCFileInputFormat.class.getName()); sd.setOutputFormat(RCFileOutputFormat.class.getName()); Map<String, String> tableParams = new HashMap<String, String>(); tbl.setParameters(tableParams); msc.createTable(tbl); }
private Table getTable(String dbName, String tblName, String typeName) throws NoSuchObjectException, MetaException, TException, AlreadyExistsException, InvalidObjectException { msc.dropTable(dbName, tblName); silentDropDatabase(dbName); msc.dropType(typeName); Type typ1 = new Type(); typ1.setName(typeName); typ1.setFields(new ArrayList<FieldSchema>(1)); typ1.getFields().add(new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, "")); msc.createType(typ1); Table tbl = new Table(); tbl.setDbName(dbName); tbl.setTableName(tblName); StorageDescriptor sd = new StorageDescriptor(); sd.setSerdeInfo(new SerDeInfo()); sd.getSerdeInfo().setName(tblName); sd.getSerdeInfo().setParameters(new HashMap<String, String>()); sd.getSerdeInfo().setSerializationLib(LazySimpleSerDe.class.getName()); sd.setInputFormat(HiveInputFormat.class.getName()); sd.setOutputFormat(HiveOutputFormat.class.getName()); tbl.setSd(sd); sd.setCols(typ1.getFields()); sd.setSerdeInfo(new SerDeInfo()); return tbl; }
sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); sd.getSerdeInfo().setSerializationLib(OrcSerde.class.getName()); sd.setInputFormat(OrcInputFormat.class.getName()); sd.setOutputFormat(OrcOutputFormat.class.getName());