@Override public List<Entry<String, ByteString>> getSharedData() { return Collections.singletonList( new SimpleEntry<>( HIVE_ATTRIBUTE_KEY, ByteString.copyFrom(dataset.getReadDefinition().getExtendedProperty().asReadOnlyByteBuffer()) ) ); }
public ElasticTableXattr getExtendedAttributes(){ if(extendedAttributes == null){ try { extendedAttributes = ElasticTableXattr.parseFrom(tableMetadata.getReadDefinition().getExtendedProperty().toByteArray()); } catch (InvalidProtocolBufferException e) { throw Throwables.propagate(e); } } return extendedAttributes; }
private HiveTableXattr getExtended(){ if(extended == null){ try { extended = HiveTableXattr.parseFrom(getTableMetadata().getReadDefinition().getExtendedProperty().toByteArray()); } catch (InvalidProtocolBufferException e) { throw Throwables.propagate(e); } } return extended; }
@Override public boolean matches(RelOptRuleCall call) { final HiveScanDrel scan = call.rel(1); if (scan.getFilter() != null) { return false; } try { final HiveTableXattr tableXattr = HiveTableXattr.parseFrom(scan.getTableMetadata().getReadDefinition().getExtendedProperty().toByteArray()); final Optional<String> inputFormat = HiveReaderProtoUtil.getTableInputFormat(tableXattr); return inputFormat.isPresent() && inputFormat.get().equals(OrcInputFormat.class.getCanonicalName()); } catch (InvalidProtocolBufferException e) { logger.warn("Failure while attempting to deserialize hive table attributes.", e); } return false; }
@Override public SubScan getSpecificScan(List<SplitWork> work) throws ExecutionSetupException { List<DatasetSplit> splitWork = FluentIterable.from(work).transform(new Function<SplitWork, DatasetSplit>(){ @Override public DatasetSplit apply(SplitWork input) { return input.getSplit(); }}).toList(); return new ElasticsearchSubScan( getUserName(), getDataset().getStoragePluginId(), spec, splitWork, getColumns(), Iterables.getOnlyElement(getReferencedTables()), getSchema(), getDataset().getReadDefinition().getExtendedProperty() ); }
@Override public SubScan getSpecificScan(List<SplitWork> work) throws ExecutionSetupException { final BatchSchema schema = cachedRelDataType == null ? getDataset().getSchema(): BatchSchema.fromCalciteRowType(cachedRelDataType); // Create an abridged version of the splits to save network bytes. List<DatasetSplit> splits = work.stream().map( workSplit -> ProtostuffUtil.copy(workSplit.getSplit()) .setExtendedProperty(convertToScanXAttr(workSplit.getSplit().getExtendedProperty())) ).collect(Collectors.toList()); return new ParquetSubScan(dataset.getFormatSettings(), splits, getUserName(), schema, getDataset().getName().getPathComponents(), filter == null ? null : filter.getConditions(), dataset.getStoragePluginId(), columns, dataset.getReadDefinition().getPartitionColumnsList(), globalDictionaryEncodedColumns, dataset.getReadDefinition().getExtendedProperty()); }
@Override public void upgrade(UpgradeContext context) throws Exception { final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); try { for (SourceConfig source : namespaceService.getSources()) { if (!"HIVE".equalsIgnoreCase(ConnectionReader.toType(source))) { continue; } System.out.printf(" Handling Hive source %s%n", source.getName()); for (NamespaceKey datasetPath : namespaceService.getAllDatasets(new NamespaceKey(source.getName()))) { final DatasetConfig datasetConfig = namespaceService.getDataset(datasetPath); if (datasetConfig.getReadDefinition() == null || datasetConfig.getReadDefinition().getExtendedProperty() == null) { continue; } System.out.printf(" Clearing read definition of table %s%n", datasetPath.getSchemaPath()); datasetConfig.setReadDefinition(null); namespaceService.addOrUpdateDataset(datasetPath, datasetConfig); } } } catch (NamespaceException e) { throw new RuntimeException("Hive121BasedInputSplits failed", e); } }
if (datasetConfig.getReadDefinition() == null || datasetConfig.getReadDefinition().getExtendedProperty() == null) { continue; final byte[] original = readDefinition.getExtendedProperty().toByteArray();
@Override public SubScan getSpecificScan(List<SplitWork> work) throws ExecutionSetupException { final List<DatasetSplit> splits = new ArrayList<>(work.size()); final BatchSchema schema = getDataset().getSchema(); for(SplitWork split : work){ splits.add(split.getSplit()); } return new EasySubScan( getDataset().getFormatSettings(), splits, getUserName(), schema, getDataset().getName().getPathComponents(), dataset.getStoragePluginId(), columns, getDataset().getReadDefinition().getPartitionColumnsList(), getDataset().getReadDefinition().getExtendedProperty()); }
private PrelWithDictionaryInfo visitParquetScanPrel(ParquetScanPrel parquetScanPrel, Void value) throws RuntimeException { final ReadDefinition readDefinition = parquetScanPrel.getTableMetadata().getReadDefinition(); if (readDefinition == null || readDefinition.getExtendedProperty() == null) { return new PrelWithDictionaryInfo(parquetScanPrel); readDefinition.getExtendedProperty().toByteArray()); final DictionaryEncodedColumns dictionaryEncodedColumns = xAttr.getDictionaryEncodedColumns(); if (dictionaryEncodedColumns != null) {
HiveTableXattr tableXattr = HiveTableXattr.parseFrom(datasetConfig.getReadDefinition().getExtendedProperty().toByteArray()); return hasFSPermission(user, key, readSignature.getFsPartitionUpdateKeysList(), tableXattr);
} else { final HiveTableXattr tableXattr = HiveTableXattr.parseFrom(datasetConfig.getReadDefinition().getExtendedProperty().toByteArray());
ElasticTableXattr xattr = ElasticTableXattr.parseFrom(newConfig.getReadDefinition().getExtendedProperty().toByteArray()); newConfig.getReadDefinition().setExtendedProperty(ByteString.copyFrom(xattr.toBuilder().clearAnnotation().addAllAnnotation(result.getAnnotations()).build().toByteArray())); newConfig.setRecordSchema(ByteString.copyFrom(result.getSchema().serialize()));