@Override public Set<String> getSystemTagsToAdd() { Set<String> tags = new HashSet<>(); if (dataset instanceof RecordScannable) { tags.add(EXPLORE_TAG); } if (dataset instanceof FileSet || dataset instanceof PartitionedFileSet) { if (FileSetProperties.isExploreEnabled(dsProperties.getProperties())) { tags.add(EXPLORE_TAG); } } if (dataset instanceof BatchReadable || dataset instanceof BatchWritable || dataset instanceof InputFormatProvider || dataset instanceof OutputFormatProvider) { tags.add(BATCH_TAG); } boolean isLocalDataset = Boolean.parseBoolean(dsProperties.getProperties().get(Constants.AppFabric.WORKFLOW_LOCAL_DATASET_PROPERTY)); if (isLocalDataset) { tags.add(LOCAL_DATASET_TAG); } return tags; }
@Override public Set<String> getSystemTagsToAdd() { Set<String> tags = new HashSet<>(); if (dataset instanceof RecordScannable) { tags.add(EXPLORE_TAG); } if (dataset instanceof FileSet || dataset instanceof PartitionedFileSet) { if (FileSetProperties.isExploreEnabled(dsProperties.getProperties())) { tags.add(EXPLORE_TAG); } } if (dataset instanceof BatchReadable || dataset instanceof BatchWritable || dataset instanceof InputFormatProvider || dataset instanceof OutputFormatProvider) { tags.add(BATCH_TAG); } boolean isLocalDataset = Boolean.parseBoolean(dsProperties.getProperties().get(Constants.AppFabric.WORKFLOW_LOCAL_DATASET_PROPERTY)); if (isLocalDataset) { tags.add(LOCAL_DATASET_TAG); } return tags; }
if (FileSetProperties.isExploreEnabled(properties)) {
public PartitionedFileSetDataset(DatasetContext datasetContext, String name, Partitioning partitioning, FileSet fileSet, IndexedTable partitionTable, DatasetSpecification spec, Map<String, String> arguments, Provider<ExploreFacade> exploreFacadeProvider) { super(name, partitionTable); this.files = fileSet; this.partitionsTable = partitionTable; this.spec = spec; this.isExternal = FileSetProperties.isDataExternal(spec.getProperties()); this.exploreEnabled = FileSetProperties.isExploreEnabled(spec.getProperties()); this.runtimeArguments = arguments; this.partitioning = partitioning; this.exploreFacadeProvider = exploreFacadeProvider; this.datasetInstanceId = new DatasetId(datasetContext.getNamespaceId(), name); }
public PartitionedFileSetDataset(DatasetContext datasetContext, String name, Partitioning partitioning, FileSet fileSet, IndexedTable partitionTable, DatasetSpecification spec, Map<String, String> arguments, Provider<ExploreFacade> exploreFacadeProvider) { super(name, partitionTable); this.files = fileSet; this.partitionsTable = partitionTable; this.spec = spec; this.isExternal = FileSetProperties.isDataExternal(spec.getProperties()); this.exploreEnabled = FileSetProperties.isExploreEnabled(spec.getProperties()); this.runtimeArguments = arguments; this.partitioning = partitioning; this.exploreFacadeProvider = exploreFacadeProvider; this.datasetInstanceId = new DatasetId(datasetContext.getNamespaceId(), name); }
if (FileSetProperties.isExploreEnabled(properties)) {
@Override public void truncate() throws IOException { super.truncate(); // after underlying datasets are truncated, we need to clean up any existing Hive partitions // NOTE: if an error occurs below it may leave the dataset unexplorable, but re-issuing the command // should fix this. if (FileSetProperties.isExploreEnabled(spec.getProperties())) { ExploreFacade exploreFacade = exploreFacadeProvider.get(); if (exploreFacade != null) { DatasetId instanceId = new DatasetId(context.getNamespaceId(), spec.getName()); try { exploreFacade.disableExploreDataset(instanceId, spec); exploreFacade.enableExploreDataset(instanceId, spec, true); } catch (Exception e) { throw new DataSetException(String.format( "Unable to reset explore on dataset %s", instanceId), e); } } } } }
@Override public void truncate() throws IOException { super.truncate(); // after underlying datasets are truncated, we need to clean up any existing Hive partitions // NOTE: if an error occurs below it may leave the dataset unexplorable, but re-issuing the command // should fix this. if (FileSetProperties.isExploreEnabled(spec.getProperties())) { ExploreFacade exploreFacade = exploreFacadeProvider.get(); if (exploreFacade != null) { DatasetId instanceId = new DatasetId(context.getNamespaceId(), spec.getName()); try { exploreFacade.disableExploreDataset(instanceId, spec); exploreFacade.enableExploreDataset(instanceId, spec, true); } catch (Exception e) { throw new DataSetException(String.format( "Unable to reset explore on dataset %s", instanceId), e); } } } } }
if (FileSetProperties.isExploreEnabled(properties)) { LOG.debug("Enabling explore for dataset instance {}", datasetName); return generateFileSetCreateStatement(datasetId, dataset, properties, truncating);
if (FileSetProperties.isExploreEnabled(properties)) { LOG.debug("Enabling explore for dataset instance {}", datasetName); return generateFileSetCreateStatement(datasetId, dataset, properties, truncating);