private DatasetProperties createProperties(OutputFormatProvider outputFormatProvider) { FileSetProperties.Builder properties = FileSetProperties.builder(); if (!Strings.isNullOrEmpty(tpfsSinkConfig.basePath)) { properties.setBasePath(tpfsSinkConfig.basePath); } properties.setOutputFormat(outputFormatProvider.getOutputFormatClassName()); for (Map.Entry<String, String> formatProperty : outputFormatProvider.getOutputFormatConfiguration().entrySet()) { properties.setOutputProperty(formatProperty.getKey(), formatProperty.getValue()); } addFileSetProperties(properties); return properties.build(); }
Map<String, String> tableProperties = FileSetProperties.getTableProperties(properties); Map<String, String> oldTableProperties = FileSetProperties.getTableProperties(oldProperties); if (!oldTableProperties.equals(tableProperties)) { alterStatements.add(new AlterStatementBuilder(datasetName, databaseName, tableName, shouldEscapeColumns) String format = FileSetProperties.getExploreFormat(properties); Map<String, String> formatProps = FileSetProperties.getExploreFormatProperties(properties); String oldFormat = FileSetProperties.getExploreFormat(oldProperties); Map<String, String> oldFormatProps = FileSetProperties.getExploreFormatProperties(oldProperties); String schema = FileSetProperties.getExploreSchema(properties); Preconditions.checkNotNull(schema, "for native formats, explore schema must be given in dataset properties"); String delimiter = null; if ("text".equals(format)) { delimiter = FileSetProperties.getExploreFormatProperties(properties).get("delimiter"); } else if ("csv".equals(format)) { delimiter = ","; String serde = FileSetProperties.getSerDe(properties); String oldSerde = FileSetProperties.getSerDe(oldProperties); String inputFormat = FileSetProperties.getExploreInputFormat(properties); String oldInputFormat = FileSetProperties.getExploreInputFormat(oldProperties); String outputFormat = FileSetProperties.getExploreOutputFormat(properties); String oldOutputFormat = FileSetProperties.getExploreOutputFormat(oldProperties); if (!Objects.equals(FileSetProperties.getBasePath(properties), FileSetProperties.getBasePath(oldProperties))) { String schema = FileSetProperties.getExploreSchema(properties); String oldSchema = FileSetProperties.getExploreSchema(oldProperties);
if (FileSetProperties.isExploreEnabled(properties)) { if (FileSetProperties.isUseExisting(spec.getProperties())) { return null;
/** * @return the properties for the explore format */ public static Map<String, String> getExploreFormatProperties(Map<String, String> properties) { String format = getExploreFormat(properties); if (format == null) { return Collections.emptyMap(); } return propertiesWithPrefix(properties, String.format("%s.%s.", PROPERTY_EXPLORE_FORMAT, format)); }
FileSetAdmin(DatasetContext datasetContext, CConfiguration cConf, LocationFactory locationFactory, NamespacePathLocator namespacePathLocator, DatasetSpecification spec) throws IOException { this.spec = spec; this.isExternal = FileSetProperties.isDataExternal(spec.getProperties()); this.useExisting = FileSetProperties.isUseExisting(spec.getProperties()); this.possessExisting = FileSetProperties.isPossessExisting(spec.getProperties()); this.baseLocation = FileSetDataset.determineBaseLocation(datasetContext, cConf, spec, locationFactory, namespacePathLocator); this.datasetContext = datasetContext; this.cConf = cConf; this.locationFactory = locationFactory; this.namespacePathLocator = namespacePathLocator; }
Map<String, String> tableProperties = FileSetProperties.getTableProperties(properties); if (FileSetProperties.isUseExisting(tableProperties) || (FileSetProperties.isPossessExisting(tableProperties) && !truncating)) { try { exploreService.getTableInfo(datasetId.getNamespace(), databaseName, tableName); .setTableProperties(tableProperties); String schema = FileSetProperties.getExploreSchema(properties); String format = FileSetProperties.getExploreFormat(properties); if (format != null) { if ("parquet".equals(format)) { return createStatementBuilder.setSchema(FileSetProperties.getExploreSchema(properties)) .buildWithFileFormat("parquet"); String delimiter = null; if ("text".equals(format)) { delimiter = FileSetProperties.getExploreFormatProperties(properties).get("delimiter"); } else if ("csv".equals(format)) { delimiter = ","; String serde = FileSetProperties.getSerDe(properties); String inputFormat = FileSetProperties.getExploreInputFormat(properties); String outputFormat = FileSetProperties.getExploreOutputFormat(properties);
framework.addInstance(FileSet.class.getName(), MY_DS, FileSetProperties.builder() .setBasePath(baseDir.getPath()) .setDataExternal(true) framework.updateInstance(MY_DS, FileSetProperties.builder() .setBasePath(baseDir.getPath()) .setDataExternal(true) spec = framework.getDatasetSpec(MY_DS); Assert.assertNotNull(spec); Assert.assertEquals(baseDir.getPath(), FileSetProperties.getBasePath(spec.getProperties()));
@Override public DatasetSpecification reconfigure(String instanceName, DatasetProperties newProperties, DatasetSpecification currentSpec) throws IncompatibleUpdateException { validateProperties(newProperties.getProperties()); boolean wasExternal = FileSetProperties.isDataExternal(currentSpec.getProperties()); boolean isExternal = FileSetProperties.isDataExternal(newProperties.getProperties()); String oldPath = FileSetProperties.getBasePath(currentSpec.getProperties()); String newPath = FileSetProperties.getBasePath(newProperties.getProperties()); // validate that we are not "internalizing" an external location if (wasExternal && !isExternal) { throw new IncompatibleUpdateException( String.format("Cannot convert external file set '%s' to non-external.", instanceName)); } // allow change of path only if the dataset is external if (!Objects.equals(oldPath, newPath) && !isExternal) { throw new IncompatibleUpdateException( String.format("Cannot change the base path of non-external file set '%s'.", instanceName)); } return configure(instanceName, newProperties); }
@Override public Set<String> getSystemTagsToAdd() { Set<String> tags = new HashSet<>(); if (dataset instanceof RecordScannable) { tags.add(EXPLORE_TAG); } if (dataset instanceof FileSet || dataset instanceof PartitionedFileSet) { if (FileSetProperties.isExploreEnabled(dsProperties.getProperties())) { tags.add(EXPLORE_TAG); } } if (dataset instanceof BatchReadable || dataset instanceof BatchWritable || dataset instanceof InputFormatProvider || dataset instanceof OutputFormatProvider) { tags.add(BATCH_TAG); } boolean isLocalDataset = Boolean.parseBoolean(dsProperties.getProperties().get(Constants.AppFabric.WORKFLOW_LOCAL_DATASET_PROPERTY)); if (isLocalDataset) { tags.add(LOCAL_DATASET_TAG); } return tags; }
public PartitionedFileSetDataset(DatasetContext datasetContext, String name, Partitioning partitioning, FileSet fileSet, IndexedTable partitionTable, DatasetSpecification spec, Map<String, String> arguments, Provider<ExploreFacade> exploreFacadeProvider) { super(name, partitionTable); this.files = fileSet; this.partitionsTable = partitionTable; this.spec = spec; this.isExternal = FileSetProperties.isDataExternal(spec.getProperties()); this.exploreEnabled = FileSetProperties.isExploreEnabled(spec.getProperties()); this.runtimeArguments = arguments; this.partitioning = partitioning; this.exploreFacadeProvider = exploreFacadeProvider; this.datasetInstanceId = new DatasetId(datasetContext.getNamespaceId(), name); }
String basePath = FileSetProperties.getBasePath(spec.getProperties()); if (basePath == null) { basePath = spec.getName().replace('.', '/');
private String generateDisableStatement(DatasetId datasetId, DatasetSpecification spec) throws ExploreException { String tableName = tableNaming.getTableName(datasetId, spec.getProperties()); String databaseName = ExploreProperties.getExploreDatabaseName(spec.getProperties()); // If table does not exist, nothing to be done try { exploreService.getTableInfo(datasetId.getNamespace(), databaseName, tableName); } catch (TableNotFoundException e) { // Ignore exception, since this means table was not found. return null; } try (SystemDatasetInstantiator datasetInstantiator = datasetInstantiatorFactory.create()) { Dataset dataset = datasetInstantiator.getDataset(datasetId); try { if (dataset instanceof FileSet || dataset instanceof PartitionedFileSet) { // do not drop the explore table that dataset is reusing an existing table if (FileSetProperties.isUseExisting(spec.getProperties())) { return null; } } return generateDeleteStatement(dataset, databaseName, tableName); } finally { Closeables.closeQuietly(dataset); } } catch (IOException e) { LOG.error("Exception creating dataset classLoaderProvider for dataset {}.", datasetId, e); throw new ExploreException("Exception instantiating dataset " + datasetId); } }
Map<String, String> tableProperties = FileSetProperties.getTableProperties(properties); if (FileSetProperties.isUseExisting(tableProperties) || (FileSetProperties.isPossessExisting(tableProperties) && !truncating)) { try { exploreService.getTableInfo(datasetId.getNamespace(), databaseName, tableName); .setTableProperties(tableProperties); String schema = FileSetProperties.getExploreSchema(properties); String format = FileSetProperties.getExploreFormat(properties); if (format != null) { if ("parquet".equals(format)) { return createStatementBuilder.setSchema(FileSetProperties.getExploreSchema(properties)) .buildWithFileFormat("parquet"); String delimiter = null; if ("text".equals(format)) { delimiter = FileSetProperties.getExploreFormatProperties(properties).get("delimiter"); } else if ("csv".equals(format)) { delimiter = ","; String serde = FileSetProperties.getSerDe(properties); String inputFormat = FileSetProperties.getExploreInputFormat(properties); String outputFormat = FileSetProperties.getExploreOutputFormat(properties);
FileSetAdmin(DatasetContext datasetContext, CConfiguration cConf, LocationFactory locationFactory, NamespacedLocationFactory namespacedLocationFactory, DatasetSpecification spec) throws IOException { this.spec = spec; this.isExternal = FileSetProperties.isDataExternal(spec.getProperties()); this.useExisting = FileSetProperties.isUseExisting(spec.getProperties()); this.possessExisting = FileSetProperties.isPossessExisting(spec.getProperties()); this.baseLocation = FileSetDataset.determineBaseLocation(datasetContext, cConf, spec, locationFactory, namespacedLocationFactory); this.datasetContext = datasetContext; this.cConf = cConf; this.locationFactory = locationFactory; this.namespacedLocationFactory = namespacedLocationFactory; }
@Override public DatasetSpecification reconfigure(String instanceName, DatasetProperties newProperties, DatasetSpecification currentSpec) throws IncompatibleUpdateException { validateProperties(newProperties.getProperties()); boolean wasExternal = FileSetProperties.isDataExternal(currentSpec.getProperties()); boolean isExternal = FileSetProperties.isDataExternal(newProperties.getProperties()); String oldPath = FileSetProperties.getBasePath(currentSpec.getProperties()); String newPath = FileSetProperties.getBasePath(newProperties.getProperties()); // validate that we are not "internalizing" an external location if (wasExternal && !isExternal) { throw new IncompatibleUpdateException( String.format("Cannot convert external file set '%s' to non-external.", instanceName)); } // allow change of path only if the dataset is external if (!Objects.equals(oldPath, newPath) && !isExternal) { throw new IncompatibleUpdateException( String.format("Cannot change the base path of non-external file set '%s'.", instanceName)); } return configure(instanceName, newProperties); }
@Override public Set<String> getSystemTagsToAdd() { Set<String> tags = new HashSet<>(); if (dataset instanceof RecordScannable) { tags.add(EXPLORE_TAG); } if (dataset instanceof FileSet || dataset instanceof PartitionedFileSet) { if (FileSetProperties.isExploreEnabled(dsProperties.getProperties())) { tags.add(EXPLORE_TAG); } } if (dataset instanceof BatchReadable || dataset instanceof BatchWritable || dataset instanceof InputFormatProvider || dataset instanceof OutputFormatProvider) { tags.add(BATCH_TAG); } boolean isLocalDataset = Boolean.parseBoolean(dsProperties.getProperties().get(Constants.AppFabric.WORKFLOW_LOCAL_DATASET_PROPERTY)); if (isLocalDataset) { tags.add(LOCAL_DATASET_TAG); } return tags; }
public PartitionedFileSetDataset(DatasetContext datasetContext, String name, Partitioning partitioning, FileSet fileSet, IndexedTable partitionTable, DatasetSpecification spec, Map<String, String> arguments, Provider<ExploreFacade> exploreFacadeProvider) { super(name, partitionTable); this.files = fileSet; this.partitionsTable = partitionTable; this.spec = spec; this.isExternal = FileSetProperties.isDataExternal(spec.getProperties()); this.exploreEnabled = FileSetProperties.isExploreEnabled(spec.getProperties()); this.runtimeArguments = arguments; this.partitioning = partitioning; this.exploreFacadeProvider = exploreFacadeProvider; this.datasetInstanceId = new DatasetId(datasetContext.getNamespaceId(), name); }
String basePath = FileSetProperties.getBasePath(spec.getProperties()); if (basePath == null) { basePath = spec.getName().replace('.', '/');
private String generateDisableStatement(DatasetId datasetId, DatasetSpecification spec) throws ExploreException { String tableName = tableNaming.getTableName(datasetId, spec.getProperties()); String databaseName = ExploreProperties.getExploreDatabaseName(spec.getProperties()); // If table does not exist, nothing to be done try { exploreService.getTableInfo(datasetId.getNamespace(), databaseName, tableName); } catch (TableNotFoundException e) { // Ignore exception, since this means table was not found. return null; } try (SystemDatasetInstantiator datasetInstantiator = datasetInstantiatorFactory.create()) { Dataset dataset = datasetInstantiator.getDataset(datasetId); try { if (dataset instanceof FileSet || dataset instanceof PartitionedFileSet) { // do not drop the explore table that dataset is reusing an existing table if (FileSetProperties.isUseExisting(spec.getProperties())) { return null; } } return generateDeleteStatement(dataset, databaseName, tableName); } finally { Closeables.closeQuietly(dataset); } } catch (IOException e) { LOG.error("Exception creating dataset classLoaderProvider for dataset {}.", datasetId, e); throw new ExploreException("Exception instantiating dataset " + datasetId); } }
private DatasetProperties createProperties(InputFormatProvider inputFormatProvider) { FileSetProperties.Builder properties = FileSetProperties.builder(); if (!Strings.isNullOrEmpty(config.getBasePath())) { properties.setBasePath(config.getBasePath()); } properties.setInputFormat(inputFormatProvider.getInputFormatClassName()); for (Map.Entry<String, String> formatProperty : inputFormatProvider.getInputFormatConfiguration().entrySet()) { properties.setInputProperty(formatProperty.getKey(), formatProperty.getValue()); } addFileSetProperties(properties); return properties.build(); }