public static Pair<ExternalScanOperatorDescriptor, AlgebricksPartitionConstraint> createExternalIndexingOp( JobSpecification spec, MetadataProvider metadataProvider, Dataset dataset, ARecordType itemType, RecordDescriptor indexerDesc, List<ExternalFile> files, SourceLocation sourceLoc) throws HyracksDataException, AlgebricksException { return getIndexingOperator(metadataProvider, spec, itemType, dataset, files == null ? MetadataManager.INSTANCE .getDatasetExternalFiles(metadataProvider.getMetadataTxnContext(), dataset) : files, indexerDesc, sourceLoc); }
protected IAdapterFactory getConfiguredAdapterFactory(Dataset dataset, String adapterName, Map<String, String> configuration, ARecordType itemType, ARecordType metaType) throws AlgebricksException { try { configuration.put(ExternalDataConstants.KEY_DATAVERSE, dataset.getDataverseName()); IAdapterFactory adapterFactory = AdapterFactoryProvider.getAdapterFactory( getApplicationContext().getServiceContext(), adapterName, configuration, itemType, metaType); // check to see if dataset is indexed Index filesIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(), dataset.getDatasetName(), dataset.getDatasetName().concat(IndexingConstants.EXTERNAL_FILE_INDEX_NAME_SUFFIX)); if (filesIndex != null && filesIndex.getPendingOp() == 0) { // get files List<ExternalFile> files = MetadataManager.INSTANCE.getDatasetExternalFiles(mdTxnCtx, dataset); Iterator<ExternalFile> iterator = files.iterator(); while (iterator.hasNext()) { if (iterator.next().getPendingOp() != ExternalFilePendingOp.NO_OP) { iterator.remove(); } } } return adapterFactory; } catch (Exception e) { throw new AlgebricksException("Unable to create adapter", e); } }
dataset.getDatasetName()); if (indexes.isEmpty()) { List<ExternalFile> files = MetadataManager.INSTANCE.getDatasetExternalFiles(mdTxnCtx, dataset); for (ExternalFile file : files) { MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
metadataFiles = MetadataManager.INSTANCE.getDatasetExternalFiles(mdTxnCtx, ds); deletedFiles = new ArrayList<>(); addedFiles = new ArrayList<>();
if (datasetState == TransactionState.BEGIN) { List<ExternalFile> files = MetadataManager.INSTANCE.getDatasetExternalFiles(mdTxnCtx, dataset); } else if (datasetState == TransactionState.READY_TO_COMMIT) { List<ExternalFile> files = MetadataManager.INSTANCE.getDatasetExternalFiles(mdTxnCtx, dataset);