public Element toElement(final FixedWidthDatastore datastore, final String filename) { final Element ds = getDocument().createElement("fixed-width-datastore"); ds.setAttribute("name", datastore.getName()); if (!Strings.isNullOrEmpty(datastore.getDescription())) { ds.setAttribute("description", datastore.getDescription()); appendElement(ds, "encoding", datastore.getEncoding()); final int fixedValueWidth = datastore.getFixedValueWidth(); if (fixedValueWidth > -1) { final String valueOf = String.valueOf(fixedValueWidth); appendElement(widthElement, "fixed-value-width", valueOf); } else { final int[] valueWidths = datastore.getValueWidths(); for (int i = 0; i < valueWidths.length; i++) { appendElement(widthElement, "value-width", String.valueOf(valueWidths[i])); final List<String> customColumnNames = datastore.getCustomColumnNames(); if (customColumnNames != null && !customColumnNames.isEmpty()) { final Element columnNamesElement = getDocument().createElement("custom-column-names"); appendElement(ds, "header-line-number", datastore.getHeaderLineNumber()); appendElement(ds, "fail-on-inconsistencies", String.valueOf(datastore.isFailOnInconsistencies())); appendElement(ds, "skip-ebcdic-header", String.valueOf(datastore.isSkipEbcdicHeader())); appendElement(ds, "eol-present", String.valueOf(datastore.isEolPresent()));
_encodingComboBox.setSelectedItem(originalDatastore.getEncoding()); _failOnInconsistenciesCheckBox.setSelected(originalDatastore.isFailOnInconsistencies()); _skipEbcdicHeaderCheckBox.setSelected(originalDatastore.isSkipEbcdicHeader()); _eolPresentCheckBox.setSelected(originalDatastore.isEolPresent()); final List<String> customColumnNames = originalDatastore.getCustomColumnNames(); final int[] valueWidths = originalDatastore.getValueWidths(); _headerLineComboBox.setSelectedIndex(originalDatastore.getHeaderLineNumber());
private boolean isEbcdic() { // This is just a way how to differentiate between EBCDIC and normal FixedWidth configuration. // Perhaps there is a better way how to do this. return getEncoding().startsWith(EBCDIC_PREFIX); }
final Resource resource = fixedWidthDatastore.getResource(); final String datastorePath = resource.getQualifiedPath(); final FixedWidthConfiguration fixedWidthConfiguration = fixedWidthDatastore.getConfiguration(); final JavaRDD<String> rawInput; if (_minPartitions != null) {
private FixedWidthDatastore createDatastore(final String name, final Resource resource, final boolean failOnInconsistencies, final boolean skipEbcdicHeader, final boolean eolPresent) { final int[] valueWidths = getValueWidths(failOnInconsistencies); return new FixedWidthDatastore(name, resource, resource.getQualifiedPath(), _encodingComboBox.getSelectedItem(), valueWidths, failOnInconsistencies, skipEbcdicHeader, eolPresent, getHeaderLine(), getColumnNames()); }
public static boolean isHdfsResourcedDatastore(final Datastore datastore) { if (datastore instanceof CsvDatastore) { final CsvDatastore csvDatastore = (CsvDatastore) datastore; final Resource resource = csvDatastore.getResource(); if (!isHdfsResource(resource)) { return false; } } else if (datastore instanceof JsonDatastore) { final JsonDatastore jsonDatastore = (JsonDatastore) datastore; final Resource resource = jsonDatastore.getResource(); if (!isHdfsResource(resource)) { return false; } } else if (datastore instanceof FixedWidthDatastore) { final FixedWidthDatastore fixedWidthDatastore = (FixedWidthDatastore) datastore; final Resource resource = fixedWidthDatastore.getResource(); if (!isHdfsResource(resource)) { return false; } } else { // other type of datastore return false; } return true; }
@Override public String toString() { return "FixedWidthDatastore[name=" + getName() + ", filename=" + _filename + ", encoding=" + _encoding + ", headerLineNumber=" + _headerLineNumber + ", valueWidths=" + Arrays.toString(_valueWidths) + ", fixedValueWidth=" + _fixedValueWidth + "]"; } }
@Override protected UsageAwareDatastoreConnection<DataContext> createDatastoreConnection() { final FixedWidthConfiguration configuration = getConfiguration(); final Resource resource = _resourceRef.get(); final DataContext dataContext; if (resource == null) { logger.warn("Resource was not available, a local file reference will be created with path: {}", _filename); dataContext = DataContextFactory.createFixedWidthDataContext(new File(_filename), configuration); } else { dataContext = DataContextFactory.createFixedWidthDataContext(resource, configuration); } return new DatastoreConnectionImpl<>(dataContext, this); }
ds = new FixedWidthDatastore(name, resource, filename, encoding, valueWidths, failOnInconsistencies, skipEbcdicHeader, eolPresent, headerLineNumber, customColumnNames); } else { ds = new FixedWidthDatastore(name, resource, filename, encoding, fixedValueWidth, failOnInconsistencies, skipEbcdicHeader, eolPresent, headerLineNumber);
elem = toElement((JsonDatastore) datastore, filename); } else if (datastore instanceof FixedWidthDatastore) { final Resource resource = ((FixedWidthDatastore) datastore).getResource(); final String filename = toFilename(resource); elem = toElement((FixedWidthDatastore) datastore, filename);