try (final InputStream is = new FileInputStream(csvFile)) { try (final InputStreamReader reader = new InputStreamReader(is, charset)) { final CSVParser records = csvFormat.withFirstRecordAsHeader().parse(reader); RecordSchema lookupRecordSchema = null; for (final CSVRecord record : records) { final String key = record.get(lookupKeyColumn); record.toMap().forEach((k, v) -> { if (!lookupKeyColumn.equals(k)) { properties.put(k, v);
Iterator<String> it = csvRecord.iterator(); if (!result.isEmpty()) { String next = it.next();
@Override public List<Object> deserialize(ByteBuffer ser) { try { String data = new String(Utils.toByteArray(ser), StandardCharsets.UTF_8); CSVParser parser = CSVParser.parse(data, CSVFormat.RFC4180); CSVRecord record = parser.getRecords().get(0); Preconditions.checkArgument(record.size() == fieldNames.size(), "Invalid schema"); ArrayList<Object> list = new ArrayList<>(fieldNames.size()); for (int i = 0; i < record.size(); i++) { list.add(record.get(i)); } return list; } catch (IOException e) { throw new RuntimeException(e); } }
@Override public GenericRow next(GenericRow reuse) { CSVRecord record = _iterator.next(); for (FieldSpec fieldSpec : _schema.getAllFieldSpecs()) { String column = fieldSpec.getName(); String token = record.isSet(column) ? record.get(column) : null; Object value; if (fieldSpec.isSingleValueField()) { value = RecordReaderUtils.convertToDataType(token, fieldSpec); } else { String[] tokens = token != null ? StringUtils.split(token, _multiValueDelimiter) : null; value = RecordReaderUtils.convertToDataTypeArray(tokens, fieldSpec); } reuse.putField(column, value); } return reuse; }
.withHeader() .withDelimiter(delimiter) .withQuote(quote) .withRecordSeparator(recordSeparator); try (final CSVParser parser = new CSVParser(reader, format)) { ThreadMethods.throttledExecution(StreamMethods.enumerate(StreamMethods.stream(parser.spliterator(), false)).skip(skip).limit(limit), e -> { Integer rId = e.getKey(); CSVRecord row = e.getValue(); if (!row.isConsistent()) { logger.warn("WARNING: Skipping row {} because its size does not match the header size.", row.getRecordNumber()); TypeInference.DataType dataType = entry.getValue(); Object value = TypeInference.DataType.parse(row.get(column), dataType); //parse the string value according to the DataType if (yVariable != null && yVariable.equals(column)) { y = value;
try (CSVParser csvParser = new CSVParser(reader, CSVFormat.TDF)) { Iterator<CSVRecord> iterator = csvParser.iterator(); String field = record.get(0); if ((field.toUpperCase(Locale.ENGLISH).equals(field)) && (record.size() == 1)) { investigationSection = Arrays.asList(sections).contains(field); studySection = (studyFileName != null) && (field.equals(studySectionField)); break; String value = record.get(1); map.put(field, value); studyTarget = (field.equals(studyFileNameField)) && (value.equals(studyFileName));
/** * Returns a value by {@link Enum}. * * @param e * an enum * @return the String at the given enum String */ public String get(final Enum<?> e) { return get(e.toString()); }
try (CSVParser parser = new CSVParser(new StringReader(content), getCSVFormat().withHeader())) { Set<String> columns = parser.getHeaderMap().keySet(); Map<String, Field> fieldMap = getFieldMap(objectType); for (String column : columns) { Field field = fieldMap.get(caseSensitiveFieldNames ? column : column.toLowerCase()); String value = record.get(column); Object object = objectFromString(value, field.getType()); field.set(o, object);
public static void assertLabels(File labelFile, org.apache.samoa.TestParams testParams) throws IOException { LOG.info("Checking labels file " + labelFile.getAbsolutePath()); //1. parse result file with csv parser Reader in = new FileReader(labelFile); long lineCount = 0; long expectedLineCount = testParams.getInputInstances() / testParams.getLabelSamplingSize(); Iterable<CSVRecord> records = CSVFormat.EXCEL.withSkipHeaderRecord(false) .withIgnoreEmptyLines(true).withDelimiter(',').withCommentMarker('#').parse(in); Iterator<CSVRecord> iterator = records.iterator(); CSVRecord header = iterator.next(); while (iterator.hasNext()) { iterator.next(); lineCount = lineCount + 1; } Assert.assertEquals("Unexpected column", org.apache.samoa.TestParams.INSTANCE_ID, header.get(0).trim()); Assert.assertEquals("Unexpected column", org.apache.samoa.TestParams.TRUE_CLASS_VALUE, header.get(1).trim()); Assert.assertEquals("Unexpected column", org.apache.samoa.TestParams.PREDICTED_CLASS_VALUE, header.get(2).trim()); for (int i = 3; i < header.size(); i++) Assert.assertEquals("Unexpected column", org.apache.samoa.TestParams.VOTES, header.get(i).trim().substring(0, org.apache.samoa.TestParams.VOTES.length())); Assert.assertEquals("Wrong number of lines in prediction file", expectedLineCount, lineCount); }
@Override public Map<String, String> next() throws IOException { if (csvParser == null) { csvParser = CSVFormat.DEFAULT.withHeader().withDelimiter(delimiter).parse(inputStream); iterator = csvParser.iterator(); } if (!iterator.hasNext()) { return null; } CSVRecord csvRecord = iterator.next(); return csvRecord.toMap(); }
String input = "column1,column2\r\n" + "1,Foo\r\n" + "\"2\",\"Bar\"\r\n"; CSVFormat csvFormat = CSVFormat.RFC4180.withFirstRecordAsHeader() .withIgnoreEmptyLines(true) .withTrim(); try (CSVParser records = csvFormat.parse(new StringReader(input))) { for (CSVRecord record : records) { String column1 = record.get("column1"); String column2 = record.get("column2"); System.out.println(column1 + ": "+ column2); } }
public static Map<String, LatLong> postcodeToLatLong(String datasetProvider, DownloadUtils downloadUtils) throws Exception { Map<String, LatLong> postcodeToCoordMap = new HashMap<>(); InputStreamReader postcodeIsr = new InputStreamReader(downloadUtils.fetchInputStream(new URL(POSTCODE_TO_COORDINATE_URL), datasetProvider, ".csv")); CSVParser csvFileParser = new CSVParser(postcodeIsr, CSVFormat.DEFAULT); Iterator<CSVRecord> iter = csvFileParser.getRecords().iterator(); CSVRecord header = iter.next(); while (iter.hasNext()) { CSVRecord record = iter.next(); postcodeToCoordMap.put(record.get(1), new LatLong(record.get(2), record.get(3))); } return postcodeToCoordMap; } }
public List<Tuple> read(InputStream is) throws IOException { CSVFormat format = CSVFormat.RFC4180.withHeader().withDelimiter(delim.charAt(0)); CSVParser csvParser = new CSVParser(new InputStreamReader(is), format); List<CSVRecord> records = csvParser.getRecords(); header = csvParser.getHeaderMap().entrySet().stream() .sorted((e1, e2) -> e1.getValue().compareTo(e2.getValue())) .map(Map.Entry::getKey).toArray(String[]::new); labelCol = labelCol == -1 ? records.get(0).size() - 1 : labelCol; List<Tuple> ds = records.stream().parallel().map(this::extractValuedFeat).collect(Collectors.toList()); return ds; }
(double) totalHunspellComputationTime.get() / numMatches.get()))); SuggestionsOrdererConfig.setNgramsPath(args[1]); try (CSVParser parser = new CSVParser(new FileReader(args[0]), CSVFormat.DEFAULT.withFirstRecordAsHeader())) { for (CSVRecord record : parser) { String lang = record.get("language"); String covered = record.get("covered"); String replacement = record.get("replacement"); String sentenceStr = record.get("sentence");
final CSVParser parser = new CSVParser(in, CSVFormat.DEFAULT.withHeader()); String group = record.get("QUERY_GROUP"); String query = record.get("QUERY"); String explain = record.get("EXPLAIN_PLAN"); String tenantId = record.get("TENANT_ID"); long avgTime = Long.parseLong(record.get("AVG_TIME_MS")); long minTime = Long.parseLong(record.get("AVG_MIN_TIME_MS")); long numRuns = Long.parseLong(record.get("RUN_COUNT")); long rowCount = Long.parseLong(record.get("RESULT_ROW_COUNT")); Node node = new Node(minTime, avgTime, numRuns, explain, query, tenantId, label, rowCount); parser.close();
try (final InputStream is = new FileInputStream(csvFile)) { try (final InputStreamReader reader = new InputStreamReader(is, charset)) { final Iterable<CSVRecord> records = csvFormat.withFirstRecordAsHeader().parse(reader); for (final CSVRecord record : records) { final String key = record.get(lookupKeyColumn); final String value = record.get(lookupValueColumn); if (StringUtils.isBlank(key)) { throw new IllegalStateException("Empty lookup key encountered in: " + csvFile);
assertTrue(testName + " require 1 param", split.length >= 1); CSVFormat format = CSVFormat.newFormat(',').withQuote('"'); boolean checkComments = false; for (int i = 1; i < split.length; i++) { final String[] option_parts = option.split("=", 2); if ("IgnoreEmpty".equalsIgnoreCase(option_parts[0])) { format = format.withIgnoreEmptyLines(Boolean.parseBoolean(option_parts[1])); } else if ("IgnoreSpaces".equalsIgnoreCase(option_parts[0])) { format = format.withIgnoreSurroundingSpaces(Boolean.parseBoolean(option_parts[1])); try (final CSVParser parser = CSVParser.parse(resource, Charset.forName("UTF-8"), format)) { for (final CSVRecord record : parser) { String parsed = Arrays.toString(record.values()); if (checkComments) { final String comment = record.getComment().replace("\n", "\\n"); if (comment != null) { parsed += "#" + comment; final int count = record.size(); assertEquals(testName, readTestData(), count + ":" + parsed);
@Override public Map<String, String> getMapFromXLS(String coloumn1, String coloumn2) { Map<String, String> ret = new HashMap<>(); try { if (text == null) { text = http.getPageAsString(config.get(fileUrlKey)); } final CSVParser parser = new CSVParser(new StringReader(text), CSVFormat.EXCEL.withDelimiter(';').withFirstRecordAsHeader()); for (CSVRecord record : parser.getRecords()) { String c1 = record.get(coloumn1); String c2 = record.get(coloumn2); ret.put(c1, c2); } } catch (Exception e) { log.error(e.getMessage(), e); } return ret; }
/** * {@inheritDoc} */ @Override public DataModelIF<Long, Long> parseData(final File f) throws IOException { DataModelIF<Long, Long> dataset = new DataModel<>(); Reader in = new InputStreamReader(new FileInputStream(f), "UTF-8"); Iterable<CSVRecord> records; if (isHasHeader()) { records = CSVFormat.EXCEL.withDelimiter(getDelimiter()).withHeader().parse(in); } else { records = CSVFormat.EXCEL.withDelimiter(getDelimiter()).parse(in); } for (CSVRecord record : records) { long userID = Long.parseLong(record.get(getUserTok())); long itemID = Long.parseLong(record.get(getItemTok())); double preference = Double.parseDouble(record.get(getPrefTok())); dataset.addPreference(userID, itemID, preference); } in.close(); return dataset; } }