@JsonIgnore public TimeUnit getIncomingTimeUnit() { return (_timeFieldSpec != null) ? _timeFieldSpec.getIncomingGranularitySpec().getTimeType() : null; }
public AvroRecordToPinotRowGenerator(@Nonnull Schema schema) { _schema = schema; // For time field, we use the incoming time field spec TimeFieldSpec timeFieldSpec = schema.getTimeFieldSpec(); Preconditions.checkNotNull(timeFieldSpec); _incomingTimeFieldSpec = new TimeFieldSpec(timeFieldSpec.getIncomingGranularitySpec()); }
timeUnits.put(col, tfs.getIncomingGranularitySpec().getTimeType()); break;
private List<GenericRow> createTestDataWithTimespec(TimeFieldSpec timeFieldSpec) { List<GenericRow> rows = new ArrayList<>(); Random random = new Random(); Map<String, Object> fields; for (int i = 0; i < NUM_ROWS; i++) { fields = new HashMap<>(); fields.put(D1, RandomStringUtils.randomAlphabetic(2)); fields.put(D2, RandomStringUtils.randomAlphabetic(5)); fields.put(M1, Math.abs(random.nextInt())); fields.put(M2, Math.abs(random.nextFloat())); long timestamp = System.currentTimeMillis(); Object timeColumnValue = timeFieldSpec.getIncomingGranularitySpec().fromMillis(timestamp); fields.put(timeFieldSpec.getName(), timeColumnValue); GenericRow row = new GenericRow(); row.init(fields); rows.add(row); } return rows; }
/** * @deprecated Load outside the class and use the setter for schema setting. * @throws IOException */ @Deprecated public void loadConfigFiles() throws IOException { Schema schema; if (_schemaFile != null) { schema = Schema.fromFile(new File(_schemaFile)); setSchema(schema); } else if (_format == FileFormat.AVRO) { schema = AvroUtils.getPinotSchemaFromAvroDataFile(new File(_inputFilePath)); setSchema(schema); } else { throw new RuntimeException("Input format " + _format + " requires schema."); } setTimeColumnName(schema.getTimeColumnName()); TimeFieldSpec timeFieldSpec = schema.getTimeFieldSpec(); if (timeFieldSpec != null) { setSegmentTimeUnit(timeFieldSpec.getIncomingGranularitySpec().getTimeType()); } else { setSegmentTimeUnit(TimeUnit.DAYS); } if (_readerConfigFile != null) { setReaderConfig(JsonUtils.fileToObject(new File(_readerConfigFile), CSVRecordReaderConfig.class)); } }
private List<GenericRow> createTestDataWithTimespec(TimeFieldSpec timeFieldSpec, DateTimeFieldSpec dateTimeFieldSpec) { List<GenericRow> rows = new ArrayList<>(); Random random = new Random(); Map<String, Object> fields; for (int i = 0; i < NUM_ROWS; i++) { fields = new HashMap<>(); fields.put(D1, RandomStringUtils.randomAlphabetic(2)); fields.put(D2, RandomStringUtils.randomAlphabetic(5)); fields.put(M1, Math.abs(random.nextInt())); fields.put(M2, Math.abs(random.nextFloat())); long timestamp = System.currentTimeMillis(); Object timeColumnValue = timeFieldSpec.getIncomingGranularitySpec().fromMillis(timestamp); fields.put(timeFieldSpec.getName(), timeColumnValue); DateTimeFormatSpec toFormat = new DateTimeFormatSpec(dateTimeFieldSpec.getFormat()); Object dateTimeColumnValue = toFormat.fromMillisToFormat(timestamp, Object.class); fields.put(dateTimeFieldSpec.getName(), dateTimeColumnValue); GenericRow row = new GenericRow(); row.init(fields); rows.add(row); } return rows; }
public TimeTransformer(Schema schema) { TimeFieldSpec timeFieldSpec = schema.getTimeFieldSpec(); if (timeFieldSpec != null) { TimeGranularitySpec incomingGranularitySpec = timeFieldSpec.getIncomingGranularitySpec(); TimeGranularitySpec outgoingGranularitySpec = timeFieldSpec.getOutgoingGranularitySpec(); // Perform time conversion only if incoming and outgoing granularity spec are different if (!incomingGranularitySpec.equals(outgoingGranularitySpec)) { _incomingTimeColumn = incomingGranularitySpec.getName(); _outgoingTimeColumn = outgoingGranularitySpec.getName(); _incomingTimeConverter = new TimeConverter(incomingGranularitySpec); _outgoingTimeConverter = new TimeConverter(outgoingGranularitySpec); } } }
@Test public void testNoVirtualColumnsInSchema() { Schema schema = new Schema(); FieldSpec spec = new DimensionFieldSpec("col1", FieldSpec.DataType.STRING, true); schema.addField(spec); TimeFieldSpec tfs = new TimeFieldSpec("col1", FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "col2", FieldSpec.DataType.LONG, TimeUnit.DAYS); schema.addField(tfs); VirtualColumnProviderFactory.addBuiltInVirtualColumnsToSchema(schema); Assert.assertEquals(schema.getColumnNames().size(), 5); Assert.assertEquals(schema.getTimeFieldSpec().getIncomingGranularitySpec().getTimeType(), TimeUnit.MILLISECONDS); RealtimeSegmentConverter converter = new RealtimeSegmentConverter(null, "", schema, "testTable", "col1", "segment1", "col1"); Schema newSchema = converter.getUpdatedSchema(schema); Assert.assertEquals(newSchema.getColumnNames().size(), 2); Assert.assertEquals(newSchema.getTimeFieldSpec().getIncomingGranularitySpec().getTimeType(), TimeUnit.DAYS); } }
@Test(dataProvider = "backfillRecordReaderDataProvider") public void testBackfillDateTimeRecordReader(RecordReader baseRecordReader, TimeFieldSpec timeFieldSpec, DateTimeFieldSpec dateTimeFieldSpec, Schema schemaExpected) throws Exception { BackfillDateTimeColumn backfillDateTimeColumn = new BackfillDateTimeColumn(new File("original"), new File("backup"), timeFieldSpec, dateTimeFieldSpec); try (BackfillDateTimeRecordReader wrapperReader = backfillDateTimeColumn .getBackfillDateTimeRecordReader(baseRecordReader)) { // check that schema has new column Schema schemaActual = wrapperReader.getSchema(); Assert.assertEquals(schemaActual, schemaExpected); DateTimeFieldSpec dateTimeFieldSpecActual = schemaActual.getDateTimeSpec(dateTimeFieldSpec.getName()); TimeFieldSpec timeFieldSpecActual = schemaActual.getTimeFieldSpec(); Assert.assertEquals(dateTimeFieldSpecActual, dateTimeFieldSpec); Assert.assertEquals(timeFieldSpecActual, timeFieldSpec); while (wrapperReader.hasNext()) { GenericRow next = wrapperReader.next(); // check that new datetime column is generated Object dateTimeColumnValueActual = next.getValue(dateTimeFieldSpec.getName()); Assert.assertNotNull(dateTimeColumnValueActual); Object timeColumnValueActual = next.getValue(timeFieldSpec.getName()); Assert.assertNotNull(timeColumnValueActual); // check that datetime column has correct value as per its format Long timeColumnValueMS = timeFieldSpec.getIncomingGranularitySpec().toMillis(timeColumnValueActual); DateTimeFormatSpec toFormat = new DateTimeFormatSpec(dateTimeFieldSpec.getFormat()); Object dateTimeColumnValueExpected = toFormat.fromMillisToFormat(timeColumnValueMS, Object.class); Assert.assertEquals(dateTimeColumnValueActual, dateTimeColumnValueExpected); } } }