@Override public Row merge(Row a, Row b) { return new RowWithSchema(SCHEMA, name, a.<Boolean>getAs("result") && b.<Boolean>getAs("result")); }
public static Row remove(Row row, String fieldName) { List<StructField> removedFields = Lists.newArrayList(row.schema().fields()); removedFields.remove(row.fieldIndex(fieldName)); StructType removedSchema = new StructType(removedFields.toArray(new StructField[removedFields.size()])); List<Object> removedValues = Lists.newArrayList(RowUtils.valuesFor(row)); removedValues.remove(row.fieldIndex(fieldName)); return new RowWithSchema(removedSchema, removedValues.toArray()); }
@Override public Row getPrecedingTime(Row row) { Date date; try { date = format.parse(row.<String>getAs(field.name())); } catch (ParseException e) { throw new RuntimeException(e); } String precedingDateString = format.format(precedingDate(date)); return new RowWithSchema(getSchema(), precedingDateString); }
@Override public Dataset<Row> check(Dataset<Row> dataset, Map<String, Dataset<Row>> stepDependencies) { boolean schemasMatch = schemasMatch(requiredSchema, dataset.schema(), exactMatch); List<Row> datasetRows = Lists.newArrayList((Row)new RowWithSchema(SCHEMA, name, schemasMatch)); return Contexts.getSparkSession().createDataFrame(datasetRows, SCHEMA); }
public static Row subsetRow(Row row, StructType subsetSchema) { Object[] values = new Object[subsetSchema.length()]; int i = 0; for (String fieldName : subsetSchema.fieldNames()) { values[i] = row.get(row.fieldIndex(fieldName)); i++; } Row subset = new RowWithSchema(subsetSchema, values); return subset; }
public static Row append(Row row, String fieldName, DataType fieldType, Object value) { StructType appendedSchema = row.schema().add(fieldName, fieldType); Object[] appendedValues = ObjectArrays.concat(valuesFor(row), value); Row appendedRow = new RowWithSchema(appendedSchema, appendedValues); return appendedRow; }
@Override public Row getTime(Row row) { return new RowWithSchema(getSchema(), RowUtils.get(row, nanoField.name()), RowUtils.get(row, seqNumField.name())); }
private Row getCurrentSystemTimeRow(long currentSystemTimeMillis) { StructType schema = RowUtils.appendFields(systemEffectiveFromTimeModel.getSchema(), Lists.newArrayList(systemEffectiveToTimeModel.getSchema().fields())); Object[] nulls = new Object[schema.size()]; Row row = new RowWithSchema(schema, nulls); row = systemEffectiveFromTimeModel.setCurrentSystemTime(row); row = systemEffectiveToTimeModel.setCurrentSystemTime(row); return row; }
public static Row set(Row row, String fieldName, Object replacement) { Object[] values = new Object[row.length()]; for (int i = 0; i < row.schema().fields().length; i++) { if (i == row.fieldIndex(fieldName)) { values[i] = replacement; } else { values[i] = row.get(i); } } return new RowWithSchema(row.schema(), values); }
@Override public Row getPrecedingTime(Row row) { return new RowWithSchema(getSchema(), row.<Long>getAs(field.name()) - 1); } }
@Test public void testDifferent() { StructField field1 = DataTypes.createStructField("field1", DataTypes.StringType, true); StructField field2 = DataTypes.createStructField("field2", DataTypes.IntegerType, true); StructField field3 = DataTypes.createStructField("field3", DataTypes.FloatType, true); StructType schema = DataTypes.createStructType(Lists.newArrayList(field1, field2, field3)); Row row1 = new RowWithSchema(schema, "hello", 1, 2.0); Row row2 = new RowWithSchema(schema, "hello", 10, -2.0); assertTrue(RowUtils.different(row1, row2, Lists.newArrayList("field1", "field2", "field3"))); assertTrue(!RowUtils.different(row1, row2, Lists.newArrayList("field1"))); }
@Test public void testDifferentForDifferentSchemas() { StructField field1 = DataTypes.createStructField("field1", DataTypes.StringType, true); StructField field2 = DataTypes.createStructField("field2", DataTypes.IntegerType, true); StructField field3 = DataTypes.createStructField("field3", DataTypes.FloatType, true); StructType schema1 = DataTypes.createStructType(Lists.newArrayList(field1, field2, field3)); StructType schema2 = DataTypes.createStructType(Lists.newArrayList(field3, field2, field1)); Row row1 = new RowWithSchema(schema1, "hello", 1, 2.0); Row row2 = new RowWithSchema(schema2, -2.0, 10, "hello"); assertTrue(RowUtils.different(row1, row2, Lists.newArrayList("field1", "field2", "field3"))); assertTrue(!RowUtils.different(row1, row2, Lists.newArrayList("field1"))); }
@Test public void testCopyPrecedingTime() { Row row = new RowWithSchema(schemaWithTMs, 1000L, null); Row copied = PlannerUtils.copyPrecedingTime(row, firstTM, row, secondTM); assertEquals(copied.<Long>getAs("second"), (Long)999L); }
@Test public void testGetMutationType() { Row row = new RowWithSchema(schemaWithMT, "hello", MutationType.DELETE.toString()); assertEquals(MutationType.DELETE, PlannerUtils.getMutationType(row)); }
@Test public void testAppendMutationTypeField() { Row rowWithout = new RowWithSchema(schemaWithoutMT, "hello"); Row rowWith = PlannerUtils.appendMutationTypeField(rowWithout); assertEquals(rowWith.schema(), schemaWithMT); }
@Test public void testSetMutationType() { Row rowWith = new RowWithSchema(schemaWithMT, "hello", null); rowWith = PlannerUtils.setMutationType(rowWith, MutationType.DELETE); assertEquals(rowWith.<String>getAs(MutationType.MUTATION_TYPE_FIELD_NAME), MutationType.DELETE.toString()); Row rowWithout = new RowWithSchema(schemaWithoutMT, "hello"); rowWithout = PlannerUtils.setMutationType(rowWithout, MutationType.DELETE); assertEquals(rowWithout.<String>getAs(MutationType.MUTATION_TYPE_FIELD_NAME), MutationType.DELETE.toString()); }
@Test public void testSubsetRowAllFields() { StructField field1 = DataTypes.createStructField("field1", DataTypes.StringType, true); StructField field2 = DataTypes.createStructField("field2", DataTypes.IntegerType, true); StructField field3 = DataTypes.createStructField("field3", DataTypes.FloatType, true); StructType schema = DataTypes.createStructType(Lists.newArrayList(field1, field2, field3)); Row row = new RowWithSchema(schema, "hello", 1, 2.0); Row subsetRow = RowUtils.subsetRow(row, schema); assertEquals(row, subsetRow); }
@Test public void testRemoveMutationTypeField() { Row rowWith = new RowWithSchema(schemaWithMT, "hello", MutationType.DELETE.toString()); Row rowWithout = PlannerUtils.removeMutationTypeField(rowWith); assertEquals(rowWithout.schema(), schemaWithoutMT); }
@Test public void testHasMutationTypeField() { Row rowWith = new RowWithSchema(schemaWithMT, "hello", MutationType.DELETE.toString()); Row rowWithout = PlannerUtils.removeMutationTypeField(rowWith); assertTrue(PlannerUtils.hasMutationTypeField(rowWith)); assertFalse(PlannerUtils.hasMutationTypeField(rowWithout)); }
@Test public void testGet() { StructField field1 = DataTypes.createStructField("field1", DataTypes.StringType, true); StructField field2 = DataTypes.createStructField("field2", DataTypes.IntegerType, true); StructField field3 = DataTypes.createStructField("field3", DataTypes.FloatType, true); StructType schema = DataTypes.createStructType(Lists.newArrayList(field1, field2, field3)); Row row = new RowWithSchema(schema, "hello", 1, 2.0); assertEquals(RowUtils.get(row, "field1"), "hello"); assertEquals(RowUtils.get(row, "field2"), 1); assertEquals(RowUtils.get(row, "field3"), 2.0); }