@Override public void process(InputStream in) throws IOException { try (DataFileStream<Record> stream = new DataFileStream<>( in, AvroUtil.newDatumReader(schema, Record.class))) { IncompatibleSchemaException.check( SchemaValidationUtil.canRead(stream.getSchema(), schema), "Incompatible file schema %s, expected %s", stream.getSchema(), schema); long written = 0L; try (DatasetWriter<Record> writer = target.newWriter()) { for (Record record : stream) { writer.write(record); written += 1; } } finally { session.adjustCounter("Stored records", written, true /* cannot roll back the write */); } } } });
/** * Precondition-style validation that throws a {@link ValidationException}. * * @param isValid * {@code true} if valid, {@code false} if an exception should be * thrown * @param message * A String message for the exception. */ public static void check(boolean isValid, String message, Object... args) { if (!isValid) { String[] argStrings = new String[args.length]; for (int i = 0; i < args.length; i += 1) { argStrings[i] = String.valueOf(args[i]); } throw new IncompatibleSchemaException( String.format(String.valueOf(message), (Object[]) argStrings)); } } }
throw new IncompatibleSchemaException("Schema cannot read data " + "written using existing schema. Schema: " + schema.toString(true) +
+ " New schema: " + entitySchema.getRawSchema(); throw new IncompatibleSchemaException(msg); + " New schema: " + entitySchema.getRawSchema(); throw new IncompatibleSchemaException(msg); + " New schema: " + entitySchema.getRawSchema(); throw new IncompatibleSchemaException(msg);
/** * Merges two {@link Schema} instances if they are compatible. * <p> * Two schemas are incompatible if: * <ul> * <li>The {@link Schema.Type} does not match.</li> * <li>For record schemas, the record name does not match</li> * <li>For enum schemas, the enum name does not match</li> * </ul> * <p> * Map value and array element types will use unions if necessary, and union * schemas are merged recursively. * * @param left a {@code Schema} * @param right a {@code Schema} * @return a merged {@code Schema} * @throws IncompatibleSchemaException if the schemas are not compatible */ public static Schema merge(Schema left, Schema right) { Schema merged = mergeOnly(left, right); IncompatibleSchemaException.check(merged != null, "Cannot merge %s and %s", left, right); return merged; }
+ " New schema: " + entitySchema.getRawSchema(); throw new IncompatibleSchemaException(msg); + " New schema: " + entitySchema.getRawSchema(); throw new IncompatibleSchemaException(msg); + " New schema: " + entitySchema.getRawSchema(); throw new IncompatibleSchemaException(msg);
protected void checkSchemaForWrite() { IncompatibleSchemaException.check(canWrite, "Cannot write data with this view's schema, " + "it cannot be read with the dataset's schema:\n" + "Current schema: %s\nDataset schema: %s", getSchema(), dataset.getDescriptor().getSchema()); }
throw new IncompatibleSchemaException( "Schema already exists as version: " + getEntityVersion(tableName, entityName, newEntitySchema)); + schemaString + " New schema: " + newEntitySchema.getRawSchema(); throw new IncompatibleSchemaException(msg); + Integer.toString(version) + ": Old schema: " + schemaString + " New schema: " + newEntitySchema.getRawSchema(); throw new IncompatibleSchemaException(msg);
protected void checkSchemaForRead() { IncompatibleSchemaException.check(canRead, "Cannot read data with this view's schema:\n" + "Current schema: %s\nDataset schema: %s", dataset.getDescriptor().getSchema(), getSchema()); } }
throw new IncompatibleSchemaException( "Schema already exists as version: " + getEntityVersion(tableName, entityName, newEntitySchema)); + schemaString + " New schema: " + newEntitySchema.getRawSchema(); throw new IncompatibleSchemaException(msg); + Integer.toString(version) + ": Old schema: " + schemaString + " New schema: " + newEntitySchema.getRawSchema(); throw new IncompatibleSchemaException(msg);
@Override public void process(InputStream in) throws IOException { try (DataFileStream<Record> stream = new DataFileStream<>( in, AvroUtil.newDatumReader(schema, Record.class))) { IncompatibleSchemaException.check( SchemaValidationUtil.canRead(stream.getSchema(), schema), "Incompatible file schema %s, expected %s", stream.getSchema(), schema); long written = 0L; try (DatasetWriter<Record> writer = target.newWriter()) { for (Record record : stream) { writer.write(record); written += 1; } } finally { session.adjustCounter("Stored records", written, true /* cannot roll back the write */); } } } });
/** * Checks that the {@code existing} {@link DatasetDescriptor} is compatible * with {@code test}. * * @param existing the current {@code DatasetDescriptor} for a dataset * @param test a new {@code DatasetDescriptor} for the same dataset */ public static void checkCompatible(DatasetDescriptor existing, DatasetDescriptor test) { checkNotChanged("format", existing.getFormat(), test.getFormat()); checkNotChanged("partitioning", existing.isPartitioned(), test.isPartitioned()); if (existing.isPartitioned()) { checkStrategyUpdate( existing.getPartitionStrategy(), test.getPartitionStrategy(), test.getSchema()); } // check can read records written with old schema using new schema Schema oldSchema = existing.getSchema(); Schema testSchema = test.getSchema(); if (!SchemaValidationUtil.canRead(oldSchema, testSchema)) { throw new IncompatibleSchemaException("Schema cannot read data " + "written using existing schema. Schema: " + testSchema.toString(true) + "\nExisting schema: " + oldSchema.toString(true)); } }
protected AbstractRefinableView(AbstractRefinableView<?> view, Schema schema, Class<E> type) { if (view.dataset instanceof AbstractDataset) { this.dataset = ((AbstractDataset<?>) view.dataset).asType(type); } else { this.dataset = Datasets.load(view.dataset.getUri(), type); } this.comparator = view.comparator; this.constraints = view.constraints; // thread-safe, so okay to reuse when views share a partition strategy this.keys = view.keys; // Resolve our type according to the given schema this.accessor = DataModelUtil.accessor(type, schema); this.entityTest = constraints.toEntityPredicate(accessor); Schema datasetSchema = dataset.getDescriptor().getSchema(); this.canRead = SchemaValidationUtil.canRead( datasetSchema, accessor.getReadSchema()); this.canWrite = SchemaValidationUtil.canRead( accessor.getWriteSchema(), datasetSchema); IncompatibleSchemaException.check(canRead || canWrite, "The type cannot be used to read from or write to the dataset:\n" + "Type schema: %s\nDataset schema: %s", getSchema(), datasetSchema); }
ManagedSchema managedSchema = getManagedSchema(tableName, entityName); if (managedSchema != null) { throw new IncompatibleSchemaException( "Cannot create schema when one already exists");
accessor.getWriteSchema(), datasetSchema); IncompatibleSchemaException.check(canRead || canWrite, "The type cannot be used to read from or write to the dataset:\n" + "Type schema: %s\nDataset schema: %s",
ManagedSchema managedSchema = getManagedSchema(tableName, entityName); if (managedSchema != null) { throw new IncompatibleSchemaException( "Cannot create schema when one already exists");