/** * Returns whether this cell is encoded with Avro. * @return whether this cell is encoded with Avro. */ public boolean isAvro() { Preconditions.checkNotNull(mCellSchema); return (mCellSchema.getType() == SchemaType.INLINE) || (mCellSchema.getType() == SchemaType.CLASS) || (mCellSchema.getType() == SchemaType.AVRO); }
switch (cellSchema.getType()) { case AVRO: { if (cellSchema.getReaders() == null) { cellSchema.setReaders(Lists.<AvroSchema>newArrayList()); if (cellSchema.getWriters() == null) { cellSchema.setWriters(Lists.<AvroSchema>newArrayList()); if (cellSchema.getWritten() == null) { cellSchema.setWritten(Lists.<AvroSchema>newArrayList()); if (cellSchema.getDefaultReader() != null) { normalizeAvroSchema(cellSchema.getDefaultReader(), options); normalizeAvroSchemaCollection(cellSchema.getReaders(), options); normalizeAvroSchemaCollection(cellSchema.getWriters(), options); normalizeAvroSchemaCollection(cellSchema.getWritten(), options); throw new InvalidLayoutException(String.format( "Unsupported cell type: %s from CellSchema '%s'.", cellSchema.getType(), cellSchema));
switch (cellSchema.getType()) { case INLINE: { try { return new Schema.Parser().parse(cellSchema.getValue()); } catch (RuntimeException re) { throw new InvalidLayoutException(String.format( "Invalid Avro schema: '%s' caused error: %s.", cellSchema.getValue(), re.getMessage())); final String className = cellSchema.getValue(); return getSchemaFromClass(className); throw new InvalidLayoutException("Invalid schema type: " + cellSchema.getType());
/** * Initializes a ProtobufCellDecoder. * * @param cellSpec Specification of the cell encoding. * @throws IOException on I/O error. */ public ProtobufCellDecoder(CellSpec cellSpec) throws IOException { Preconditions.checkNotNull(cellSpec); Preconditions.checkArgument( cellSpec.getCellSchema().getType() == SchemaType.PROTOBUF); final String className = cellSpec.getCellSchema().getProtobufClassName(); try { mProtoClass = Class.forName(className).asSubclass(AbstractMessageLite.class); } catch (ClassNotFoundException cnfe) { throw new IOException(cnfe); } try { mParseFromMethod = mProtoClass.getMethod("parseFrom", byte[].class); } catch (NoSuchMethodException nsme) { throw new IOException(nsme); } }
throws IOException { Schema.Parser parser = new Schema.Parser(); switch (cellSchema.getType()) { case INLINE: return parser.parse(cellSchema.getValue()); case AVRO: AvroSchema avroSchema = cellSchema.getDefaultReader(); if (avroSchema.getUid() != null) { return schemaTable.getSchema(avroSchema.getUid()); String className = cellSchema.getValue(); try { SpecificRecord clazz = (SpecificRecord) Class.forName(className).newInstance(); default: throw new UnsupportedOperationException( "CellSchema " + cellSchema.getType() + " unsupported.");
mCellSchema = CellSchema.newBuilder(cellSchema).build(); // deep copy if (isAvro()) { switch (mCellSchema.getType()) { case AVRO: { mAvroReaderSchema = AvroReaderSchema.EXPLICIT; try { mReaderSchema = new Schema.Parser().parse(cellSchema.getValue()); } catch (RuntimeException re) { throw new InvalidLayoutException(String.format( "Invalid Avro schema '%s' caused error: %s.", cellSchema.getValue(), re.getMessage())); mAvroReaderSchema = AvroReaderSchema.EXPLICIT; try { mReaderSchema = getSchemaFromClass(cellSchema.getValue()); } catch (SchemaClassNotFoundException scnfe) {
switch (schema.getType()) { case INLINE: case CLASS: "Cell type %s requires table layout version >= %s, " + "got version %s in cell specification %s.", schema.getType(), Versions.LAYOUT_VALIDATION_VERSION, layoutVersion, "Cell type %s requires table layout version >= %s, " + "got version %s in cell specification %s.", schema.getType(), Versions.RAW_BYTES_CELL_ENCODING_VERSION, layoutVersion, "Cell type %s requires table layout version >= %s, " + "got version %s in cell specification %s.", schema.getType(), Versions.PROTOBUF_CELL_ENCODING_VERSION, layoutVersion, CellSpec.readAvroSchema(schema); } catch (SchemaClassNotFoundException scnfe) { LOG.debug(String.format("Avro schema class '%s' not found.", schema.getValue())); if (schema.getStorage() == SchemaStorage.FINAL) { switch (schema.getType()) { case INLINE:
/** * Returns a mutable list of registered schemas (READER, WRITER, WRITTEN) of the provided column. * * <p> * The list of schemas comes from a deeply nested record within the mutable table layout * descriptor. Therefore, mutating this list effectively mutates the table layout descriptor. * </p> * * @param columnName whose schema ids to list * @param schemaRegistrationType of the schemas to list: (READER, WRITER, WRITTEN). * @return The list of schema ids. * Returns empty list if schema validation was not previously enabled. * @throws NoSuchColumnException when column not found * @throws InvalidLayoutException if the column is final or non-AVRO */ private List<AvroSchema> getMutableRegisteredSchemaList( final FijiColumnName columnName, final SchemaRegistrationType schemaRegistrationType) throws NoSuchColumnException, InvalidLayoutException { Preconditions.checkNotNull(columnName); Preconditions.checkNotNull(schemaRegistrationType); final CellSchema cellSchema = getColumnSchema(columnName); // Avoid tracking schemas final or non-AVRO cells. if ((SchemaType.AVRO != cellSchema.getType()) || (SchemaStorage.FINAL == cellSchema.getStorage())) { throw new InvalidLayoutException("Final or non-AVRO column schema cannot be modified."); } final String fieldName = schemaRegistrationType.getCellSchemaFieldName(); return (List<AvroSchema>) Preconditions.checkNotNull(cellSchema.get(fieldName)); }
Preconditions.checkArgument(cellSpec.getCellSchema().getType() == SchemaType.PROTOBUF); mProtobufFullName = cellSpec.getCellSchema().getProtobufFullName(); Preconditions.checkNotNull(mProtobufFullName); final String className = cellSpec.getCellSchema().getProtobufClassName(); Class<?> protoClass = null; try {
throws IOException { switch (cellSchema.getAvroValidationPolicy()) { case NONE: { final Set<AvroSchema> writerSchemaIDs = Sets.newHashSet(); if (cellSchema.getReaders() != null) { readerSchemaIDs.addAll(cellSchema.getReaders()); if (cellSchema.getWriters() != null) { writerSchemaIDs.addAll(cellSchema.getWriters()); if (cellSchema.getWritten() != null) { writerSchemaIDs.addAll(cellSchema.getWritten()); if (refCellSchema.getReaders() != null) { readerSchemaIDs.addAll(refCellSchema.getReaders()); if (refCellSchema.getWriters() != null) { writerSchemaIDs.addAll(refCellSchema.getWriters()); if (refCellSchema.getWritten() != null) { writerSchemaIDs.addAll(refCellSchema.getWritten()); "Unknown validation policy: " + cellSchema.getAvroValidationPolicy());
/** * Get the Avro Schema for a given column marked FINAL in a given layout. This will either be * defined INLINE as a Schema JSON or it will be the only writer schema in the layout 1.3+ * Avro writers field. * * @param layout table layout from which to get the Schema for the given column. * @param column column for which to get the Schema. * @return Avro Schema of the given column. * @throws NoSuchColumnException in case the column does not exist. */ private static Schema getSchemaForFinalColumn(FijiTableLayout layout, FijiColumnName column) throws NoSuchColumnException { final List<AvroSchema> writers = layout.getCellSchema(column).getWriters(); if (null == writers) { return new Schema.Parser().parse(layout.getCellSchema(column).getValue()); } else { Preconditions.checkState(writers.size() == 1, "FINAL columns must have exactly one writer schema, found: " + writers); return new SchemaTableAvroResolver(layout.getSchemaTable()).apply(writers.get(0)); } }
/** * Returns the default Avro schema to use to decode cells from this column, if any. * * <p> For Avro-encoded cells only. </p> * * @return the default Avro schema to use to decode cells from this column, if any. */ public Schema getDefaultReaderSchema() { Preconditions.checkState(isAvro()); if ((null == mDefaultReaderSchema) && (mCellSchema.getDefaultReader() != null)) { Preconditions.checkNotNull(mAvroSchemaResolver); mDefaultReaderSchema = mAvroSchemaResolver.apply(mCellSchema.getDefaultReader()); } return mDefaultReaderSchema; }
/** * Get the Avro validation policy for a given column. * * @param column the column for which to get the Avro validation policy. * @return the Avro validation policy for the given column. * @throws NoSuchColumnException in case the column does not exist. * @throws InvalidLayoutException in case a fully qualified column in a map family is specified. */ public AvroValidationPolicy getAvroValidationPolicy(final FijiColumnName column) throws NoSuchColumnException, InvalidLayoutException { return getColumnSchema(column).getAvroValidationPolicy(); }
/** * Returns the underlying CellSchema Avro record, creating it if necessary. * @return the underlying CellSchema Avro record, creating it if necessary. */ private CellSchema getOrCreateCellSchema() { if (null == mCellSchema) { mCellSchema = new CellSchema(); } return mCellSchema; }
: mTableImportDescriptor.getColumnNameSourceMap().keySet()) { CellSchema cellSchema = mOutputTableLayout.getCellSchema(fijiColumnName); switch(cellSchema.getType()) { case AVRO: AvroSchema as = cellSchema.getDefaultReader(); if (as.getUid() != null) { Schema schema = schemaTable.getSchema(as.getUid()); FIJI_AVRO_TYPE_TO_CLASS_MAP.get(schemaType)); } else { throw new IOException("Unsupported described output type: " + cellSchema.getValue()); if (FIJI_CELL_TYPE_TO_CLASS_MAP.containsKey(cellSchema.getValue())) { columnNameClassMap.put(fijiColumnName, FIJI_CELL_TYPE_TO_CLASS_MAP.get(cellSchema.getValue())); } else { throw new IOException("Unsupported described output type: " + cellSchema.getValue()); throw new IOException("Unsupported described output type: " + cellSchema.getType()); default: throw new IOException("Unsupported described output type: " + cellSchema.getType());
/** * Get the Avro Schema to use to decode cells. * * @param layout FijiTableLayout from which to get storage information. * @param spec Specification of the cell encoding. * @return the Avro Schema to use to decode cells. * @throws IOException in case the specified column does not exist. */ private static Schema getReaderSchema(FijiTableLayout layout, BoundColumnReaderSpec spec) throws IOException { switch (spec.getColumnReaderSpec().getAvroReaderSchemaType()) { case DEFAULT: { final CellSchema cellSchema = layout.getCellSchema(spec.getColumn()); final SchemaTableAvroResolver resolver = new SchemaTableAvroResolver(layout.getSchemaTable()); return resolver.apply(cellSchema.getDefaultReader()); } case EXPLICIT: return spec.getColumnReaderSpec().getAvroReaderSchema(); case WRITER: return null; default: throw new InternalFijiError( "Unknown AvroReaderSchemaType: " + spec.getColumnReaderSpec().getAvroReaderSchemaType()); } }
/** * Reports the Avro schema validation policy. * * @param cellSpec to get the Avro schema validation policy from. * @return the schema validation policy. */ private static AvroValidationPolicy getAvroValidationPolicy(final CellSpec cellSpec) { final String validationPolicy = System.getProperty(SCHEMA_VALIDATION_POLICY); if (validationPolicy != null) { try { return AvroValidationPolicy.valueOf(validationPolicy); } catch (IllegalArgumentException iae) { throw new FijiEncodingException( String.format("Unrecognized validation policy: %s", validationPolicy), iae); } } else { return cellSpec.getCellSchema().getAvroValidationPolicy(); } }
/** * Returns whether this cell is a counter. * @return whether this cell is a counter. */ public boolean isCounter() { Preconditions.checkNotNull(mCellSchema); return mCellSchema.getType() == SchemaType.COUNTER; }
/** * Initializes a new RawBytesCellEncoder. * * @param cellSpec Specification of the cell to encode. * @throws IOException on I/O error. */ public RawBytesCellEncoder(final CellSpec cellSpec) throws IOException { mCellSpec = Preconditions.checkNotNull(cellSpec); Preconditions.checkArgument(cellSpec.getCellSchema().getType() == SchemaType.RAW_BYTES); }
/** * Initializes a new RawBytesCellDecoder. * * @param cellSpec Specification of the cell encoding. */ public RawBytesCellDecoder(CellSpec cellSpec) { Preconditions.checkNotNull(cellSpec); Preconditions.checkArgument( cellSpec.getCellSchema().getType() == SchemaType.RAW_BYTES); }