public static void writeSchemaToFile(Schema schema, Path filePath, FileSystem fs, boolean overwrite) throws IOException { writeSchemaToFile(schema, filePath, fs, overwrite, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.READ)); }
/** * If url for schema already exists, return the url. If not create a new temporary schema file and return a the url. */ private Path getOrGenerateSchemaFile(Schema schema) throws IOException { Preconditions.checkNotNull(schema, "Avro Schema should not be null"); String hashedSchema = Hashing.sha256().hashString(schema.toString(), StandardCharsets.UTF_8).toString(); if (!this.schemaPaths.containsKey(hashedSchema)) { Path schemaFilePath = new Path(this.schemaDir, String.valueOf(System.currentTimeMillis() + ".avsc")); AvroUtils.writeSchemaToFile(schema, schemaFilePath, fs, true); this.schemaPaths.put(hashedSchema, schemaFilePath); } return this.schemaPaths.get(hashedSchema); } }
/** * Add a {@link Schema} obtained from an Avro data file to the given {@link HiveRegistrationUnit}. * * <p> * If the length of the schema is less than {@link #SCHEMA_LITERAL_LENGTH_LIMIT}, it will be added via * {@link #SCHEMA_LITERAL}. Otherwise, the schema will be written to {@link #SCHEMA_FILE_NAME} and added * via {@link #SCHEMA_URL}. * </p> */ protected void addSchemaFromAvroFile(Schema schema, Path schemaFile, HiveRegistrationUnit hiveUnit) throws IOException { Preconditions.checkNotNull(schema); String schemaStr = schema.toString(); if (schemaStr.length() <= this.schemaLiteralLengthLimit) { hiveUnit.setSerDeProp(SCHEMA_LITERAL, schema.toString()); } else { AvroUtils.writeSchemaToFile(schema, schemaFile, this.fs, true); log.info("Using schema file " + schemaFile.toString()); hiveUnit.setSerDeProp(SCHEMA_URL, schemaFile.toString()); } }