/** * Extract user defined keys by looking at "gobblin.converter.http.keys" * If keys are defined, extract key-value pair from inputRecord and set it to HttpOperation * If keys are not defined, generate HttpOperation by HttpUtils.toHttpOperation */ @Override protected HttpOperation generateHttpOperation (GenericRecord inputRecord, State state) { Map<String, String> keyAndValue = new HashMap<>(); Optional<Iterable<String>> keys = getKeys(state); HttpOperation operation; if (keys.isPresent()) { for (String key : keys.get()) { String value = inputRecord.get(key).toString(); log.debug("Http join converter: key is {}, value is {}", key, value); keyAndValue.put(key, value); } operation = new HttpOperation(); operation.setKeys(keyAndValue); } else { operation = HttpUtils.toHttpOperation(inputRecord); } return operation; }
URI uri = HttpUtils.buildURI(urlTemplate, httpOperation.getKeys(), httpOperation.getQueryParams()); if (uri == null) { return null; Map<String, String> headers = httpOperation.getHeaders(); if (headers != null && headers.size() != 0) { for (Map.Entry<String, String> header : headers.entrySet()) { int bytesWritten = addPayload(builder, httpOperation.getBody()); if (bytesWritten == -1) { throw new RuntimeException("Fail to write payload into request");
/** * Convert the given {@link GenericRecord} to {@link HttpOperation} */ public static HttpOperation toHttpOperation(GenericRecord record) { if (record instanceof HttpOperation) { return (HttpOperation) record; } HttpOperation.Builder builder = HttpOperation.newBuilder(); Map<String, String> stringMap = AvroUtils.toStringMap(record.get(HttpConstants.KEYS)); if (stringMap != null) { builder.setKeys(stringMap); } stringMap = AvroUtils.toStringMap(record.get(HttpConstants.QUERY_PARAMS)); if (stringMap != null) { builder.setQueryParams(stringMap); } stringMap = AvroUtils.toStringMap(record.get(HttpConstants.HEADERS)); if (stringMap != null) { builder.setHeaders(stringMap); } Object body = record.get(HttpConstants.BODY); if (body != null) { builder.setBody(body.toString()); } return builder.build(); }
public static Queue<BufferedRecord<GenericRecord>> createQueue(int size, boolean isHttpOperation) { Queue<BufferedRecord<GenericRecord>> queue = new ArrayDeque<>(size); for (int i = 0; i < size; i++) { Map<String, String> keys = new HashMap<>(); keys.put("part1", i + "1"); keys.put("part2", i + "2"); Map<String, String> queryParams = new HashMap<>(); queryParams.put("param1", i + "1"); GenericRecord record = isHttpOperation ? new HttpOperation() : new MockGenericRecord(); record.put("keys", keys); record.put("queryParams", queryParams); record.put("body", "{\"id\":\"id" + i + "\"}"); BufferedRecord<GenericRecord> item = new BufferedRecord<>(record, null); queue.add(item); } return queue; }
@Override public HttpOperation build() { try { HttpOperation record = new HttpOperation(); record.keys = fieldSetFlags()[0] ? this.keys : (java.util.Map<java.lang.String,java.lang.String>) defaultValue(fields()[0]); record.queryParams = fieldSetFlags()[1] ? this.queryParams : (java.util.Map<java.lang.String,java.lang.String>) defaultValue(fields()[1]); record.headers = fieldSetFlags()[2] ? this.headers : (java.util.Map<java.lang.String,java.lang.String>) defaultValue(fields()[2]); record.body = fieldSetFlags()[3] ? this.body : (java.lang.String) defaultValue(fields()[3]); return record; } catch (Exception e) { throw new org.apache.avro.AvroRuntimeException(e); } } }
HttpOperation httpOperation = HttpUtils.toHttpOperation(record.getRecord()); URI uri = HttpUtils.buildURI(urlTemplate, httpOperation.getKeys(), httpOperation.getQueryParams()); if (uri == null) { return null; Map<String, String> headers = httpOperation.getHeaders(); if (headers != null && headers.size() != 0) { builder.setHeaders(headers); int bytesWritten = addPayload(builder, httpOperation.getBody()); if (bytesWritten == -1) { throw new RuntimeException("Fail to write payload into request");
/** * Extract user defined keys by looking at "gobblin.converter.http.keys" * If keys are defined, extract key-value pair from inputRecord and set it to HttpOperation * If keys are not defined, generate HttpOperation by HttpUtils.toHttpOperation */ @Override protected HttpOperation generateHttpOperation (GenericRecord inputRecord, State state) { Map<String, String> keyAndValue = new HashMap<>(); Optional<Iterable<String>> keys = getKeys(state); HttpOperation operation; if (keys.isPresent()) { for (String key : keys.get()) { String value = inputRecord.get(key).toString(); log.debug("Http join converter: key is {}, value is {}", key, value); keyAndValue.put(key, value); } operation = new HttpOperation(); operation.setKeys(keyAndValue); } else { operation = HttpUtils.toHttpOperation(inputRecord); } return operation; }
@Override public HttpOperation build() { try { HttpOperation record = new HttpOperation(); record.keys = fieldSetFlags()[0] ? this.keys : (java.util.Map<java.lang.String,java.lang.String>) defaultValue(fields()[0]); record.queryParams = fieldSetFlags()[1] ? this.queryParams : (java.util.Map<java.lang.String,java.lang.String>) defaultValue(fields()[1]); record.headers = fieldSetFlags()[2] ? this.headers : (java.util.Map<java.lang.String,java.lang.String>) defaultValue(fields()[2]); record.body = fieldSetFlags()[3] ? this.body : (java.lang.String) defaultValue(fields()[3]); return record; } catch (Exception e) { throw new org.apache.avro.AvroRuntimeException(e); } } }
/** * Convert the given {@link GenericRecord} to {@link HttpOperation} */ public static HttpOperation toHttpOperation(GenericRecord record) { if (record instanceof HttpOperation) { return (HttpOperation) record; } HttpOperation.Builder builder = HttpOperation.newBuilder(); Map<String, String> stringMap = AvroUtils.toStringMap(record.get(HttpConstants.KEYS)); if (stringMap != null) { builder.setKeys(stringMap); } stringMap = AvroUtils.toStringMap(record.get(HttpConstants.QUERY_PARAMS)); if (stringMap != null) { builder.setQueryParams(stringMap); } stringMap = AvroUtils.toStringMap(record.get(HttpConstants.HEADERS)); if (stringMap != null) { builder.setHeaders(stringMap); } Object body = record.get(HttpConstants.BODY); if (body != null) { builder.setBody(body.toString()); } return builder.build(); }
URI uri = HttpUtils.buildURI(urlTemplate, httpOperation.getKeys(), httpOperation.getQueryParams()); if (uri == null) { return null; Map<String, String> headers = httpOperation.getHeaders(); if (headers != null && headers.size() != 0) { for (Map.Entry<String, String> header : headers.entrySet()) { int bytesWritten = addPayload(builder, httpOperation.getBody()); if (bytesWritten == -1) { throw new RuntimeException("Fail to write payload into request");
HttpOperation httpOperation = HttpUtils.toHttpOperation(record.getRecord()); URI uri = HttpUtils.buildURI(urlTemplate, httpOperation.getKeys(), httpOperation.getQueryParams()); if (uri == null) { return null; Map<String, String> headers = httpOperation.getHeaders(); if (headers != null && headers.size() != 0) { builder.setHeaders(headers); int bytesWritten = addPayload(builder, httpOperation.getBody()); if (bytesWritten == -1) { throw new RuntimeException("Fail to write payload into request");