@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getSequenceNumber() == null) ? 0 : getSequenceNumber().hashCode()); hashCode = prime * hashCode + ((getApproximateArrivalTimestamp() == null) ? 0 : getApproximateArrivalTimestamp().hashCode()); hashCode = prime * hashCode + ((getData() == null) ? 0 : getData().hashCode()); hashCode = prime * hashCode + ((getPartitionKey() == null) ? 0 : getPartitionKey().hashCode()); hashCode = prime * hashCode + ((getEncryptionType() == null) ? 0 : getEncryptionType().hashCode()); return hashCode; }
JavaSparkContext sc; JavaRDD<String> data = sc.textFile("path/input.csv"); JavaSQLContext sqlContext = new JavaSQLContext(sc); JavaRDD<Record> rdd_records = sc.textFile(data).map( new Function<String, Record>() { public Record call(String line) throws Exception { // Here you can use JSON // Gson gson = new Gson(); // gson.fromJson(line, Record.class); String[] fields = line.split(","); Record sd = new Record(fields[0], fields[1], fields[2].trim(), fields[3]); return sd; } });
public Record unmarshall(JsonUnmarshallerContext context) throws Exception { Record record = new Record(); if (context.testExpression("SequenceNumber", targetDepth)) { context.nextToken(); record.setSequenceNumber(context.getUnmarshaller(String.class).unmarshall(context)); record.setApproximateArrivalTimestamp(context.getUnmarshaller(java.util.Date.class).unmarshall(context)); record.setData(context.getUnmarshaller(java.nio.ByteBuffer.class).unmarshall(context)); record.setPartitionKey(context.getUnmarshaller(String.class).unmarshall(context)); record.setEncryptionType(context.getUnmarshaller(String.class).unmarshall(context));
data = Collections.singletonList(toByteArray(kinesisRecord.getData())); streamPartition.getStream(), streamPartition.getPartitionId(), kinesisRecord.getSequenceNumber(), data );
public InputEvent(Record record) { this.sequenceNumber = record.getSequenceNumber(); this.partitionKey = record.getPartitionKey(); this.data = record.getData().array(); }
/** * Creates a copy of the record so we don't get interference from bolts that execute in the same JVM. * We invoke ByteBuffer.duplicate() so the ByteBuffer state is decoupled. * * @param record Kinesis record * @return Copied record. */ private Record copyRecord(Record record) { Record duplicate = new Record(); duplicate.setPartitionKey(record.getPartitionKey()); duplicate.setSequenceNumber(record.getSequenceNumber()); duplicate.setData(record.getData().duplicate()); return duplicate; }
private int calculateSize(List<Record> records) { int bytes = 0; for (Record r : records) { bytes += r.getData().remaining(); } // We get it in binary, but it's actually sent as Base64 return bytes * 3 / 2; }
boolean isAggregated = true; long subSeqNum = 0; ByteBuffer bb = r.getData(); List<String> pks = ar.getPartitionKeyTableList(); List<String> ehks = ar.getExplicitHashKeyTableList(); long aat = r.getApproximateArrivalTimestamp() == null ? -1 : r.getApproximateArrivalTimestamp().getTime(); try { int recordsInCurrRecord = 0; Record record = new Record() .withData(ByteBuffer.wrap(mr.getData().toByteArray())) .withPartitionKey(partitionKey) .withSequenceNumber(r.getSequenceNumber()) .withApproximateArrivalTimestamp(aat < 0 ? null : new Date(aat)); result.add(new UserRecord(true, record, subSeqNum++, explicitHashKey)); .append("dataLen=").append(mr.getData().toByteArray().length).append("]\n"); sb.append("Sequence number: ").append(r.getSequenceNumber()).append("\n") .append("Raw data: ") .append(javax.xml.bind.DatatypeConverter.printBase64Binary(messageData)).append("\n");
private void emitNewRecord (SpoutOutputCollector collector) { for (Map.Entry<String, LinkedList<Record>> entry: toEmitPerShard.entrySet()) { String shardId = entry.getKey(); LinkedList<Record> listOfRecords = entry.getValue(); Record record; while ((record = listOfRecords.pollFirst()) != null) { KinesisMessageId kinesisMessageId = new KinesisMessageId(kinesisConfig.getStreamName(), shardId, record.getSequenceNumber()); if (emitRecord(collector, record, kinesisMessageId)) { return; } } } }
public Record convertToRecord() { return new Record() .withApproximateArrivalTimestamp(arrivalTimestamp.toDate()) .withData(ByteBuffer.wrap(data.getBytes(StandardCharsets.UTF_8))) .withSequenceNumber(sequenceNumber) .withPartitionKey(""); }
private KinesisWrapper(final InternalEvent internal) { KinesisEventRecord eventRecord = ((KinesisInternalEvent) internal).getRecord(); Record record = eventRecord.getKinesis(); this.partitionKey = record.getPartitionKey(); this.sequenceNumber = record.getSequenceNumber(); this.eventSource = eventRecord.getEventSource(); this.sourceArn = eventRecord.getEventSourceARN(); this.functionName = internal.getCtx().getContext().getFunctionName(); this.functionVersion = internal.getCtx().getContext().getFunctionVersion(); this.processingTime = System.currentTimeMillis(); this.arrivalTime = record.getApproximateArrivalTimestamp().getTime(); this.timestamp = internal.getEventTime(); this.processingDelay = processingTime - timestamp; if (internal.getEventObj() != null) { this.payload = internal.getEventObj().getPayload(); } else { this.payload = null; } }
public void updateMetrics(String stream, Record record) { eventReadRates.get(stream).inc(); aggEventReadRate.inc(); long recordSize = record.getData().array().length + record.getPartitionKey().length(); eventByteReadRates.get(stream).inc(recordSize); aggEventByteReadRate.inc(recordSize); long latencyMs = Duration.between(Instant.now(), record.getApproximateArrivalTimestamp().toInstant()).toMillis(); readLatencies.get(stream).update(latencyMs); aggReadLatency.update(latencyMs); } }
Record recordToRetry() { Record recordToRetry = null; String sequenceNumber = retryQueue.peek(); if (sequenceNumber != null) { RecordNode node = seqNumToRecordInfoMap.get(sequenceNumber); if (node != null) { recordToRetry = node.getRecord(); if (LOG.isInfoEnabled()) { LOG.info("Retrying record with partition key " + recordToRetry.getPartitionKey() + " sequence number " + recordToRetry.getSequenceNumber() + ". Retry attempt " + (node.getRetryCount() + 1)); } } } return recordToRetry; }
@Override protected void processRecord(Record record) { String partitionKey = record.getPartitionKey(); ByteBuffer data = record.getData(); logger.info("partitionKey={} ", partitionKey); byte[] dataBytes = new byte[data.remaining()]; data.get(dataBytes, 0, dataBytes.length); long key = Long.valueOf(partitionKey); TestPOJO expected = new TestPOJO(key); TestPOJO read = (TestPOJO)fieldValueGenerator.deserializeObject(dataBytes); if (!read.outputFieldsEquals(expected)) { logger.error("read is not same as expected. read={}, expected={}", read, expected); Assert.assertTrue(false); } else { logger.info("read is same as expected. read={}, expected={}", read, expected); } } }
String line = scan.nextLine(); Record rec = new Record(); rec.withPartitionKey("1").withSequenceNumber(r + "") .withData(ByteBuffer.wrap(line.getBytes())) .withApproximateArrivalTimestamp(approximateArrivalTimestamp);
@Override public List<Object> deserialize(Record record) { final List<Object> l = new ArrayList<>(); l.add(record.getPartitionKey()); l.add(record); return l; }
JSONObject jObj = new JSONObject(results); JSONArray albums = jObj.getJSONArray("pickups"); for (int i = 0; i < albums.length(); i++) { JSONObject c = albums.getJSONObject(i); String id = c.getString("id"); Record objRecord=new Record(); objRecord.setId(c.getString("id")); objRecord.setName(c.getString("name")); objRecord.setNumber(c.getString("number")); objRecord.setTime(c.getString("time")); PickupList.arrayList.add(objRecord); }