public static IdealState cloneIdealState(IdealState idealState) { return new IdealState( (ZNRecord) ZN_RECORD_SERIALIZER.deserialize(ZN_RECORD_SERIALIZER.serialize(idealState.getRecord()))); }
ZNRecordSerializer znRecordSerializer = new ZNRecordSerializer(); IdealState idealStateCopy = new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); Map<String, Map<String, String>> oldMapFields = idealStateCopy.getRecord().getMapFields(); Map<String, LLCRealtimeSegmentZKMetadata> oldMetadataMap = new HashMap<>(segmentManager._metadataMap.size());
_realtimeSegmentRelocator.setTagToInstance(serverTenantCompleted, completedInstanceList); IdealState prevIdealState = new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); _realtimeSegmentRelocator.relocateSegments(realtimeTagConfig, idealState); Assert.assertEquals(idealState, prevIdealState); idealState.setInstanceStateMap("segment0", instanceStateMap0); prevIdealState = new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); _realtimeSegmentRelocator.relocateSegments(realtimeTagConfig, idealState); Assert.assertEquals(idealState, prevIdealState); new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); _realtimeSegmentRelocator.relocateSegments(realtimeTagConfig, idealState); Assert.assertEquals(idealState, prevIdealState); new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); _realtimeSegmentRelocator.relocateSegments(realtimeTagConfig, idealState); Assert.assertEquals(idealState, prevIdealState); new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); _realtimeSegmentRelocator.relocateSegments(realtimeTagConfig, idealState); Assert.assertNotSame(idealState, prevIdealState); new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); _realtimeSegmentRelocator.relocateSegments(realtimeTagConfig, idealState); Assert.assertNotSame(idealState, prevIdealState); _realtimeSegmentRelocator.setTagToInstance(serverTenantCompleted, completedInstanceList);
nPartitions = expectedPartitionAssignment.getNumPartitions(); IdealState idealStateCopy = new IdealState( (ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); Map<String, Map<String, String>> oldMapFields = idealStateCopy.getRecord().getMapFields(); nPartitions = expectedPartitionAssignment.getNumPartitions(); IdealState idealStateCopy = new IdealState( (ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); Map<String, Map<String, String>> oldMapFields = idealStateCopy.getRecord().getMapFields(); (ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); Map<String, Map<String, String>> oldMapFields = idealStateCopy.getRecord().getMapFields(); (ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); Map<String, Map<String, String>> oldMapFields = idealStateCopy.getRecord().getMapFields(); (ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); Map<String, Map<String, String>> oldMapFields = idealStateCopy.getRecord().getMapFields(); (ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); Map<String, Map<String, String>> oldMapFields = idealStateCopy.getRecord().getMapFields();
@Override public byte[] serialize(ZNRecord data) throws PropertyStoreException { return _serializer.serialize(data); }
@Override public byte[] serialize(ZNRecord data) throws PropertyStoreException { return _serializer.serialize(data); }
public void readZNode(String path) { ZNRecord record = _zkclient.readData(path, true); if (record == null) { System.out.println("null"); } else { System.out.println(new String(_serializer.serialize(record))); } }
public void readZNode(String path) { ZNRecord record = _zkclient.readData(path, true); if (record == null) { System.out.println("null"); } else { System.out.println(new String(_serializer.serialize(record))); } }
public static void main(String[] args) { ZNRecordSerializer serializer = new ZNRecordSerializer(); System.out.println(new String(serializer.serialize(generateConfigForMasterSlave()))); }
public static void main(String[] args) { ZNRecordSerializer serializer = new ZNRecordSerializer(); System.out.println(new String(serializer.serialize(generateConfigForMasterSlave()))); }
for (ZNRecord record : dumpRecords) { if (record != null) { LOG.info(new String(_jsonSerializer.serialize(record)));
/** * get configs * @param type config-scope-type, e.g. CLUSTER, RESOURCE, etc. * @param scopeArgsCsv csv-formatted scope-args, e.g myCluster,testDB * @param keysCsv csv-formatted keys. e.g. k1,k2 * @return json-formated key-value pairs, e.g. {k1=v1,k2=v2} */ public String getConfig(ConfigScopeProperty type, String scopeArgsCsv, String keysCsv) { // ConfigScope scope = new ConfigScopeBuilder().build(scopesStr); String[] scopeArgs = scopeArgsCsv.split("[\\s,]"); HelixConfigScope scope = new HelixConfigScopeBuilder(type, scopeArgs).build(); String[] keys = keysCsv.split("[\\s,]"); // parse keys // String[] keys = keysStr.split("[\\s,]"); // Set<String> keysSet = new HashSet<String>(Arrays.asList(keys)); Map<String, String> keyValueMap = _admin.getConfig(scope, Arrays.asList(keys)); ZNRecord record = new ZNRecord(type.toString()); // record.setMapField(scopesStr, propertiesMap); record.getSimpleFields().putAll(keyValueMap); ZNRecordSerializer serializer = new ZNRecordSerializer(); return new String(serializer.serialize(record)); }
/** * get configs * @param type config-scope-type, e.g. CLUSTER, RESOURCE, etc. * @param scopeArgsCsv csv-formatted scope-args, e.g myCluster,testDB * @param keysCsv csv-formatted keys. e.g. k1,k2 * @return json-formated key-value pairs, e.g. {k1=v1,k2=v2} */ public String getConfig(ConfigScopeProperty type, String scopeArgsCsv, String keysCsv) { // ConfigScope scope = new ConfigScopeBuilder().build(scopesStr); String[] scopeArgs = scopeArgsCsv.split("[\\s,]"); HelixConfigScope scope = new HelixConfigScopeBuilder(type, scopeArgs).build(); String[] keys = keysCsv.split("[\\s,]"); // parse keys // String[] keys = keysStr.split("[\\s,]"); // Set<String> keysSet = new HashSet<String>(Arrays.asList(keys)); Map<String, String> keyValueMap = _admin.getConfig(scope, Arrays.asList(keys)); ZNRecord record = new ZNRecord(type.toString()); // record.setMapField(scopesStr, propertiesMap); record.getSimpleFields().putAll(keyValueMap); ZNRecordSerializer serializer = new ZNRecordSerializer(); return new String(serializer.serialize(record)); }
@Test (enabled = false) public void testPerformance() { ZNRecord record = createZnRecord(); ZNRecordSerializer serializer1 = new ZNRecordSerializer(); ZNRecordStreamingSerializer serializer2 = new ZNRecordStreamingSerializer(); int loop = 100000; long start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { serializer1.serialize(record); } System.out.println("ZNRecordSerializer serialize took " + (System.currentTimeMillis() - start) + " ms"); byte[] data = serializer1.serialize(record); start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { serializer1.deserialize(data); } System.out.println("ZNRecordSerializer deserialize took " + (System.currentTimeMillis() - start) + " ms"); start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { data = serializer2.serialize(record); } System.out.println("ZNRecordStreamingSerializer serialize took " + (System.currentTimeMillis() - start) + " ms"); start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { ZNRecord result = (ZNRecord) serializer2.deserialize(data); } System.out.println("ZNRecordStreamingSerializer deserialize took " + (System.currentTimeMillis() - start) + " ms"); }
@Test public void testBasicCompression() { ZNRecord record = new ZNRecord("testId"); int numPartitions = 1024; int replicas = 3; int numNodes = 100; Random random = new Random(); for (int p = 0; p < numPartitions; p++) { Map<String, String> map = new HashMap<String, String>(); for (int r = 0; r < replicas; r++) { map.put("host_" + random.nextInt(numNodes), "ONLINE"); } record.setMapField("TestResource_" + p, map); } ZNRecordSerializer serializer = new ZNRecordSerializer(); byte[] serializedBytes; serializedBytes = serializer.serialize(record); int uncompressedSize = serializedBytes.length; System.out.println("raw serialized data length = " + serializedBytes.length); record.setSimpleField("enableCompression", "true"); serializedBytes = serializer.serialize(record); int compressedSize = serializedBytes.length; System.out.println("compressed serialized data length = " + serializedBytes.length); System.out.printf("compression ratio: %.2f \n", (uncompressedSize * 1.0 / compressedSize)); ZNRecord result = (ZNRecord) serializer.deserialize(serializedBytes); Assert.assertEquals(result, record); }
@Test (enabled = false) public void testParallelPerformance() throws ExecutionException, InterruptedException { final ZNRecord record = createZnRecord(); final ZNRecordSerializer serializer1 = new ZNRecordSerializer(); final ZNRecordStreamingSerializer serializer2 = new ZNRecordStreamingSerializer(); int loop = 100000; ExecutorService executorService = Executors.newFixedThreadPool(10000); long start = System.currentTimeMillis(); batchSerialize(serializer1, executorService, loop, record); System.out.println("ZNRecordSerializer serialize took " + (System.currentTimeMillis() - start) + " ms"); byte[] data = serializer1.serialize(record); start = System.currentTimeMillis(); batchSerialize(serializer2, executorService, loop, record); System.out.println("ZNRecordSerializer deserialize took " + (System.currentTimeMillis() - start) + " ms"); start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { data = serializer2.serialize(record); } System.out.println("ZNRecordStreamingSerializer serialize took " + (System.currentTimeMillis() - start) + " ms"); start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { ZNRecord result = (ZNRecord) serializer2.deserialize(data); } System.out.println("ZNRecordStreamingSerializer deserialize took " + (System.currentTimeMillis() - start) + " ms"); }
/** * Test that the payload is not included whenever it is not null. This is mainly to maintain * backward * compatibility. */ @Test public void testRawPayloadMissingIfUnspecified() { final String RECORD_ID = "testRawPayloadMissingIfUnspecified"; ZNRecord znRecord = new ZNRecord(RECORD_ID); ZNRecordSerializer znRecordSerializer = new ZNRecordSerializer(); byte[] serialized = znRecordSerializer.serialize(znRecord); ZNRecordStreamingSerializer znRecordStreamingSerializer = new ZNRecordStreamingSerializer(); byte[] streamingSerialized = znRecordStreamingSerializer.serialize(znRecord); ObjectMapper mapper = new ObjectMapper(); try { JsonNode jsonNode = mapper.readTree(new String(serialized)); Assert.assertFalse(jsonNode.has("rawPayload")); JsonNode streamingJsonNode = mapper.readTree(new String(streamingSerialized)); Assert.assertFalse(streamingJsonNode.has("rawPayload")); } catch (JsonProcessingException e) { Assert.fail(); } catch (IOException e) { Assert.fail(); } }
/** * Test the normal case of serialize/deserialize where ZNRecord is well-formed */ @Test public void basicTest() { ZNRecord record = new ZNRecord("testId"); record.setMapField("k1", ImmutableMap.of("a", "b", "c", "d")); record.setMapField("k2", ImmutableMap.of("e", "f", "g", "h")); record.setListField("k3", ImmutableList.of("a", "b", "c", "d")); record.setListField("k4", ImmutableList.of("d", "e", "f", "g")); record.setSimpleField("k5", "a"); record.setSimpleField("k5", "b"); ZNRecordSerializer serializer = new ZNRecordSerializer(); ZNRecord result = (ZNRecord) serializer.deserialize(serializer.serialize(record)); Assert.assertEquals(result, record); }
@Test public void testNullFields() { ZNRecord record = new ZNRecord("testId"); record.setMapField("K1", null); record.setListField("k2", null); record.setSimpleField("k3", null); ZNRecordSerializer serializer = new ZNRecordSerializer(); byte [] data = serializer.serialize(record); ZNRecord result = (ZNRecord) serializer.deserialize(data); Assert.assertEquals(result, record); Assert.assertNull(result.getMapField("K1")); Assert.assertNull(result.getListField("K2")); Assert.assertNull(result.getSimpleField("K3")); Assert.assertNull(result.getListField("K4")); }
/** * Test that the payload can be deserialized after serializing and deserializing the ZNRecord * that encloses it. This uses ZNRecordSerializer. */ @Test public void testFullZNRecordSerializeDeserialize() { final String RECORD_ID = "testFullZNRecordSerializeDeserialize"; SampleDeserialized sample = getSample(); ZNRecord znRecord = new ZNRecord(RECORD_ID); znRecord.setPayloadSerializer(new JacksonPayloadSerializer()); znRecord.setPayload(sample); ZNRecordSerializer znRecordSerializer = new ZNRecordSerializer(); byte[] serialized = znRecordSerializer.serialize(znRecord); ZNRecord deserialized = (ZNRecord) znRecordSerializer.deserialize(serialized); deserialized.setPayloadSerializer(new JacksonPayloadSerializer()); SampleDeserialized duplicate = deserialized.getPayload(SampleDeserialized.class); Assert.assertEquals(duplicate, sample); }