@Override public ZNRecord toZNRecord() { ZNRecord znRecord = super.toZNRecord(); znRecord.setSimpleField(CommonConstants.Segment.Realtime.STATUS, _status.toString()); znRecord.setLongField(CommonConstants.Segment.FLUSH_THRESHOLD_SIZE, _sizeThresholdToFlushSegment); znRecord.setSimpleField(CommonConstants.Segment.FLUSH_THRESHOLD_TIME, _timeThresholdToFlushSegment); return znRecord; }
/** * Wrap {@link Schema} into a {@link ZNRecord}. */ public static ZNRecord toZNRecord(@Nonnull Schema schema) { ZNRecord record = new ZNRecord(schema.getSchemaName()); record.setSimpleField("schemaJSON", schema.getJSONSchema()); return record; }
@Override public ZNRecord toZNRecord() { ZNRecord znRecord = super.toZNRecord(); znRecord.setSimpleField(CommonConstants.Segment.Offline.DOWNLOAD_URL, _downloadUrl); znRecord.setLongField(CommonConstants.Segment.Offline.PUSH_TIME, _pushTime); znRecord.setLongField(CommonConstants.Segment.Offline.REFRESH_TIME, _refreshTime); return znRecord; }
/** * * @param tableName is the offline table name * @param segmentName is name of the segment * @param crc is the CRC of the new segment */ public SegmentRefreshMessage(String tableName, String segmentName, long crc) { super(MessageType.USER_DEFINE_MSG, UUID.randomUUID().toString()); setResourceName(tableName); setPartitionName(segmentName); setMsgSubType(REFRESH_SEGMENT_MSG_SUB_TYPE); // Give it infinite time to process the message, as long as session is alive setExecutionTimeout(-1); getRecord().setSimpleField(SIMPLE_FIELD_CRC, Long.toString(crc)); }
@Override public ZNRecord toZNRecord() { ZNRecord znRecord = new ZNRecord(_segmentName); znRecord.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, _segmentName); znRecord.setSimpleField(CommonConstants.Segment.TABLE_NAME, _tableName); znRecord.setSimpleField(CommonConstants.Segment.CRYPTER_NAME, _crypterName); znRecord.setSimpleField(CommonConstants.Segment.TIME_UNIT, NULL); } else { znRecord.setEnumField(CommonConstants.Segment.TIME_UNIT, _timeUnit); znRecord.setLongField(CommonConstants.Segment.END_TIME, _endTime); znRecord.setSimpleField(CommonConstants.Segment.INDEX_VERSION, _indexVersion); znRecord.setLongField(CommonConstants.Segment.TOTAL_DOCS, _totalRawDocs); znRecord.setLongField(CommonConstants.Segment.CRC, _crc); try { String partitionMetadataJson = _partitionMetadata.toJsonString(); znRecord.setSimpleField(CommonConstants.Segment.PARTITION_METADATA, partitionMetadataJson); } catch (IOException e) { LOGGER
/** * Add or update a {@link ZNRecord} to include a simple field with the given key and value. If the supplied * {@link ZNRecord} is null, a new {@link ZNRecord} will be created. * @param oldRecord A {@link ZNRecord} to add or update its simple field. {@code null} indicates to create a * new {@link ZNRecord}. * @param key The key for the simple field. * @param value The value for the simple field. * @return A {@link ZNRecord} including a simple field with the given key and value. */ private ZNRecord makeZNRecordWithSimpleField(ZNRecord oldRecord, String key, String value) { ZNRecord zNRecord = oldRecord == null ? new ZNRecord(String.valueOf(System.currentTimeMillis())) : oldRecord; if (key != null && value != null) { zNRecord.setSimpleField(key, value); } return zNRecord; }
@Override public ZNRecord toZNRecord() { ZNRecord znRecord = super.toZNRecord(); znRecord.setLongField(START_OFFSET, _startOffset); znRecord.setLongField(END_OFFSET, _endOffset); znRecord.setIntField(NUM_REPLICAS, _numReplicas); znRecord.setSimpleField(DOWNLOAD_URL, _downloadUrl); return znRecord; }
record.setSimpleField(StateModelDefinitionProperty.INITIAL_STATE.toString(), OFFLINE_STATE);
private ZNRecord getTestOfflineSegmentZNRecord() { String segmentName = "testTable_O_3000_4000"; ZNRecord record = new ZNRecord(segmentName); record.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, segmentName); record.setSimpleField(CommonConstants.Segment.TABLE_NAME, "testTable"); record.setSimpleField(CommonConstants.Segment.CRYPTER_NAME, "testCrypter"); record.setSimpleField(CommonConstants.Segment.INDEX_VERSION, "v1"); record.setEnumField(CommonConstants.Segment.SEGMENT_TYPE, CommonConstants.Segment.SegmentType.OFFLINE); record.setLongField(CommonConstants.Segment.START_TIME, 1000); record.setLongField(CommonConstants.Segment.END_TIME, 2000); record.setSimpleField(CommonConstants.Segment.TIME_UNIT, TimeUnit.HOURS.toString()); record.setLongField(CommonConstants.Segment.TOTAL_DOCS, 50000); record.setLongField(CommonConstants.Segment.CRC, 54321); record.setLongField(CommonConstants.Segment.CREATION_TIME, 1000); record.setSimpleField(CommonConstants.Segment.Offline.DOWNLOAD_URL, "http://localhost:8000/testTable_O_3000_4000"); record.setLongField(CommonConstants.Segment.Offline.PUSH_TIME, 4000); record.setLongField(CommonConstants.Segment.Offline.REFRESH_TIME, 8000); return record; }
public IdealStateBuilderUtil disableIdealState() { _idealState.getRecord().setSimpleField(IdealState.IdealStateProperty.HELIX_ENABLED.name(), "false"); return this; }
private ZNRecord getTestInProgressRealtimeSegmentZNRecord() { String segmentName = "testTable_R_1000_groupId0_part0"; ZNRecord record = new ZNRecord(segmentName); record.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, segmentName); record.setSimpleField(CommonConstants.Segment.TABLE_NAME, "testTable"); record.setSimpleField(CommonConstants.Segment.INDEX_VERSION, "v1"); record.setEnumField(CommonConstants.Segment.SEGMENT_TYPE, CommonConstants.Segment.SegmentType.REALTIME); record.setEnumField(CommonConstants.Segment.Realtime.STATUS, CommonConstants.Segment.Realtime.Status.IN_PROGRESS); record.setLongField(CommonConstants.Segment.START_TIME, 1000); record.setLongField(CommonConstants.Segment.END_TIME, -1); record.setSimpleField(CommonConstants.Segment.TIME_UNIT, TimeUnit.HOURS.toString()); record.setLongField(CommonConstants.Segment.TOTAL_DOCS, -1); record.setLongField(CommonConstants.Segment.CRC, -1); record.setLongField(CommonConstants.Segment.CREATION_TIME, 1000); record.setIntField(CommonConstants.Segment.FLUSH_THRESHOLD_SIZE, 1234); record.setSimpleField(CommonConstants.Segment.FLUSH_THRESHOLD_TIME, "6h"); return record; }
private ZNRecord getTestDoneRealtimeSegmentZNRecord() { String segmentName = "testTable_R_1000_2000_groupId0_part0"; ZNRecord record = new ZNRecord(segmentName); record.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, segmentName); record.setSimpleField(CommonConstants.Segment.TABLE_NAME, "testTable"); record.setSimpleField(CommonConstants.Segment.INDEX_VERSION, "v1"); record.setEnumField(CommonConstants.Segment.SEGMENT_TYPE, CommonConstants.Segment.SegmentType.REALTIME); record.setEnumField(CommonConstants.Segment.Realtime.STATUS, CommonConstants.Segment.Realtime.Status.DONE); record.setLongField(CommonConstants.Segment.START_TIME, 1000); record.setLongField(CommonConstants.Segment.END_TIME, 2000); record.setSimpleField(CommonConstants.Segment.TIME_UNIT, TimeUnit.HOURS.toString()); record.setLongField(CommonConstants.Segment.TOTAL_DOCS, 10000); record.setLongField(CommonConstants.Segment.CRC, 1234); record.setLongField(CommonConstants.Segment.CREATION_TIME, 3000); record.setIntField(CommonConstants.Segment.FLUSH_THRESHOLD_SIZE, 1234); record.setSimpleField(CommonConstants.Segment.FLUSH_THRESHOLD_TIME, "6h"); return record; }
private void modifyExistingInstanceConfig(ZkClient zkClient) throws InterruptedException { String instanceName = "Server_localhost_" + new Random().nextInt(NUM_INSTANCES); String instanceConfigPath = PropertyPathBuilder.instanceConfig(_helixClusterName, instanceName); Assert.assertTrue(zkClient.exists(instanceConfigPath)); ZNRecord znRecord = zkClient.readData(instanceConfigPath, null); InstanceConfig cachedInstanceConfig = _helixResourceManager.getHelixInstanceConfig(instanceName); String originalPort = cachedInstanceConfig.getPort(); Assert.assertNotNull(originalPort); String newPort = Long.toString(System.currentTimeMillis()); Assert.assertTrue(!newPort.equals(originalPort)); // Set new port to this instance config. znRecord.setSimpleField(InstanceConfig.InstanceConfigProperty.HELIX_PORT.toString(), newPort); zkClient.writeData(instanceConfigPath, znRecord); long maxTime = System.currentTimeMillis() + MAX_TIMEOUT_IN_MILLISECOND; InstanceConfig latestCachedInstanceConfig = _helixResourceManager.getHelixInstanceConfig(instanceName); String latestPort = latestCachedInstanceConfig.getPort(); while (!newPort.equals(latestPort) && System.currentTimeMillis() < maxTime) { Thread.sleep(100L); latestCachedInstanceConfig = _helixResourceManager.getHelixInstanceConfig(instanceName); latestPort = latestCachedInstanceConfig.getPort(); } Assert.assertTrue(System.currentTimeMillis() < maxTime, "Timeout when waiting for adding instance config"); // Set original port back to this instance config. znRecord.setSimpleField(InstanceConfig.InstanceConfigProperty.HELIX_PORT.toString(), originalPort); zkClient.writeData(instanceConfigPath, znRecord); }
znRecord.setSimpleField(CommonConstants.Segment.PARTITION_METADATA, expectedPartitionMetadata.toJsonString()); SegmentZKMetadata expectedSegmentMetadata = new OfflineSegmentZKMetadata(znRecord); SegmentPartitionMetadata actualPartitionMetadata = expectedSegmentMetadata.getPartitionMetadata(); znRecord.setSimpleField(CommonConstants.Segment.PARTITION_METADATA, expectedPartitionMetadata.toJsonString()); expectedSegmentMetadata = new RealtimeSegmentZKMetadata(znRecord);
InstanceConfig instanceConfig = new InstanceConfig("Server_localhost_1234"); instanceConfigs.add(instanceConfig); instanceConfig.getRecord().setSimpleField(CommonConstants.Helix.IS_SHUTDOWN_IN_PROGRESS, "false");
@Test public void testRoutingTableExcludesDisabledAndRebootingInstances() { final String tableName = "fakeTable_OFFLINE"; final int segmentCount = 100; final int replicationFactor = 6; final int instanceCount = 50; ExternalView externalView = createExternalView(tableName, segmentCount, replicationFactor, instanceCount); List<InstanceConfig> instanceConfigs = createInstanceConfigs(instanceCount); final InstanceConfig disabledHelixInstance = instanceConfigs.get(0); final String disabledHelixInstanceName = disabledHelixInstance.getInstanceName(); disabledHelixInstance.setInstanceEnabled(false); final InstanceConfig shuttingDownInstance = instanceConfigs.get(1); final String shuttingDownInstanceName = shuttingDownInstance.getInstanceName(); shuttingDownInstance.getRecord() .setSimpleField(CommonConstants.Helix.IS_SHUTDOWN_IN_PROGRESS, Boolean.toString(true)); validateAssertionForOneRoutingTable(new RoutingTableValidator() { @Override public boolean isRoutingTableValid(Map<String, List<String>> routingTable, ExternalView externalView, List<InstanceConfig> instanceConfigs) { for (String serverName : routingTable.keySet()) { // These servers should not appear in the routing table if (serverName.equals(disabledHelixInstanceName) || serverName.equals(shuttingDownInstanceName)) { return false; } } return true; } }, "Routing table should not contain disabled instances", externalView, instanceConfigs, tableName); }
InstanceConfig instanceConfig = new InstanceConfig(instanceName); instanceConfigs.add(instanceConfig); instanceConfig.getRecord().setSimpleField(CommonConstants.Helix.IS_SHUTDOWN_IN_PROGRESS, "false");
InstanceConfig instanceConfig = new InstanceConfig("Server_localhost_1234"); instanceConfigs.add(instanceConfig); instanceConfig.getRecord().setSimpleField(CommonConstants.Helix.IS_SHUTDOWN_IN_PROGRESS, "true");
znrecord.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, "myTable_0"); znrecord.setSimpleField(CommonConstants.Segment.TABLE_NAME, "myTable_OFFLINE"); znrecord.setSimpleField(CommonConstants.Segment.INDEX_VERSION, "v1"); znrecord.setEnumField(CommonConstants.Segment.SEGMENT_TYPE, CommonConstants.Segment.SegmentType.OFFLINE); znrecord.setLongField(CommonConstants.Segment.START_TIME, 1000); znrecord.setLongField(CommonConstants.Segment.END_TIME, 2000); znrecord.setSimpleField(CommonConstants.Segment.TIME_UNIT, TimeUnit.HOURS.toString()); znrecord.setLongField(CommonConstants.Segment.TOTAL_DOCS, 10000); znrecord.setLongField(CommonConstants.Segment.CRC, 1234); znrecord.setLongField(CommonConstants.Segment.CREATION_TIME, 3000); znrecord.setSimpleField(CommonConstants.Segment.Offline.DOWNLOAD_URL, "http://localhost:8000/myTable_0"); znrecord.setLongField(CommonConstants.Segment.Offline.PUSH_TIME, System.currentTimeMillis()); znrecord.setLongField(CommonConstants.Segment.Offline.REFRESH_TIME, System.currentTimeMillis());
znrecord.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, "myTable_0"); znrecord.setSimpleField(CommonConstants.Segment.TABLE_NAME, "myTable_OFFLINE"); znrecord.setSimpleField(CommonConstants.Segment.INDEX_VERSION, "v1"); znrecord.setEnumField(CommonConstants.Segment.SEGMENT_TYPE, CommonConstants.Segment.SegmentType.OFFLINE); znrecord.setLongField(CommonConstants.Segment.START_TIME, 1000); znrecord.setLongField(CommonConstants.Segment.END_TIME, 2000); znrecord.setSimpleField(CommonConstants.Segment.TIME_UNIT, TimeUnit.HOURS.toString()); znrecord.setLongField(CommonConstants.Segment.TOTAL_DOCS, 10000); znrecord.setLongField(CommonConstants.Segment.CRC, 1234); znrecord.setLongField(CommonConstants.Segment.CREATION_TIME, 3000); znrecord.setSimpleField(CommonConstants.Segment.Offline.DOWNLOAD_URL, "http://localhost:8000/myTable_0"); znrecord.setLongField(CommonConstants.Segment.Offline.PUSH_TIME, System.currentTimeMillis()); znrecord.setLongField(CommonConstants.Segment.Offline.REFRESH_TIME, System.currentTimeMillis());