ZNRecord record = new ZNRecord(PINOT_SEGMENT_ONLINE_OFFLINE_STATE_MODEL); record.setSimpleField(StateModelDefinitionProperty.INITIAL_STATE.toString(), OFFLINE_STATE); statePriorityList.add(OFFLINE_STATE); statePriorityList.add(DROPPED_STATE); record.setListField(StateModelDefinitionProperty.STATE_PRIORITY_LIST.toString(), statePriorityList); if (state.equals(ONLINE_STATE)) { metadata.put("count", "R"); record.setMapField(key, metadata); record.setMapField(key, metadata); record.setMapField(key, metadata); metadata.put(OFFLINE_STATE, OFFLINE_STATE); metadata.put(DROPPED_STATE, DROPPED_STATE); record.setMapField(key, metadata); metadata.put(ONLINE_STATE, ONLINE_STATE); metadata.put(DROPPED_STATE, DROPPED_STATE); record.setMapField(key, metadata); stateTransitionPriorityList.add("OFFLINE-DROPPED"); record.setListField(StateModelDefinitionProperty.STATE_TRANSITION_PRIORITYLIST.toString(), stateTransitionPriorityList);
public SegmentZKMetadata(ZNRecord znRecord) { _segmentName = znRecord.getSimpleField(CommonConstants.Segment.SEGMENT_NAME); _tableName = znRecord.getSimpleField(CommonConstants.Segment.TABLE_NAME); _crypterName = znRecord.getSimpleField(CommonConstants.Segment.CRYPTER_NAME); _segmentType = znRecord.getEnumField(CommonConstants.Segment.SEGMENT_TYPE, SegmentType.class, SegmentType.OFFLINE); _startTime = znRecord.getLongField(CommonConstants.Segment.START_TIME, -1); _endTime = znRecord.getLongField(CommonConstants.Segment.END_TIME, -1); if (znRecord.getSimpleFields().containsKey(CommonConstants.Segment.TIME_UNIT) && !znRecord .getSimpleField(CommonConstants.Segment.TIME_UNIT).equals(NULL)) { setTimeUnit(znRecord.getEnumField(CommonConstants.Segment.TIME_UNIT, TimeUnit.class, TimeUnit.DAYS)); } _indexVersion = znRecord.getSimpleField(CommonConstants.Segment.INDEX_VERSION); _totalRawDocs = znRecord.getLongField(CommonConstants.Segment.TOTAL_DOCS, -1); _crc = znRecord.getLongField(CommonConstants.Segment.CRC, -1); _creationTime = znRecord.getLongField(CommonConstants.Segment.CREATION_TIME, -1); try { String partitionMetadataJson = znRecord.getSimpleField(CommonConstants.Segment.PARTITION_METADATA); if (partitionMetadataJson != null) { _partitionMetadata = SegmentPartitionMetadata.fromJsonString(partitionMetadataJson); } } catch (IOException e) { LOGGER.error( "Exception caught while reading partition info from zk metadata for segment '{}', partition info dropped.", _segmentName, e); } _segmentUploadStartTime = znRecord.getLongField(CommonConstants.Segment.SEGMENT_UPLOAD_START_TIME, -1); _customMap = znRecord.getMapField(CommonConstants.Segment.CUSTOM_MAP); }
public static SegmentMergeLineage fromZNRecord(ZNRecord record) { String tableNameWithType = record.getId(); Map<String, List<String>> segmentGroupLineageMap = record.getListFields(); Map<Integer, Map<String, List<String>>> groupToSegmentsMap = new HashMap<>(); for (Map.Entry<String, Map<String, String>> entry : record.getMapFields().entrySet()) { String levelKey = entry.getKey(); Integer level = Integer.parseInt(levelKey.substring(LEVEL_KEY_PREFIX.length())); Map<String, List<String>> groupToSegmentsForLevel = new HashMap<>(); for (Map.Entry<String, String> groupEntry : entry.getValue().entrySet()) { String groupId = groupEntry.getKey(); String segmentsString = groupEntry.getValue(); List<String> segments = Arrays.asList(segmentsString.split(SEGMENT_DELIMITER)); groupToSegmentsForLevel.put(groupId, new ArrayList<>(segments)); } groupToSegmentsMap.put(level, groupToSegmentsForLevel); } return new SegmentMergeLineage(tableNameWithType, segmentGroupLineageMap, groupToSegmentsMap); }
@Override public ZNRecord toZNRecord() { ZNRecord znRecord = super.toZNRecord(); znRecord.setLongField(START_OFFSET, _startOffset); znRecord.setLongField(END_OFFSET, _endOffset); znRecord.setIntField(NUM_REPLICAS, _numReplicas); znRecord.setSimpleField(DOWNLOAD_URL, _downloadUrl); return znRecord; }
@Override public ZNRecord toZNRecord() { ZNRecord znRecord = new ZNRecord(getId()); znRecord.setMapField(KAFKA_HIGH_LEVEL_CONSUMER_GROUP_MAP, _groupIdMap); znRecord.setMapField(KAFKA_HIGH_LEVEL_CONSUMER_PARTITION_MAP, _partitionMap); return znRecord; }
public OfflineSegmentZKMetadata(ZNRecord znRecord) { super(znRecord); setSegmentType(SegmentType.OFFLINE); _downloadUrl = znRecord.getSimpleField(CommonConstants.Segment.Offline.DOWNLOAD_URL); _pushTime = znRecord.getLongField(CommonConstants.Segment.Offline.PUSH_TIME, Long.MIN_VALUE); _refreshTime = znRecord.getLongField(CommonConstants.Segment.Offline.REFRESH_TIME, Long.MIN_VALUE); }
@Override public ZNRecord toZNRecord() { ZNRecord znRecord = new ZNRecord(_segmentName); znRecord.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, _segmentName); znRecord.setSimpleField(CommonConstants.Segment.TABLE_NAME, _tableName); znRecord.setSimpleField(CommonConstants.Segment.CRYPTER_NAME, _crypterName); znRecord.setEnumField(CommonConstants.Segment.SEGMENT_TYPE, _segmentType); if (_timeUnit == null) { znRecord.setSimpleField(CommonConstants.Segment.TIME_UNIT, NULL); } else { znRecord.setEnumField(CommonConstants.Segment.TIME_UNIT, _timeUnit); znRecord.setLongField(CommonConstants.Segment.START_TIME, _startTime); znRecord.setLongField(CommonConstants.Segment.END_TIME, _endTime); znRecord.setSimpleField(CommonConstants.Segment.INDEX_VERSION, _indexVersion); znRecord.setLongField(CommonConstants.Segment.TOTAL_DOCS, _totalRawDocs); znRecord.setLongField(CommonConstants.Segment.CRC, _crc); znRecord.setLongField(CommonConstants.Segment.CREATION_TIME, _creationTime); znRecord.setSimpleField(CommonConstants.Segment.PARTITION_METADATA, partitionMetadataJson); } catch (IOException e) { LOGGER znRecord.setLongField(CommonConstants.Segment.SEGMENT_UPLOAD_START_TIME, _segmentUploadStartTime); znRecord.setMapField(CommonConstants.Segment.CUSTOM_MAP, _customMap);
private static void setZNRecord(ZNRecord diff, ZNRecord record, String keySuffix) { if (diff == null || record == null) { return; } for (String key : record.getSimpleFields().keySet()) { diff.setSimpleField(key + "/" + keySuffix, record.getSimpleField(key)); } for (String key : record.getListFields().keySet()) { diff.setListField(key + "/" + keySuffix, record.getListField(key)); } for (String key : record.getMapFields().keySet()) { diff.setMapField(key + "/" + keySuffix, record.getMapField(key)); } }
/** * Wrap {@link Schema} into a {@link ZNRecord}. */ public static ZNRecord toZNRecord(@Nonnull Schema schema) { ZNRecord record = new ZNRecord(schema.getSchemaName()); record.setSimpleField("schemaJSON", schema.getJSONSchema()); return record; }
@Nullable @Override public IdealState apply(@Nullable IdealState input) { Map<String, Map<String, String>> existingMapField = input.getRecord().getMapFields(); for (Map.Entry<String, Map<String, String>> segmentEntry : proposedIdealState.entrySet()) { existingMapField.put(segmentEntry.getKey(), segmentEntry.getValue()); } return input; } }, RetryPolicies.exponentialBackoffRetryPolicy(5, 500L, 2.0f));
"ZNRecord does not exist on path={} in HelixPropertyStore when updating accounts. Creating a new ZNRecord.", FULL_ACCOUNT_METADATA_PATH); recordToUpdate = new ZNRecord(ZN_RECORD_ID); } else { recordToUpdate = recordFromZk; Map<String, String> accountMap = recordToUpdate.getMapField(ACCOUNT_METADATA_MAP_KEY); if (accountMap == null) { logger.debug("AccountMap does not exist in ZNRecord when updating accounts. Creating a new accountMap"); recordToUpdate.setMapField(ACCOUNT_METADATA_MAP_KEY, accountMap); potentialNewState = accountMap; return recordToUpdate;
public static List<ZNRecordRow> convertSimpleFields(ZNRecord record) { List<ZNRecordRow> result = new ArrayList<ZNRecordRow>(); for (String key : record.getSimpleFields().keySet()) { ZNRecordRow row = new ZNRecordRow(); row.putField(ZNRECORD_ID, record.getId()); row.putField(SIMPLE_KEY, key); row.putField(SIMPLE_VALUE, record.getSimpleField(key)); result.add(row); } return result; }
/** * Initialize the property with an existing ZNRecord with new record id * @param record * @param id */ public HelixProperty(ZNRecord record, String id) { _record = new ZNRecord(record, id); _stat = new Stat(_record.getVersion(), _record.getCreationTime(), _record.getModifiedTime(), _record.getEphemeralOwner()); }
/** * * @param tableName is the offline table name * @param segmentName is name of the segment * @param crc is the CRC of the new segment */ public SegmentRefreshMessage(String tableName, String segmentName, long crc) { super(MessageType.USER_DEFINE_MSG, UUID.randomUUID().toString()); setResourceName(tableName); setPartitionName(segmentName); setMsgSubType(REFRESH_SEGMENT_MSG_SUB_TYPE); // Give it infinite time to process the message, as long as session is alive setExecutionTimeout(-1); getRecord().setSimpleField(SIMPLE_FIELD_CRC, Long.toString(crc)); }
public LLCRealtimeSegmentZKMetadata(ZNRecord znRecord) { super(znRecord); _startOffset = Long.valueOf(znRecord.getSimpleField(START_OFFSET)); _numReplicas = Integer.valueOf(znRecord.getSimpleField(NUM_REPLICAS)); _endOffset = Long.valueOf(znRecord.getSimpleField(END_OFFSET)); _downloadUrl = znRecord.getSimpleField(DOWNLOAD_URL); }
public static boolean comparisonZNRecords(ZNRecord record1, ZNRecord record2) { if (!record1.getId().equals(record2.getId())) { return false; Map<String, String> simpleFieldMap1 = record1.getSimpleFields(); Map<String, String> simpleFieldMap2 = record2.getSimpleFields(); if (simpleFieldMap1.size() != simpleFieldMap2.size()) { return false; Map<String, List<String>> listFieldMap1 = record1.getListFields(); Map<String, List<String>> listFieldMap2 = record2.getListFields(); if (listFieldMap1.size() != listFieldMap2.size()) { return false; Map<String, Map<String, String>> mapFieldMap1 = record1.getMapFields(); Map<String, Map<String, String>> mapFieldMap2 = record2.getMapFields(); if (mapFieldMap1.size() != mapFieldMap2.size()) { return false;
private IdealState buildWorkflowIdealState(String workflow) { CustomModeISBuilder IsBuilder = new CustomModeISBuilder(workflow); IsBuilder.setRebalancerMode(IdealState.RebalanceMode.TASK).setNumReplica(1) .setNumPartitions(1).setStateModel(TaskConstants.STATE_MODEL_NAME).disableExternalView(); IdealState is = IsBuilder.build(); is.getRecord().setListField(workflow, new ArrayList<String>()); is.getRecord().setMapField(workflow, new HashMap<String, String>()); is.setRebalancerClassName(WorkflowRebalancer.class.getName()); return is; }
@Override public ZNRecord update(ZNRecord znRecord) { if (znRecord.getMapField(task) == null) { znRecord.setMapField(task, new HashMap<String, String>()); } znRecord.getMapField(task).put(key, value); return znRecord; } }, AccessOption.PERSISTENT);
public ZNRecord toZNRecord() { ZNRecord record = new ZNRecord(_tableNameWithType); record.setListFields(_parentGroupToChildrenGroupsMap); Map<String, Map<String, String>> groupToSegmentsMap = new HashMap<>(); for (Map.Entry<Integer, Map<String, List<String>>> entry : _levelToGroupToSegmentsMap.entrySet()) { String key = LEVEL_KEY_PREFIX + entry.getKey(); Map<String, String> groupSegmentsForLevel = new HashMap<>(); for (Map.Entry<String, List<String>> groupEntry : entry.getValue().entrySet()) { String groupId = groupEntry.getKey(); String segments = String.join(SEGMENT_DELIMITER, groupEntry.getValue()); groupSegmentsForLevel.put(groupId, segments); } groupToSegmentsMap.put(key, groupSegmentsForLevel); } record.setMapFields(groupToSegmentsMap); return record; }
public StateTransitionTimeoutConfig(ZNRecord record) { _resource = record.getId(); if (record.getMapFields().containsKey(StateTransitionTimeoutProperty.TIMEOUT.name())) { _timeoutMap = record.getMapField(StateTransitionTimeoutProperty.TIMEOUT.name()); } else { _timeoutMap = new HashMap<String, String>(); } }