public static SinkStat deserialize(@NonNull final String serializedStat) { final SinkStat sinkStat = new SinkStat(); sinkStat.stats.putAll(MapUtil.deserializeMap(serializedStat)); return sinkStat; }
public static String serialize(@NonNull final SinkStat sinkStat) { return MapUtil.serializeMap(sinkStat.stats); }
@Test public void testSerDeserMap() { final Map<String, String> map = new HashMap<>(); final int mapEntries = 10; for (int i = 0; i < mapEntries; i++) { map.put("prefixKey" + i + MapUtil.KEY_VALUE_SEPARATOR + MapUtil.KEYS_SEPARATOR + "suffix", "prefixValue" + i + MapUtil.KEY_VALUE_SEPARATOR + MapUtil.KEYS_SEPARATOR + "suffix"); } Assert.assertEquals(map.entrySet(), MapUtil.deserializeMap(MapUtil.serializeMap(map)).entrySet()); } }
/** * This method will also be used by HoodieSink to retrieve and store metadata information. * It returns {@link HashMap<String, String>} with hoodie metadata information to be saved into commit file. * It returns {@link HashMap} instead of {@link Map} because hoodie needs it that way. Checkout * {@link HoodieWriteClient#commit(String, JavaRDD, java.util.Optional)} for more info. */ public HashMap<String, String> getMetadataInfo() { final HashMap<String, String> map = new HashMap<>(); map.put(HOODIE_METADATA_KEY, MapUtil.serializeMap(this.metadataMap)); return map; }
public void init() { final Optional<StringValue> serialisedStats = this.metadataManager.get(getMetakey()); if (serialisedStats.isPresent()) { final Map<String, String> statHistory = MapUtil.deserializeMap(serialisedStats.get().getValue()); for (int i = 0; i < statHistory.size(); i++) { this.sinkStatQ.add(SinkStat.deserialize(statHistory.get(Integer.toString(i)))); } } }
public void persist() { final Map<String, String> stats = new HashMap<>(); if (!this.currentStat.isEmpty()) { this.sinkStatQ.add(this.currentStat); } while (this.sinkStatQ.size() > MAX_HISTORY_SIZE) { this.sinkStatQ.poll(); } for (int i = 0; !this.sinkStatQ.isEmpty(); i++) { stats.put(Integer.toString(i), SinkStat.serialize(this.sinkStatQ.poll())); } this.metadataManager.set(getMetakey(), new StringValue(MapUtil.serializeMap(stats))); }
private static Map<String, String> readMetadataInfo( @NonNull final HoodieConfiguration hoodieConf) { try { final FileSystem fs = FSUtils.getFs(hoodieConf.getConf()); HoodieUtil.initHoodieDataset(fs, hoodieConf); final HoodieTableMetaClient hoodieTableMetaClient = new HoodieTableMetaClient(new HadoopConfiguration(hoodieConf.getConf()).getHadoopConf(), hoodieConf.getBasePath(), true); final HoodieActiveTimeline hoodieActiveTimeline = hoodieTableMetaClient.getActiveTimeline(); final java.util.Optional<HoodieInstant> lastInstant = hoodieActiveTimeline.getCommitTimeline() .filterCompletedInstants().lastInstant(); if (lastInstant.isPresent()) { log.info("using hoodie instant for reading checkpoint info :{}", lastInstant.get().getTimestamp()); final HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes(hoodieActiveTimeline.getInstantDetails(lastInstant.get()).get()); final String serCommitInfo = commitMetadata.getMetadata(HOODIE_METADATA_KEY); if (!Strings.isNullOrEmpty(serCommitInfo)) { return MapUtil.deserializeMap(serCommitInfo); } } return new HashMap<>(); } catch (IOException e) { log.error("failed to read metadata info", e); throw new JobRuntimeException("failed to read metadata information", e); } } }