@Override protected HoodieRollingStatMetadata getRollingStats() { try { Optional<HoodieInstant> lastInstant = this.getActiveTimeline().getDeltaCommitTimeline().filterCompletedInstants() .lastInstant(); if (lastInstant.isPresent()) { HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes( this.getActiveTimeline().getInstantDetails(lastInstant.get()).get(), HoodieCommitMetadata.class); Optional<String> lastRollingStat = Optional.ofNullable(commitMetadata.getExtraMetadata() .get(HoodieRollingStatMetadata.ROLLING_STAT_METADATA_KEY)); if (lastRollingStat.isPresent()) { return HoodieCommitMetadata .fromBytes(lastRollingStat.get().getBytes(), HoodieRollingStatMetadata.class); } } return null; } catch (IOException e) { throw new HoodieException(); } }
@Override protected HoodieRollingStatMetadata getRollingStats() { try { Optional<HoodieInstant> lastInstant = this.getActiveTimeline().getDeltaCommitTimeline().filterCompletedInstants() .lastInstant(); if (lastInstant.isPresent()) { HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes( this.getActiveTimeline().getInstantDetails(lastInstant.get()).get(), HoodieCommitMetadata.class); Optional<String> lastRollingStat = Optional.ofNullable(commitMetadata.getExtraMetadata() .get(HoodieRollingStatMetadata.ROLLING_STAT_METADATA_KEY)); if (lastRollingStat.isPresent()) { HoodieRollingStatMetadata rollingStatMetadata = HoodieCommitMetadata .fromBytes(lastRollingStat.get().getBytes(), HoodieRollingStatMetadata.class); return rollingStatMetadata; } } return null; } catch (IOException e) { throw new HoodieException(); } }
/** * Get the list of data file names savepointed */ public Stream<String> getSavepointedDataFiles(String savepointTime) { if (!getSavepoints().contains(savepointTime)) { throw new HoodieSavepointException( "Could not get data files for savepoint " + savepointTime + ". No such savepoint."); } HoodieInstant instant = new HoodieInstant(false, HoodieTimeline.SAVEPOINT_ACTION, savepointTime); HoodieSavepointMetadata metadata = null; try { metadata = AvroUtils .deserializeHoodieSavepointMetadata(getActiveTimeline().getInstantDetails(instant).get()); } catch (IOException e) { throw new HoodieSavepointException( "Could not get savepointed data files for savepoint " + savepointTime, e); } return metadata.getPartitionMetadata().values().stream() .flatMap(s -> s.getSavepointDataFile().stream()); }
/** * Get the list of data file names savepointed */ public Stream<String> getSavepointedDataFiles(String savepointTime) { if (!getSavepoints().contains(savepointTime)) { throw new HoodieSavepointException( "Could not get data files for savepoint " + savepointTime + ". No such savepoint."); } HoodieInstant instant = new HoodieInstant(false, HoodieTimeline.SAVEPOINT_ACTION, savepointTime); HoodieSavepointMetadata metadata = null; try { metadata = AvroUtils .deserializeHoodieSavepointMetadata(getActiveTimeline().getInstantDetails(instant).get()); } catch (IOException e) { throw new HoodieSavepointException( "Could not get savepointed data files for savepoint " + savepointTime, e); } return metadata.getPartitionMetadata().values().stream() .flatMap(s -> s.getSavepointDataFile().stream()); }
final List<Comparable[]> rows = new ArrayList<>(); HoodieRollbackMetadata metadata = AvroUtils.deserializeAvroMetadata( activeTimeline.getInstantDetails(new HoodieInstant(State.COMPLETED, ROLLBACK_ACTION, rollbackInstant)) .get(), HoodieRollbackMetadata.class); metadata.getPartitionMetadata().entrySet().forEach(e -> {
private static Map<String, String> readMetadataInfo( @NonNull final HoodieConfiguration hoodieConf) { try { final FileSystem fs = FSUtils.getFs(hoodieConf.getConf()); HoodieUtil.initHoodieDataset(fs, hoodieConf); final HoodieTableMetaClient hoodieTableMetaClient = new HoodieTableMetaClient(new HadoopConfiguration(hoodieConf.getConf()).getHadoopConf(), hoodieConf.getBasePath(), true); final HoodieActiveTimeline hoodieActiveTimeline = hoodieTableMetaClient.getActiveTimeline(); final java.util.Optional<HoodieInstant> lastInstant = hoodieActiveTimeline.getCommitTimeline() .filterCompletedInstants().lastInstant(); if (lastInstant.isPresent()) { log.info("using hoodie instant for reading checkpoint info :{}", lastInstant.get().getTimestamp()); final HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes(hoodieActiveTimeline.getInstantDetails(lastInstant.get()).get()); final String serCommitInfo = commitMetadata.getMetadata(HOODIE_METADATA_KEY); if (!Strings.isNullOrEmpty(serCommitInfo)) { return MapUtil.deserializeMap(serCommitInfo); } } return new HashMap<>(); } catch (IOException e) { log.error("failed to read metadata info", e); throw new JobRuntimeException("failed to read metadata information", e); } } }
for (HoodieInstant commitTime : timeline.getInstants().collect(Collectors.toList())) { String waf = "0"; HoodieCommitMetadata commit = HoodieCommitMetadata.fromBytes(activeTimeline.getInstantDetails(commitTime).get(), HoodieCommitMetadata.class); if (commit.fetchTotalUpdateRecordsWritten() > 0) {
try { HoodieRollbackMetadata metadata = AvroUtils.deserializeAvroMetadata( activeTimeline.getInstantDetails(instant).get(), HoodieRollbackMetadata.class); metadata.getCommitsRollback().forEach(c -> { Comparable[] row = new Comparable[5];
@Test public void checkSerDe() throws IOException, ClassNotFoundException { // check if this object is serialized and de-serialized, we are able to read from the file system HoodieTableMetaClient deseralizedMetaClient = HoodieTestUtils .serializeDeserialize(metaClient, HoodieTableMetaClient.class); assertNotNull(deseralizedMetaClient); HoodieActiveTimeline commitTimeline = deseralizedMetaClient.getActiveTimeline(); HoodieInstant instant = new HoodieInstant(true, HoodieTimeline.COMMIT_ACTION, "1"); commitTimeline.createInflight(instant); commitTimeline.saveAsComplete(instant, Optional.of("test-detail".getBytes())); commitTimeline = commitTimeline.reload(); HoodieInstant completedInstant = HoodieTimeline.getCompletedInstant(instant); assertEquals("Commit should be 1 and completed", completedInstant, commitTimeline.getInstants().findFirst().get()); assertArrayEquals("Commit value should be \"test-detail\"", "test-detail".getBytes(), commitTimeline.getInstantDetails(completedInstant).get()); }
return "Commit " + commitTime + " not found in Commits " + timeline; HoodieCommitMetadata meta = HoodieCommitMetadata.fromBytes(activeTimeline.getInstantDetails(commitInstant).get(), HoodieCommitMetadata.class); List<Comparable[]> rows = new ArrayList<>();
if (lastInstant.isPresent()) { HoodieCommitMetadata commitMetadata = HoodieCommitMetadata .fromBytes(table.getActiveTimeline().getInstantDetails(lastInstant .get()).get(), HoodieCommitMetadata.class); Optional<String> lastRollingStat = Optional.ofNullable(commitMetadata.getExtraMetadata()
if (cleanInstant.isPresent()) { HoodieCleanMetadata cleanMetadata = AvroUtils.deserializeHoodieCleanMetadata( table.getActiveTimeline().getInstantDetails(cleanInstant.get()).get()); lastCommitRetained = cleanMetadata.getEarliestCommitToRetain(); } else {
if (cleanInstant.isPresent()) { HoodieCleanMetadata cleanMetadata = AvroUtils.deserializeHoodieCleanMetadata( table.getActiveTimeline().getInstantDetails(cleanInstant.get()).get()); lastCommitRetained = cleanMetadata.getEarliestCommitToRetain(); } else {
return "Commit " + commitTime + " not found in Commits " + timeline; HoodieCommitMetadata meta = HoodieCommitMetadata.fromBytes(activeTimeline.getInstantDetails(commitInstant).get(), HoodieCommitMetadata.class); List<Comparable[]> rows = new ArrayList<>();
if (lastInstant.isPresent()) { HoodieCommitMetadata commitMetadata = HoodieCommitMetadata .fromBytes(table.getActiveTimeline().getInstantDetails(lastInstant .get()).get(), HoodieCommitMetadata.class); Optional<String> lastRollingStat = Optional.ofNullable(commitMetadata.getExtraMetadata()
HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes(table.getActiveTimeline().getInstantDetails(table .getActiveTimeline().getDeltaCommitTimeline().lastInstant().get()).get(), HoodieCommitMetadata.class); HoodieRollingStatMetadata rollingStatMetadata = HoodieCommitMetadata.fromBytes(metadata.getExtraMetadata() metadata = HoodieCommitMetadata.fromBytes(table.getActiveTimeline().getInstantDetails(table .getActiveTimeline().getDeltaCommitTimeline().lastInstant().get()).get(), HoodieCommitMetadata.class); rollingStatMetadata = HoodieCommitMetadata.fromBytes(metadata.getExtraMetadata() metadata = HoodieCommitMetadata.fromBytes(table.getActiveTimeline().getInstantDetails(table .getActiveTimeline().getCommitsTimeline().lastInstant().get()).get(), HoodieCommitMetadata.class); HoodieRollingStatMetadata rollingStatMetadata1 = HoodieCommitMetadata.fromBytes(metadata.getExtraMetadata() metadata = HoodieCommitMetadata.fromBytes(table.getActiveTimeline().getInstantDetails(table .getActiveTimeline().getDeltaCommitTimeline().lastInstant().get()).get(), HoodieCommitMetadata.class); rollingStatMetadata = HoodieCommitMetadata.fromBytes(metadata.getExtraMetadata()
HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes(table.getActiveTimeline().getInstantDetails(table .getActiveTimeline().getDeltaCommitTimeline().lastInstant().get()).get(), HoodieCommitMetadata.class); HoodieRollingStatMetadata rollingStatMetadata = HoodieCommitMetadata.fromBytes(metadata.getExtraMetadata() metadata = HoodieCommitMetadata.fromBytes(table.getActiveTimeline().getInstantDetails(table .getActiveTimeline().getDeltaCommitTimeline().lastInstant().get()).get(), HoodieCommitMetadata.class); rollingStatMetadata = HoodieCommitMetadata.fromBytes(metadata.getExtraMetadata() metadata = HoodieCommitMetadata.fromBytes(table.getActiveTimeline().getInstantDetails(table .getActiveTimeline().getDeltaCommitTimeline().lastInstant().get()).get(), HoodieCommitMetadata.class); rollingStatMetadata = HoodieCommitMetadata.fromBytes(metadata.getExtraMetadata()