Tabnine Logo
FSEditLogLoader$EditLogValidation
Code IndexAdd Tabnine to your IDE (free)

How to use
FSEditLogLoader$EditLogValidation
in
org.apache.hadoop.hdfs.server.namenode

Best Java code snippets using org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader$EditLogValidation (Showing top 20 results out of 315)

origin: com.facebook.hadoop/hadoop-core

   " ops from " + in + " while determining its valid length.", t);
return new EditLogValidation(lastPos, firstTxId, lastTxId);
origin: ch.cern.hadoop/hadoop-hdfs

EditLogValidation validation =
  EditLogFileInputStream.validateEditLog(logFile);
assertTrue(!validation.hasCorruptHeader());
assertEquals(NUM_TXNS + 1, validation.getEndTxId());
   NUM_TXNS : (NUM_TXNS + 1);
 assertEquals("Failed when corrupting txn opcode at " + txOffset,
   expectedEndTxId, validation.getEndTxId());
 assertTrue(!validation.hasCorruptHeader());
   HdfsConstants.INVALID_TXID : (txId - 1);
 assertEquals("Failed when corrupting txid " + txId + " txn opcode " +
  "at " + txOffset, expectedEndTxId, validation.getEndTxId());
 assertTrue(!validation.hasCorruptHeader());
origin: ch.cern.hadoop/hadoop-hdfs

long read = (val.getEndTxId() - edits.getFirstTxId()) + 1;
LOG.info("Loading edits " + edits + " read " + read);
assertEquals(startTxId, edits.getFirstTxId());
origin: ch.cern.hadoop/hadoop-hdfs

} catch (LogHeaderCorruptException e) {
 LOG.warn("Log file " + file + " has no valid header", e);
 return new FSEditLogLoader.EditLogValidation(0,
   HdfsConstants.INVALID_TXID, true);
 return new EditLogValidation(lastPos, lastTxId, false);
} finally {
 IOUtils.closeStream(in);
origin: io.prestosql.hadoop/hadoop-apache

} catch (LogHeaderCorruptException e) {
 LOG.warn("Log file " + file + " has no valid header", e);
 return new FSEditLogLoader.EditLogValidation(0,
   HdfsConstants.INVALID_TXID, true);
 return new EditLogValidation(lastPos, lastTxId, false);
} finally {
 IOUtils.closeStream(in);
origin: ch.cern.hadoop/hadoop-hdfs

return new EditLogValidation(lastPos, lastTxId, false);
origin: io.prestosql.hadoop/hadoop-apache

return new EditLogValidation(lastPos, lastTxId, false);
origin: io.prestosql.hadoop/hadoop-apache

static EditLogValidation scanEditLog(EditLogInputStream in) {
 long lastPos = 0;
 long lastTxId = HdfsConstants.INVALID_TXID;
 long numValid = 0;
 FSEditLogOp op = null;
 while (true) {
  lastPos = in.getPosition();
  try {
   if ((op = in.readOp()) == null) { // TODO
    break;
   }
  } catch (Throwable t) {
   FSImage.LOG.warn("Caught exception after reading " + numValid +
     " ops from " + in + " while determining its valid length." +
     "Position was " + lastPos, t);
   in.resync();
   FSImage.LOG.warn("After resync, position is " + in.getPosition());
   continue;
  }
  if (lastTxId == HdfsConstants.INVALID_TXID
    || op.getTransactionId() > lastTxId) {
   lastTxId = op.getTransactionId();
  }
  numValid++;
 }
 return new EditLogValidation(lastPos, lastTxId, false);
}
origin: ch.cern.hadoop/hadoop-hdfs

static EditLogValidation scanEditLog(EditLogInputStream in) {
 long lastPos = 0;
 long lastTxId = HdfsConstants.INVALID_TXID;
 long numValid = 0;
 FSEditLogOp op = null;
 while (true) {
  lastPos = in.getPosition();
  try {
   if ((op = in.readOp()) == null) { // TODO
    break;
   }
  } catch (Throwable t) {
   FSImage.LOG.warn("Caught exception after reading " + numValid +
     " ops from " + in + " while determining its valid length." +
     "Position was " + lastPos, t);
   in.resync();
   FSImage.LOG.warn("After resync, position is " + in.getPosition());
   continue;
  }
  if (lastTxId == HdfsConstants.INVALID_TXID
    || op.getTransactionId() > lastTxId) {
   lastTxId = op.getTransactionId();
  }
  numValid++;
 }
 return new EditLogValidation(lastPos, lastTxId, false);
}
origin: io.prestosql.hadoop/hadoop-apache

static FSEditLogLoader.EditLogValidation validateEditLog(File file)
  throws IOException {
 EditLogFileInputStream in;
 try {
  in = new EditLogFileInputStream(file);
  in.getVersion(true); // causes us to read the header
 } catch (LogHeaderCorruptException e) {
  // If the header is malformed or the wrong value, this indicates a corruption
  LOG.warn("Log file " + file + " has no valid header", e);
  return new FSEditLogLoader.EditLogValidation(0,
    HdfsConstants.INVALID_TXID, true);
 }
 
 try {
  return FSEditLogLoader.validateEditLog(in);
 } finally {
  IOUtils.closeStream(in);
 }
}
origin: ch.cern.hadoop/hadoop-hdfs

static FSEditLogLoader.EditLogValidation validateEditLog(File file)
  throws IOException {
 EditLogFileInputStream in;
 try {
  in = new EditLogFileInputStream(file);
  in.getVersion(true); // causes us to read the header
 } catch (LogHeaderCorruptException e) {
  // If the header is malformed or the wrong value, this indicates a corruption
  LOG.warn("Log file " + file + " has no valid header", e);
  return new FSEditLogLoader.EditLogValidation(0,
    HdfsConstants.INVALID_TXID, true);
 }
 
 try {
  return FSEditLogLoader.validateEditLog(in);
 } finally {
  IOUtils.closeStream(in);
 }
}
origin: com.facebook.hadoop/hadoop-core

static FSEditLogLoader.EditLogValidation validateEditLog(File file) throws IOException {
 EditLogFileInputStream in;
 try {
  in = new EditLogFileInputStream(file);
 } catch (LogHeaderCorruptException corrupt) {
  // If it's missing its header, this is equivalent to no transactions
  FSImage.LOG.warn("Log at " + file + " has no valid header",
    corrupt);
  return new FSEditLogLoader.EditLogValidation(0, HdfsConstants.INVALID_TXID, 
                         HdfsConstants.INVALID_TXID);
 }
 
 try {
  return FSEditLogLoader.validateEditLog(in);
 } finally {
  IOUtils.closeStream(in);
 }
}
origin: ch.cern.hadoop/hadoop-hdfs

@Test
public void testValidateEditLogWithCorruptHeader() throws IOException {
 File testDir = new File(TEST_DIR, "testValidateEditLogWithCorruptHeader");
 SortedMap<Long, Long> offsetToTxId = Maps.newTreeMap();
 File logFile = prepareUnfinalizedTestEditLog(testDir, 2, offsetToTxId);
 RandomAccessFile rwf = new RandomAccessFile(logFile, "rw");
 try {
  rwf.seek(0);
  rwf.writeLong(42); // corrupt header
 } finally {
  rwf.close();
 }
 EditLogValidation validation = EditLogFileInputStream.validateEditLog(logFile);
 assertTrue(validation.hasCorruptHeader());
}
origin: io.prestosql.hadoop/hadoop-apache

public void scanLog() throws IOException {
 EditLogValidation val = EditLogFileInputStream.scanEditLog(file);
 this.lastTxId = val.getEndTxId();
 this.hasCorruptHeader = val.hasCorruptHeader();
}
origin: io.prestosql.hadoop/hadoop-apache

/** 
 * Find out where the edit log ends.
 * This will update the lastTxId of the EditLogFile or
 * mark it as corrupt if it is.
 */
public void validateLog() throws IOException {
 EditLogValidation val = EditLogFileInputStream.validateEditLog(file);
 this.lastTxId = val.getEndTxId();
 this.hasCorruptHeader = val.hasCorruptHeader();
}
origin: ch.cern.hadoop/hadoop-hdfs

public void scanLog() throws IOException {
 EditLogValidation val = EditLogFileInputStream.scanEditLog(file);
 this.lastTxId = val.getEndTxId();
 this.hasCorruptHeader = val.hasCorruptHeader();
}
origin: ch.cern.hadoop/hadoop-hdfs

/** 
 * Find out where the edit log ends.
 * This will update the lastTxId of the EditLogFile or
 * mark it as corrupt if it is.
 */
public void validateLog() throws IOException {
 EditLogValidation val = EditLogFileInputStream.validateEditLog(file);
 this.lastTxId = val.getEndTxId();
 this.hasCorruptHeader = val.hasCorruptHeader();
}
origin: ch.cern.hadoop/hadoop-hdfs

@Test
public void testValidateEmptyEditLog() throws IOException {
 File testDir = new File(TEST_DIR, "testValidateEmptyEditLog");
 SortedMap<Long, Long> offsetToTxId = Maps.newTreeMap();
 File logFile = prepareUnfinalizedTestEditLog(testDir, 0, offsetToTxId);
 // Truncate the file so that there is nothing except the header and
 // layout flags section.
 truncateFile(logFile, 8);
 EditLogValidation validation =
   EditLogFileInputStream.validateEditLog(logFile);
 assertTrue(!validation.hasCorruptHeader());
 assertEquals(HdfsConstants.INVALID_TXID, validation.getEndTxId());
}
origin: org.apache.hadoop/hadoop-hdfs

return new EditLogValidation(lastPos, lastTxId, false);
origin: org.apache.hadoop/hadoop-hdfs

/**
 * @param file          File being scanned and validated.
 * @param maxTxIdToScan Maximum Tx ID to try to scan.
 *                      The scan returns after reading this or a higher
 *                      ID. The file portion beyond this ID is
 *                      potentially being updated.
 * @return Result of the validation
 * @throws IOException
 */
static FSEditLogLoader.EditLogValidation scanEditLog(File file,
  long maxTxIdToScan, boolean verifyVersion)
  throws IOException {
 EditLogFileInputStream in;
 try {
  in = new EditLogFileInputStream(file);
  // read the header, initialize the inputstream, but do not check the
  // layoutversion
  in.getVersion(verifyVersion);
 } catch (LogHeaderCorruptException e) {
  LOG.warn("Log file " + file + " has no valid header", e);
  return new FSEditLogLoader.EditLogValidation(0,
    HdfsServerConstants.INVALID_TXID, true);
 }
 try {
  return FSEditLogLoader.scanEditLog(in, maxTxIdToScan);
 } finally {
  IOUtils.closeStream(in);
 }
}
org.apache.hadoop.hdfs.server.namenodeFSEditLogLoader$EditLogValidation

Most used methods

  • <init>
  • getEndTxId
  • hasCorruptHeader

Popular in Java

  • Reactive rest calls using spring rest template
  • scheduleAtFixedRate (ScheduledExecutorService)
  • putExtra (Intent)
  • getSupportFragmentManager (FragmentActivity)
  • Pointer (com.sun.jna)
    An abstraction for a native pointer data type. A Pointer instance represents, on the Java side, a na
  • URI (java.net)
    A Uniform Resource Identifier that identifies an abstract or physical resource, as specified by RFC
  • ArrayList (java.util)
    ArrayList is an implementation of List, backed by an array. All optional operations including adding
  • ResourceBundle (java.util)
    ResourceBundle is an abstract class which is the superclass of classes which provide Locale-specifi
  • Stack (java.util)
    Stack is a Last-In/First-Out(LIFO) data structure which represents a stack of objects. It enables u
  • JFileChooser (javax.swing)
  • Top plugins for WebStorm
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now