@SuppressWarnings("deprecation") @Override protected void initOutput(FileSystem fs, Path path, boolean overwritable, int bufferSize, short replication, long blockSize) throws IOException, StreamLacksCapabilityException { this.output = CommonFSUtils.createForWal(fs, path, overwritable, bufferSize, replication, blockSize, false); if (fs.getConf().getBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true)) { if (!CommonFSUtils.hasCapability(output, "hflush")) { throw new StreamLacksCapabilityException("hflush"); } if (!CommonFSUtils.hasCapability(output, "hsync")) { throw new StreamLacksCapabilityException("hsync"); } } }
!(CommonFSUtils.hasCapability(out, "hflush") && CommonFSUtils.hasCapability(out, "hsync"))) { out.close(); throw new CommonFSUtils.StreamLacksCapabilityException("hflush and hsync");
@Test(expected=NullPointerException.class) public void streamCapabilitiesDoesNotAllowNullStream() { CommonFSUtils.hasCapability(null, "hopefully any string"); }
@Test public void checkStreamCapabilitiesOnKnownNoopStream() throws IOException { FSDataOutputStream stream = new FSDataOutputStream(new ByteArrayOutputStream(), null); assertNotEquals("We expect our dummy FSDOS to claim capabilities iff the StreamCapabilities " + "class is not defined.", STREAM_CAPABILITIES_IS_PRESENT, CommonFSUtils.hasCapability(stream, "hsync")); assertNotEquals("We expect our dummy FSDOS to claim capabilities iff the StreamCapabilities " + "class is not defined.", STREAM_CAPABILITIES_IS_PRESENT, CommonFSUtils.hasCapability(stream, "hflush")); assertNotEquals("We expect our dummy FSDOS to claim capabilities iff the StreamCapabilities " + "class is not defined.", STREAM_CAPABILITIES_IS_PRESENT, CommonFSUtils.hasCapability(stream, "a capability that hopefully no filesystem will " + "implement.")); } }
if (enforceStreamCapability && !(CommonFSUtils.hasCapability(newStream, durability))) { throw new IllegalStateException("The procedure WAL relies on the ability to " + durability + " for proper operation during component failures, but the underlying filesystem does " +
@Test public void testStreamCreate() throws IOException { try (FSDataOutputStream out = CommonFSUtils.createForWal(util.getDFSCluster().getFileSystem(), new Path("/testStreamCreate"), true)) { assertTrue(CommonFSUtils.hasCapability(out, HFLUSH)); } }
@BeforeClass public static void setup() throws Exception { try { MiniDFSCluster cluster = util.startMiniDFSCluster(3); // Need 3 DNs for RS-3-2 policy DistributedFileSystem fs = cluster.getFileSystem(); Method enableAllECPolicies = DFSTestUtil.class.getMethod("enableAllECPolicies", DistributedFileSystem.class); enableAllECPolicies.invoke(null, fs); DFSClient client = fs.getClient(); Method setErasureCodingPolicy = DFSClient.class.getMethod("setErasureCodingPolicy", String.class, String.class); setErasureCodingPolicy.invoke(client, "/", "RS-3-2-1024k"); // try a built-in policy try (FSDataOutputStream out = fs.create(new Path("/canary"))) { // If this comes back as having hflush then some test setup assumption is wrong. // Fail the test so that a developer has to look and triage assertFalse("Did not enable EC!", CommonFSUtils.hasCapability(out, HFLUSH)); } } catch (NoSuchMethodException e) { // We're not testing anything interesting if EC is not available, so skip the rest of the test Assume.assumeNoException("Using an older version of hadoop; EC not available.", e); } util.getConfiguration().setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true); util.startMiniCluster(); }
@Test(expected=NullPointerException.class) public void streamCapabilitiesDoesNotAllowNullStream() { CommonFSUtils.hasCapability(null, "hopefully any string"); }
@Test(expected=NullPointerException.class) public void streamCapabilitiesDoesNotAllowNullStream() { CommonFSUtils.hasCapability(null, "hopefully any string"); }
@Test public void checkStreamCapabilitiesOnKnownNoopStream() throws IOException { FSDataOutputStream stream = new FSDataOutputStream(new ByteArrayOutputStream(), null); assertNotEquals("We expect our dummy FSDOS to claim capabilities iff the StreamCapabilities " + "class is not defined.", STREAM_CAPABILITIES_IS_PRESENT, CommonFSUtils.hasCapability(stream, "hsync")); assertNotEquals("We expect our dummy FSDOS to claim capabilities iff the StreamCapabilities " + "class is not defined.", STREAM_CAPABILITIES_IS_PRESENT, CommonFSUtils.hasCapability(stream, "hflush")); assertNotEquals("We expect our dummy FSDOS to claim capabilities iff the StreamCapabilities " + "class is not defined.", STREAM_CAPABILITIES_IS_PRESENT, CommonFSUtils.hasCapability(stream, "a capability that hopefully no filesystem will " + "implement.")); } }
@Test public void checkStreamCapabilitiesOnKnownNoopStream() throws IOException { FSDataOutputStream stream = new FSDataOutputStream(new ByteArrayOutputStream(), null); assertNotEquals("We expect our dummy FSDOS to claim capabilities iff the StreamCapabilities " + "class is not defined.", STREAM_CAPABILITIES_IS_PRESENT, CommonFSUtils.hasCapability(stream, "hsync")); assertNotEquals("We expect our dummy FSDOS to claim capabilities iff the StreamCapabilities " + "class is not defined.", STREAM_CAPABILITIES_IS_PRESENT, CommonFSUtils.hasCapability(stream, "hflush")); assertNotEquals("We expect our dummy FSDOS to claim capabilities iff the StreamCapabilities " + "class is not defined.", STREAM_CAPABILITIES_IS_PRESENT, CommonFSUtils.hasCapability(stream, "a capability that hopefully no filesystem will " + "implement.")); } }
if (enforceStreamCapability && !(CommonFSUtils.hasCapability(newStream, durability))) { throw new IllegalStateException("The procedure WAL relies on the ability to " + durability + " for proper operation during component failures, but the underlying filesystem does " +
if (enforceStreamCapability && !(CommonFSUtils.hasCapability(newStream, durability))) { throw new IllegalStateException("The procedure WAL relies on the ability to " + durability + " for proper operation during component failures, but the underlying filesystem does " +
@Test public void testStreamCreate() throws IOException { try (FSDataOutputStream out = CommonFSUtils.createForWal(util.getDFSCluster().getFileSystem(), new Path("/testStreamCreate"), true)) { assertTrue(CommonFSUtils.hasCapability(out, HFLUSH)); } }
@BeforeClass public static void setup() throws Exception { try { MiniDFSCluster cluster = util.startMiniDFSCluster(3); // Need 3 DNs for RS-3-2 policy DistributedFileSystem fs = cluster.getFileSystem(); Method enableAllECPolicies = DFSTestUtil.class.getMethod("enableAllECPolicies", DistributedFileSystem.class); enableAllECPolicies.invoke(null, fs); DFSClient client = fs.getClient(); Method setErasureCodingPolicy = DFSClient.class.getMethod("setErasureCodingPolicy", String.class, String.class); setErasureCodingPolicy.invoke(client, "/", "RS-3-2-1024k"); // try a built-in policy try (FSDataOutputStream out = fs.create(new Path("/canary"))) { // If this comes back as having hflush then some test setup assumption is wrong. // Fail the test so that a developer has to look and triage assertFalse("Did not enable EC!", CommonFSUtils.hasCapability(out, HFLUSH)); } } catch (NoSuchMethodException e) { // We're not testing anything interesting if EC is not available, so skip the rest of the test Assume.assumeNoException("Using an older version of hadoop; EC not available.", e); } util.getConfiguration().setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true); util.startMiniCluster(); }