@Test public void heartbeatFileUnderReplicatedAndLost() throws Exception { mFileContext.getOptions().setReplicationMin(2); long blockId = createBlockHelper(TEST_FILE_1, mFileContext); // Create a worker. long workerId = mBlockMaster.getWorkerId(new WorkerNetAddress().setHost("localhost") .setRpcPort(80).setDataPort(81).setWebPort(82)); mBlockMaster.workerRegister(workerId, Collections.singletonList("MEM"), ImmutableMap.of("MEM", 100L), ImmutableMap.of("MEM", 0L), NO_BLOCKS_ON_TIERS, RegisterWorkerPOptions.getDefaultInstance()); mBlockMaster.commitBlock(workerId, 50L, "MEM", blockId, 20L); // Indicate that blockId is removed on the worker. mBlockMaster.workerHeartbeat(workerId, null, ImmutableMap.of("MEM", 0L), ImmutableList.of(blockId), NO_BLOCKS_ON_TIERS, NO_METRICS); mReplicationChecker.heartbeat(); Assert.assertEquals(EMPTY, mMockReplicationHandler.getEvictRequests()); Assert.assertEquals(EMPTY, mMockReplicationHandler.getReplicateRequests()); }
@Test public void heartbeatFileUnderReplicatedBy10() throws Exception { mFileContext.getOptions().setReplicationMin(10); long blockId = createBlockHelper(TEST_FILE_1, mFileContext); mReplicationChecker.heartbeat(); Map<Long, Integer> expected = ImmutableMap.of(blockId, 10); Assert.assertEquals(EMPTY, mMockReplicationHandler.getEvictRequests()); Assert.assertEquals(expected, mMockReplicationHandler.getReplicateRequests()); }
@Test public void heartbeatFileUnderReplicatedBy1() throws Exception { mFileContext.getOptions().setReplicationMin(1); long blockId = createBlockHelper(TEST_FILE_1, mFileContext); mReplicationChecker.heartbeat(); Map<Long, Integer> expected = ImmutableMap.of(blockId, 1); Assert.assertEquals(EMPTY, mMockReplicationHandler.getEvictRequests()); Assert.assertEquals(expected, mMockReplicationHandler.getReplicateRequests()); }
setReplicationMin(other.getReplicationMin());
/** * @return Master side defaults for {@link CreateFilePOptions} */ public static CreateFilePOptions createFileDefaults() { return CreateFilePOptions.newBuilder() .setCommonOptions(commonDefaults()) .setRecursive(false) .setBlockSizeBytes(ServerConfiguration.getBytes(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT)) .setFileWriteLocationPolicy( ServerConfiguration.get(PropertyKey.USER_FILE_WRITE_LOCATION_POLICY)) .setWriteTier(ServerConfiguration.getInt(PropertyKey.USER_FILE_WRITE_TIER_DEFAULT)) .setWriteType(ServerConfiguration .getEnum(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.class).toProto()) .setMode(ModeUtils.applyFileUMask(Mode.defaults(), ServerConfiguration.get(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_UMASK)).toProto()) .setReplicationDurable(ServerConfiguration .getInt(PropertyKey.USER_FILE_REPLICATION_DURABLE)) .setReplicationMin(ServerConfiguration.getInt(PropertyKey.USER_FILE_REPLICATION_MIN)) .setReplicationMax(ServerConfiguration.getInt(PropertyKey.USER_FILE_REPLICATION_MAX)) .build(); }
@Test public void heartbeatFileWithinRange() throws Exception { mFileContext.getOptions().setReplicationMin(1).setReplicationMax(3); long blockId = createBlockHelper(TEST_FILE_1, mFileContext); // One replica, meeting replication min addBlockLocationHelper(blockId, 1); mReplicationChecker.heartbeat(); Assert.assertEquals(EMPTY, mMockReplicationHandler.getEvictRequests()); Assert.assertEquals(EMPTY, mMockReplicationHandler.getReplicateRequests()); // Two replicas, good heartbeatToAddLocationHelper(blockId, createWorkerHelper(1)); mReplicationChecker.heartbeat(); Assert.assertEquals(EMPTY, mMockReplicationHandler.getEvictRequests()); Assert.assertEquals(EMPTY, mMockReplicationHandler.getReplicateRequests()); // Three replicas, meeting replication max, still good heartbeatToAddLocationHelper(blockId, createWorkerHelper(2)); mReplicationChecker.heartbeat(); Assert.assertEquals(EMPTY, mMockReplicationHandler.getEvictRequests()); Assert.assertEquals(EMPTY, mMockReplicationHandler.getReplicateRequests()); }
@Test public void heartbeatFilesUnderAndOverReplicated() throws Exception { mFileContext.getOptions().setReplicationMin(2).setReplicationMax(-1); long blockId1 = createBlockHelper(TEST_FILE_1, mFileContext); mFileContext.getOptions().setReplicationMin(0).setReplicationMax(3); long blockId2 = createBlockHelper(TEST_FILE_2, mFileContext); addBlockLocationHelper(blockId1, 1); addBlockLocationHelper(blockId2, 5); mReplicationChecker.heartbeat(); Map<Long, Integer> expected1 = ImmutableMap.of(blockId1, 1); Map<Long, Integer> expected2 = ImmutableMap.of(blockId2, 2); Assert.assertEquals(expected2, mMockReplicationHandler.getEvictRequests()); Assert.assertEquals(expected1, mMockReplicationHandler.getReplicateRequests()); } }
@Test public void heartbeatMultipleFilesUnderReplicated() throws Exception { mFileContext.getOptions().setReplicationMin(1); long blockId1 = createBlockHelper(TEST_FILE_1, mFileContext); mFileContext.getOptions().setReplicationMin(2); long blockId2 = createBlockHelper(TEST_FILE_2, mFileContext); mReplicationChecker.heartbeat(); Map<Long, Integer> expected = ImmutableMap.of(blockId1, 1, blockId2, 2); Assert.assertEquals(EMPTY, mMockReplicationHandler.getEvictRequests()); Assert.assertEquals(expected, mMockReplicationHandler.getReplicateRequests()); }