private LocalReplica buildFinalizedReplica() throws IllegalArgumentException { if (null != fromReplica && fromReplica.getState() == ReplicaState.FINALIZED) { return new FinalizedReplica((FinalizedReplica)fromReplica); } else if (null != this.fromReplica) { throw new IllegalArgumentException("Incompatible fromReplica " + "state: " + fromReplica.getState()); } else { if (null != block) { return new FinalizedReplica(block, volume, directoryUsed, lastPartialChunkChecksum); } else { return new FinalizedReplica(blockId, length, genStamp, volume, directoryUsed, lastPartialChunkChecksum); } } }
private static ReplicaInfo createReplicaInfo(Block b) { return new FinalizedReplica(b, null, null); }
@Override // FSDatasetInterface public FinalizedReplica updateReplicaUnderRecovery(Block oldBlock, long recoveryId, long newlength) throws IOException { return new FinalizedReplica( oldBlock.getBlockId(), newlength, recoveryId, null, null); }
private static ReplicaInfo createReplicaInfo(Block b) { return new FinalizedReplica(b, null, null); }
@Before public void setup() { map.add(new FinalizedReplica(block, null, null)); }
/** * Get the list of finalized blocks from in-memory blockmap for a block pool. */ @Override public synchronized List<FinalizedReplica> getFinalizedBlocks(String bpid) { ArrayList<FinalizedReplica> finalized = new ArrayList<FinalizedReplica>(volumeMap.size(bpid)); for (ReplicaInfo b : volumeMap.replicas(bpid)) { if(b.getState() == ReplicaState.FINALIZED) { finalized.add(new FinalizedReplica((FinalizedReplica)b)); } } return finalized; }
/** * Get the list of finalized blocks from in-memory blockmap for a block pool. */ @Override public synchronized List<FinalizedReplica> getFinalizedBlocks(String bpid) { ArrayList<FinalizedReplica> finalized = new ArrayList<FinalizedReplica>(volumeMap.size(bpid)); for (ReplicaInfo b : volumeMap.replicas(bpid)) { if(b.getState() == ReplicaState.FINALIZED) { finalized.add(new FinalizedReplica((FinalizedReplica)b)); } } return finalized; }
/** * Get the list of finalized blocks from in-memory blockmap for a block pool. */ @Override public synchronized List<FinalizedReplica> getFinalizedBlocksOnPersistentStorage(String bpid) { ArrayList<FinalizedReplica> finalized = new ArrayList<FinalizedReplica>(volumeMap.size(bpid)); for (ReplicaInfo b : volumeMap.replicas(bpid)) { if(!b.getVolume().isTransientStorage() && b.getState() == ReplicaState.FINALIZED) { finalized.add(new FinalizedReplica((FinalizedReplica)b)); } } return finalized; }
/** * Get the list of finalized blocks from in-memory blockmap for a block pool. */ @Override public synchronized List<FinalizedReplica> getFinalizedBlocksOnPersistentStorage(String bpid) { ArrayList<FinalizedReplica> finalized = new ArrayList<FinalizedReplica>(volumeMap.size(bpid)); for (ReplicaInfo b : volumeMap.replicas(bpid)) { if(!b.getVolume().isTransientStorage() && b.getState() == ReplicaState.FINALIZED) { finalized.add(new FinalizedReplica((FinalizedReplica)b)); } } return finalized; }
private void setup(FSDataset dataSet) throws IOException { // setup replicas map ReplicasMap replicasMap = dataSet.volumeMap; FSVolume vol = dataSet.volumes.getNextVolume(0); ReplicaInfo replicaInfo = new FinalizedReplica( blocks[FINALIZED], vol, vol.getDir()); replicasMap.add(replicaInfo); replicaInfo.getBlockFile().createNewFile(); replicaInfo.getMetaFile().createNewFile(); replicasMap.add(new ReplicaInPipeline( blocks[TEMPORARY].getBlockId(), blocks[TEMPORARY].getGenerationStamp(), vol, vol.createTmpFile(blocks[TEMPORARY]).getParentFile())); replicaInfo = new ReplicaBeingWritten(blocks[RBW], vol, vol.createRbwFile(blocks[RBW]).getParentFile(), null); replicasMap.add(replicaInfo); replicaInfo.getBlockFile().createNewFile(); replicaInfo.getMetaFile().createNewFile(); replicasMap.add(new ReplicaWaitingToBeRecovered(blocks[RWR], vol, vol.createRbwFile(blocks[RWR]).getParentFile())); replicasMap.add(new ReplicaUnderRecovery( new FinalizedReplica(blocks[RUR], vol, vol.getDir()), 2007)); }
@Before public void setup() { map.add(bpid, new FinalizedReplica(block, null, null)); }
@Test public void testDuplicateReplicaResolution() throws IOException { FsVolumeImpl fsv1 = Mockito.mock(FsVolumeImpl.class); FsVolumeImpl fsv2 = Mockito.mock(FsVolumeImpl.class); File f1 = new File("d1/block"); File f2 = new File("d2/block"); ReplicaInfo replicaOlder = new FinalizedReplica(1,1,1,fsv1,f1); ReplicaInfo replica = new FinalizedReplica(1,2,2,fsv1,f1); ReplicaInfo replicaSame = new FinalizedReplica(1,2,2,fsv1,f1); ReplicaInfo replicaNewer = new FinalizedReplica(1,3,3,fsv1,f1); ReplicaInfo replicaOtherOlder = new FinalizedReplica(1,1,1,fsv2,f2); ReplicaInfo replicaOtherSame = new FinalizedReplica(1,2,2,fsv2,f2); ReplicaInfo replicaOtherNewer = new FinalizedReplica(1,3,3,fsv2,f2); // equivalent path so don't remove either assertNull(BlockPoolSlice.selectReplicaToDelete(replicaSame, replica)); assertNull(BlockPoolSlice.selectReplicaToDelete(replicaOlder, replica)); assertNull(BlockPoolSlice.selectReplicaToDelete(replicaNewer, replica)); // keep latest found replica assertSame(replica, BlockPoolSlice.selectReplicaToDelete(replicaOtherSame, replica)); assertSame(replicaOtherOlder, BlockPoolSlice.selectReplicaToDelete(replicaOtherOlder, replica)); assertSame(replica, BlockPoolSlice.selectReplicaToDelete(replicaOtherNewer, replica)); }
@Test public void testFuzz() throws InterruptedException { Replica[] replicas = new Replica[100000]; Random rand = new Random(0); for (int i=0; i<replicas.length; i++) { Block b = new Block(rand.nextLong(), i, i<<4); switch (rand.nextInt(2)) { case 0: replicas[i] = new FinalizedReplica(b, null, null); break; case 1: replicas[i] = new ReplicaBeingWritten(b, null, null, null); break; case 2: replicas[i] = new ReplicaWaitingToBeRecovered(b, null, null); break; } } checkReport(replicas); }
/** * This should be primarily used for testing. * @return clone of replica store in datanode memory */ ReplicaInfo fetchReplicaInfo(String bpid, long blockId) { ReplicaInfo r = volumeMap.get(bpid, blockId); if(r == null) return null; switch(r.getState()) { case FINALIZED: return new FinalizedReplica((FinalizedReplica)r); case RBW: return new ReplicaBeingWritten((ReplicaBeingWritten)r); case RWR: return new ReplicaWaitingToBeRecovered((ReplicaWaitingToBeRecovered)r); case RUR: return new ReplicaUnderRecovery((ReplicaUnderRecovery)r); case TEMPORARY: return new ReplicaInPipeline((ReplicaInPipeline)r); } return null; }
/** * This should be primarily used for testing. * @return clone of replica store in datanode memory */ ReplicaInfo fetchReplicaInfo(String bpid, long blockId) { ReplicaInfo r = volumeMap.get(bpid, blockId); if(r == null) return null; switch(r.getState()) { case FINALIZED: return new FinalizedReplica((FinalizedReplica)r); case RBW: return new ReplicaBeingWritten((ReplicaBeingWritten)r); case RWR: return new ReplicaWaitingToBeRecovered((ReplicaWaitingToBeRecovered)r); case RUR: return new ReplicaUnderRecovery((ReplicaUnderRecovery)r); case TEMPORARY: return new ReplicaInPipeline((ReplicaInPipeline)r); } return null; }
@Test public void testFinalized() { BlockListAsLongs blocks = checkReport( new FinalizedReplica(b1, null, null)); assertArrayEquals( new long[] { 1, 0, 1, 11, 111, -1, -1, -1 }, blocks.getBlockListAsLongs()); }
@Test public void testMix() { BlockListAsLongs blocks = checkReport( new FinalizedReplica(b1, null, null), new FinalizedReplica(b2, null, null), new ReplicaBeingWritten(b3, null, null, null), new ReplicaWaitingToBeRecovered(b4, null, null)); assertArrayEquals( new long[] { 2, 2, 1, 11, 111, 2, 22, 222, -1, -1, -1, 3, 33, 333, ReplicaState.RBW.getValue(), 4, 44, 444, ReplicaState.RWR.getValue() }, blocks.getBlockListAsLongs()); }
private synchronized FinalizedReplica finalizeReplica(String bpid, ReplicaInfo replicaInfo) throws IOException { FinalizedReplica newReplicaInfo = null; if (replicaInfo.getState() == ReplicaState.RUR && ((ReplicaUnderRecovery)replicaInfo).getOriginalReplica().getState() == ReplicaState.FINALIZED) { newReplicaInfo = (FinalizedReplica) ((ReplicaUnderRecovery)replicaInfo).getOriginalReplica(); } else { FsVolumeImpl v = (FsVolumeImpl)replicaInfo.getVolume(); File f = replicaInfo.getBlockFile(); if (v == null) { throw new IOException("No volume for temporary file " + f + " for block " + replicaInfo); } File dest = v.addFinalizedBlock( bpid, replicaInfo, f, replicaInfo.getBytesReserved()); newReplicaInfo = new FinalizedReplica(replicaInfo, v, dest.getParentFile()); if (v.isTransientStorage()) { ramDiskReplicaTracker.addReplica(bpid, replicaInfo.getBlockId(), v); datanode.getMetrics().addRamDiskBytesWrite(replicaInfo.getNumBytes()); } } volumeMap.add(bpid, newReplicaInfo); return newReplicaInfo; }
@Test public void testRemove() { // Test 1: null argument throws invalid argument exception try { map.remove(null); fail("Expected exception not thrown"); } catch (IllegalArgumentException expected) { } // Test 2: remove failure - generation stamp mismatch Block b = new Block(block); b.setGenerationStamp(0); assertNull(map.remove(b)); // Test 3: remove failure - blockID mismatch b.setGenerationStamp(block.getGenerationStamp()); b.setBlockId(0); assertNull(map.remove(b)); // Test 4: remove success assertNotNull(map.remove(block)); // Test 5: remove failure - invalid blockID assertNull(map.remove(0)); // Test 6: remove success map.add(new FinalizedReplica(block, null, null)); assertNotNull(map.remove(block.getBlockId())); } }
@Test public void testRemove() { // Test 1: null argument throws invalid argument exception try { map.remove(bpid, null); fail("Expected exception not thrown"); } catch (IllegalArgumentException expected) { } // Test 2: remove failure - generation stamp mismatch Block b = new Block(block); b.setGenerationStamp(0); assertNull(map.remove(bpid, b)); // Test 3: remove failure - blockID mismatch b.setGenerationStamp(block.getGenerationStamp()); b.setBlockId(0); assertNull(map.remove(bpid, b)); // Test 4: remove success assertNotNull(map.remove(bpid, block)); // Test 5: remove failure - invalid blockID assertNull(map.remove(bpid, 0)); // Test 6: remove success map.add(bpid, new FinalizedReplica(block, null, null)); assertNotNull(map.remove(bpid, block.getBlockId())); } }