@Test public void getInStreamMissingBlock() throws Exception { URIStatus dummyStatus = new URIStatus( new FileInfo().setPersisted(false).setBlockIds(Collections.singletonList(0L))); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); when(mMasterClient.getWorkerInfoList()).thenReturn(Collections.emptyList()); mException.expect(NotFoundException.class); mException.expectMessage("unavailable in both Alluxio and UFS"); mBlockStore.getInStream(BLOCK_ID, options).getAddress(); }
@Test public void getInStreamNoWorkers() throws Exception { URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L))); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); when(mMasterClient.getWorkerInfoList()).thenReturn(Collections.emptyList()); mException.expect(UnavailableException.class); mException.expectMessage("No Alluxio worker available"); mBlockStore.getInStream(BLOCK_ID, options).getAddress(); }
private void testGetInStreamFallback(int workerCount, boolean isPersisted, int[] blockLocations, Map<Integer, Long> failedWorkers, int expectedWorker) throws Exception { WorkerNetAddress[] workers = new WorkerNetAddress[workerCount]; Arrays.setAll(workers, i -> new WorkerNetAddress().setHost(String.format("worker-%d", i))); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID) .setLocations(Arrays.stream(blockLocations).mapToObj(x -> new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList())); URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(isPersisted) .setBlockIds(Collections.singletonList(BLOCK_ID)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class); when(mockPolicy.getWorker(any())).thenAnswer(arg -> arg .getArgumentAt(0, GetWorkerOptions.class).getBlockWorkerInfos().iterator().next() .getNetAddress()); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); options.setUfsReadLocationPolicy(mockPolicy); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); when(mMasterClient.getWorkerInfoList()).thenReturn(Arrays.stream(workers) .map(x -> new WorkerInfo().setAddress(x)).collect((Collectors.toList()))); Map<WorkerNetAddress, Long> failedWorkerAddresses = failedWorkers.entrySet().stream() .map(x -> new AbstractMap.SimpleImmutableEntry<>(workers[x.getKey()], x.getValue())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); BlockInStream inStream = mBlockStore.getInStream(BLOCK_ID, options, failedWorkerAddresses); assertEquals(workers[expectedWorker], inStream.getAddress()); } }
new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList())); URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(persisted) .setBlockIds(Collections.singletonList(BLOCK_ID)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
result.setOwner(userName); result.setPath(path); result.setPersisted(persisted); result.setPersistenceState(persistenceState); result.setPinned(pinned);
@Test public void getInStreamUfs() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1"); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2"); BlockInfo info = new BlockInfo().setBlockId(0); URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); OpenFilePOptions readOptions = OpenFilePOptions.newBuilder() .setFileReadLocationPolicy(MockFileWriteLocationPolicy.class.getTypeName()).build(); InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, sConf); ((MockFileWriteLocationPolicy) options.getUfsReadLocationPolicy()) .setHosts(Arrays.asList(worker1, worker2)); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); when(mMasterClient.getWorkerInfoList()).thenReturn( Arrays.asList(new WorkerInfo().setAddress(worker1), new WorkerInfo().setAddress(worker2))); // Location policy chooses worker1 first. assertEquals(worker1, mBlockStore.getInStream(BLOCK_ID, options).getAddress()); // Location policy chooses worker2 second. assertEquals(worker2, mBlockStore.getInStream(BLOCK_ID, options).getAddress()); }
ret.setPinned(isPinned()); ret.setCacheable(false); ret.setPersisted(isPersisted()); ret.setLastModificationTimeMs(getLastModificationTimeMs()); ret.setTtl(mTtl);
ret.setPinned(isPinned()); ret.setCompleted(isCompleted()); ret.setPersisted(isPersisted()); ret.setBlockIds(getBlockIds()); ret.setLastModificationTimeMs(getLastModificationTimeMs());
/** * Converts a proto type to a wire type. * * @param pInfo the proto representation of a file information * @return wire representation of the file information */ public static FileInfo fromProto(alluxio.grpc.FileInfo pInfo) { return new FileInfo().setFileId(pInfo.getFileId()).setName(pInfo.getName()) .setPath(pInfo.getPath()).setUfsPath(pInfo.getUfsPath()).setLength(pInfo.getLength()) .setBlockSizeBytes(pInfo.getBlockSizeBytes()).setCreationTimeMs(pInfo.getCreationTimeMs()) .setCompleted(pInfo.getCompleted()).setFolder(pInfo.getFolder()) .setPinned(pInfo.getPinned()).setCacheable(pInfo.getCacheable()) .setPersisted(pInfo.getPersisted()).setBlockIds(pInfo.getBlockIdsList()) .setLastModificationTimeMs(pInfo.getLastModificationTimeMs()).setTtl(pInfo.getTtl()) .setTtlAction(pInfo.getTtlAction()).setOwner(pInfo.getOwner()) .setGroup(pInfo.getGroup()).setMode(pInfo.getMode()) .setPersistenceState(pInfo.getPersistenceState()).setMountPoint(pInfo.getMountPoint()) .setFileBlockInfos(map(GrpcUtils::fromProto, pInfo.getFileBlockInfosList())) .setMountId(pInfo.getMountId()).setInAlluxioPercentage(pInfo.getInAlluxioPercentage()) .setInMemoryPercentage(pInfo.getInMemoryPercentage()) .setUfsFingerprint(pInfo.hasUfsFingerprint() ? pInfo.getUfsFingerprint() : Constants.INVALID_UFS_FINGERPRINT) .setAcl(pInfo.hasAcl() ? (fromProto(pInfo.getAcl())) : AccessControlList.EMPTY_ACL) .setDefaultAcl( pInfo.hasDefaultAcl() ? ((DefaultAccessControlList) fromProto(pInfo.getDefaultAcl())) : DefaultAccessControlList.EMPTY_DEFAULT_ACL) .setReplicationMax(pInfo.getReplicationMax()).setReplicationMin(pInfo.getReplicationMin()); }
ret.setPinned(isPinned()); ret.setCacheable(false); ret.setPersisted(isPersisted()); ret.setLastModificationTimeMs(getLastModificationTimeMs()); ret.setTtl(mTtl);
@Override public FileInfo generateClientFileInfo(String path) { FileInfo ret = new FileInfo(); // note: in-Alluxio percentage is NOT calculated here, because it needs blocks info stored in // block master ret.setFileId(getId()); ret.setName(getName()); ret.setPath(path); ret.setLength(getLength()); ret.setBlockSizeBytes(getBlockSizeBytes()); ret.setCreationTimeMs(getCreationTimeMs()); ret.setCacheable(isCacheable()); ret.setFolder(isDirectory()); ret.setPinned(isPinned()); ret.setCompleted(isCompleted()); ret.setPersisted(isPersisted()); ret.setBlockIds(getBlockIds()); ret.setLastModificationTimeMs(getLastModificationTimeMs()); ret.setTtl(mTtl); ret.setTtlAction(mTtlAction); ret.setOwner(getOwner()); ret.setGroup(getGroup()); ret.setMode(getMode()); ret.setPersistenceState(getPersistenceState().toString()); ret.setMountPoint(false); ret.setUfsFingerprint(getUfsFingerprint()); return ret; }
.setPinned(info.isPinned()) .setCacheable(info.isCacheable()) .setPersisted(info.isPersisted()) .setBlockIds(info.getBlockIds()) .setInMemoryPercentage(info.getInMemoryPercentage())