/** * Validate downloaded S3 files per storage unit information. * * @param baseDirectory the local parent directory path, relative to which the files are expected to be located * @param s3KeyPrefix the S3 key prefix that was prepended to the S3 file paths, when they were uploaded to S3 * @param storageUnit the storage unit that contains a list of storage files to be validated * * @throws IllegalStateException if files are not valid */ public void validateDownloadedS3Files(String baseDirectory, String s3KeyPrefix, StorageUnit storageUnit) throws IllegalStateException { validateDownloadedS3Files(baseDirectory, s3KeyPrefix, storageUnit.getStorageFiles()); }
/** * Validate downloaded S3 files per storage unit information. * * @param baseDirectory the local parent directory path, relative to which the files are expected to be located * @param s3KeyPrefix the S3 key prefix that was prepended to the S3 file paths, when they were uploaded to S3 * @param storageUnit the storage unit that contains a list of storage files to be validated * * @throws IllegalStateException if files are not valid */ public void validateDownloadedS3Files(String baseDirectory, String s3KeyPrefix, StorageUnit storageUnit) throws IllegalStateException { validateDownloadedS3Files(baseDirectory, s3KeyPrefix, storageUnit.getStorageFiles()); }
if (!CollectionUtils.isEmpty(storageUnit.getStorageFiles())) for (StorageFile storageFile : storageUnit.getStorageFiles())
@Test public void testValidateDownloadedS3FilesFileNoExists() throws IOException { File targetLocalDirectory = Paths.get(LOCAL_TEMP_PATH.toString(), TEST_S3_KEY_PREFIX).toFile(); createLocalFile(targetLocalDirectory.getPath(), "ACTUAL_FILE", FILE_SIZE_1_KB); StorageUnit storageUnit = createStorageUnit(TEST_S3_KEY_PREFIX, Collections.singletonList("EXPECTED_FILE"), FILE_SIZE_1_KB); // Try to validate non-existing local files. try { storageFileHelper.validateDownloadedS3Files(LOCAL_TEMP_PATH.toString(), TEST_S3_KEY_PREFIX, storageUnit); fail("Should throw a RuntimeException when actual local files do not exist."); } catch (RuntimeException e) { String expectedErrMsg = String.format("Downloaded \"%s\" file doesn't exist.", Paths.get(LOCAL_TEMP_PATH.toString(), storageUnit.getStorageFiles().get(0).getFilePath()).toFile().getPath()); assertEquals(expectedErrMsg, e.getMessage()); } }
@Test public void testValidateDownloadedS3FilesFileSizeMismatch() throws IOException { File targetLocalDirectory = Paths.get(LOCAL_TEMP_PATH.toString(), TEST_S3_KEY_PREFIX).toFile(); createLocalFiles(targetLocalDirectory.getPath(), FILE_SIZE_1_KB); StorageUnit storageUnit = createStorageUnit(TEST_S3_KEY_PREFIX, LOCAL_FILES, FILE_SIZE_1_KB * 2); // Try to validate the local files, when actual file sizes do not not match to the storage unit information. try { storageFileHelper.validateDownloadedS3Files(LOCAL_TEMP_PATH.toString(), TEST_S3_KEY_PREFIX, storageUnit); fail("Should throw a RuntimeException when actual file sizes do not not match to the storage unit information."); } catch (RuntimeException e) { String expectedErrMsg = String .format("Size of the downloaded \"%s\" S3 file does not match the expected value (expected %d bytes, actual %d bytes).", Paths.get(LOCAL_TEMP_PATH.toString(), storageUnit.getStorageFiles().get(0).getFilePath()).toFile().getPath(), FILE_SIZE_1_KB * 2, FILE_SIZE_1_KB); assertEquals(expectedErrMsg, e.getMessage()); } }
@Test public void testValidateDownloadedS3FilesFileCountMismatch() throws IOException { File targetLocalDirectory = Paths.get(LOCAL_TEMP_PATH.toString(), TEST_S3_KEY_PREFIX).toFile(); createLocalFiles(targetLocalDirectory.getPath(), FILE_SIZE_1_KB); createLocalFile(targetLocalDirectory.getPath(), "EXTRA_FILE", FILE_SIZE_1_KB); StorageUnit storageUnit = createStorageUnit(TEST_S3_KEY_PREFIX, LOCAL_FILES, FILE_SIZE_1_KB); // Try to validate the local files, when number of files does not match to the storage unit information. try { storageFileHelper.validateDownloadedS3Files(LOCAL_TEMP_PATH.toString(), TEST_S3_KEY_PREFIX, storageUnit); fail("Should throw a RuntimeException when number of local files does not match to the storage unit information."); } catch (RuntimeException e) { String expectedErrMsg = String .format("Number of downloaded files does not match the storage unit information (expected %d files, actual %d files).", storageUnit.getStorageFiles().size(), LOCAL_FILES.size() + 1); assertEquals(expectedErrMsg, e.getMessage()); } }
@Test public void testValidateStorageUnitS3FilesS3KeyPrefixMismatch() throws IOException { StorageUnit storageUnit = createStorageUnit("SOME_S3_KEY_PREFIX", LOCAL_FILES, FILE_SIZE_1_KB); // Try to validate S3 files when we have not registered S3 file. try { storageFileHelper.validateStorageUnitS3Files(storageUnit, new ArrayList<>(), TEST_S3_KEY_PREFIX); fail("Should throw a RuntimeException when registered S3 file does match S3 key prefix."); } catch (RuntimeException e) { String expectedErrMsg = String.format("Storage file S3 key prefix \"%s\" does not match the expected S3 key prefix \"%s\".", storageUnit.getStorageFiles().get(0).getFilePath(), TEST_S3_KEY_PREFIX); assertEquals(expectedErrMsg, e.getMessage()); } }
@Test public void testValidateStorageUnitS3FilesRegisteredFileNoExists() throws IOException { StorageUnit storageUnit = createStorageUnit(TEST_S3_KEY_PREFIX, LOCAL_FILES, FILE_SIZE_1_KB); List<String> actualS3Files = new ArrayList<>(); // Try to validate S3 files when actual S3 files do not exist. try { storageFileHelper.validateStorageUnitS3Files(storageUnit, actualS3Files, TEST_S3_KEY_PREFIX); fail("Should throw a RuntimeException when actual S3 files do not exist."); } catch (RuntimeException e) { String expectedErrMsg = String .format("Registered file \"%s\" does not exist in \"%s\" storage.", storageUnit.getStorageFiles().get(0).getFilePath(), storageUnit.getStorage().getName()); assertEquals(expectedErrMsg, e.getMessage()); } }
/** * Validates a download single initiation response as compared to the upload initiation response. * * @param uploadSingleInitiationResponse the upload single initiation response. * @param downloadSingleInitiationResponse the download single initiation response. */ public void validateDownloadSingleInitiationResponse(UploadSingleInitiationResponse uploadSingleInitiationResponse, DownloadSingleInitiationResponse downloadSingleInitiationResponse) { BusinessObjectData targetBusinessObjectData = uploadSingleInitiationResponse.getTargetBusinessObjectData(); validateDownloadSingleInitiationResponse(targetBusinessObjectData.getNamespace(), targetBusinessObjectData.getBusinessObjectDefinitionName(), targetBusinessObjectData.getBusinessObjectFormatUsage(), targetBusinessObjectData.getBusinessObjectFormatFileType(), targetBusinessObjectData.getBusinessObjectFormatVersion(), targetBusinessObjectData.getAttributes(), targetBusinessObjectData.getStorageUnits().get(0).getStorageFiles().get(0).getFileSizeBytes(), downloadSingleInitiationResponse); }
@Test public void testS3Message() throws Exception { setLogLevel(UploadDownloadHelperServiceImpl.class, LogLevel.OFF); uploadDownloadServiceTestHelper.createDatabaseEntitiesForUploadDownloadTesting(); UploadSingleInitiationResponse resultUploadSingleInitiationResponse = uploadDownloadService.initiateUploadSingle(uploadDownloadServiceTestHelper.createUploadSingleInitiationRequest()); String filePath = resultUploadSingleInitiationResponse.getSourceBusinessObjectData().getStorageUnits().get(0).getStorageFiles().get(0).getFilePath(); S3Entity s3Entity = new S3Entity(null, null, new S3ObjectEntity(filePath, 0L, null, null), null); List<S3EventNotificationRecord> records = new ArrayList<>(); records.add(new S3EventNotificationRecord(null, null, null, null, null, null, null, s3Entity, null)); S3EventNotification s3EventNotification = new S3EventNotification(records); setLogLevel(UploadDownloadServiceImpl.class, LogLevel.OFF); setLogLevel(HerdJmsMessageListener.class, LogLevel.DEBUG); herdJmsMessageListener.processMessage(jsonHelper.objectToJson(s3EventNotification), null); }
expectedNamespaceCode.trim().toLowerCase().replace('_', '-'), businessObjectData.getPartitionValue()); assertEquals(expectedStorageDirectoryPath, storageUnit.getStorageDirectory().getDirectoryPath()); assertEquals(1, storageUnit.getStorageFiles().size()); StorageFile storageFile = storageUnit.getStorageFiles().get(0); String expectedStorageFilePath = String.format("%s/%s", expectedStorageDirectoryPath, expectedFileName); assertEquals(expectedStorageFilePath, storageFile.getFilePath());
@Test public void testS3MessageS3FileNoExists() throws Exception { setLogLevel(UploadDownloadHelperServiceImpl.class, LogLevel.OFF); uploadDownloadServiceTestHelper.createDatabaseEntitiesForUploadDownloadTesting(); UploadSingleInitiationResponse resultUploadSingleInitiationResponse = uploadDownloadService.initiateUploadSingle(uploadDownloadServiceTestHelper .createUploadSingleInitiationRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, NAMESPACE, BDEF_NAME_2, FORMAT_USAGE_CODE_2, FORMAT_FILE_TYPE_CODE_2, FORMAT_VERSION_2, TARGET_S3_KEY)); String filePath = resultUploadSingleInitiationResponse.getSourceBusinessObjectData().getStorageUnits().get(0).getStorageFiles().get(0).getFilePath(); S3Entity s3Entity = new S3Entity(null, null, new S3ObjectEntity(filePath, 0L, null, null), null); List<S3EventNotificationRecord> records = new ArrayList<>(); records.add(new S3EventNotificationRecord(null, null, null, null, null, null, null, s3Entity, null)); S3EventNotification s3EventNotification = new S3EventNotification(records); setLogLevel(UploadDownloadServiceImpl.class, LogLevel.OFF); setLogLevel(HerdJmsMessageListener.class, LogLevel.OFF); // Try to process an S3 JMS message, when source S3 file does not exist. herdJmsMessageListener.processMessage(jsonHelper.objectToJson(s3EventNotification), null); }
theStorageFiles = ((this.storageFiles!= null)?this.getStorageFiles():null); strategy.appendField(locator, this, "storageFiles", buffer, theStorageFiles, (this.storageFiles!= null));
@Test public void testPerformCompleteUploadSingleMessageS3FileNoExists() { uploadDownloadServiceTestHelper.createDatabaseEntitiesForUploadDownloadTesting(); UploadSingleInitiationResponse resultUploadSingleInitiationResponse = uploadDownloadService.initiateUploadSingle(uploadDownloadServiceTestHelper .createUploadSingleInitiationRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, NAMESPACE, BDEF_NAME_2, FORMAT_USAGE_CODE_2, FORMAT_FILE_TYPE_CODE_2, FORMAT_VERSION_2, FILE_NAME)); String filePath = resultUploadSingleInitiationResponse.getTargetBusinessObjectData().getStorageUnits().get(0).getStorageFiles().get(0).getFilePath(); // Try to complete the upload, when source S3 file does not exist. UploadDownloadServiceImpl.CompleteUploadSingleMessageResult result = uploadDownloadService.performCompleteUploadSingleMessage(filePath); assertEquals(BusinessObjectDataStatusEntity.UPLOADING, result.getSourceOldBusinessObjectDataStatus()); assertEquals(BusinessObjectDataStatusEntity.DELETED, result.getSourceNewBusinessObjectDataStatus()); assertEquals(BusinessObjectDataStatusEntity.UPLOADING, result.getTargetOldBusinessObjectDataStatus()); assertEquals(BusinessObjectDataStatusEntity.INVALID, result.getTargetNewBusinessObjectDataStatus()); }
/** * Validates actualBusinessObjectData contents against specified arguments and expected (hard coded) test values. * * @param expectedDataVersion the expected business object data version * @param expectedAttributes the expected attributes * @param expectedParents the expected business object data parents * @param actualBusinessObjectData the BusinessObjectData object instance to be validated */ protected void assertBusinessObjectData(Integer expectedDataVersion, List<Attribute> expectedAttributes, List<BusinessObjectDataKey> expectedParents, BusinessObjectData actualBusinessObjectData) { assertNotNull(actualBusinessObjectData); assertEquals(TEST_BUSINESS_OBJECT_DEFINITION, actualBusinessObjectData.getBusinessObjectDefinitionName()); assertEquals(TEST_BUSINESS_OBJECT_FORMAT_USAGE, actualBusinessObjectData.getBusinessObjectFormatUsage()); assertEquals(TEST_BUSINESS_OBJECT_FORMAT_FILE_TYPE, actualBusinessObjectData.getBusinessObjectFormatFileType()); assertEquals(TEST_BUSINESS_OBJECT_FORMAT_VERSION.intValue(), actualBusinessObjectData.getBusinessObjectFormatVersion()); assertEquals(TEST_BUSINESS_OBJECT_FORMAT_PARTITION_KEY, actualBusinessObjectData.getPartitionKey()); assertEquals(TEST_PARTITION_VALUE, actualBusinessObjectData.getPartitionValue()); assertEquals(expectedDataVersion.intValue(), actualBusinessObjectData.getVersion()); assertEquals(1, actualBusinessObjectData.getStorageUnits().size()); assertEquals(StorageEntity.MANAGED_STORAGE, actualBusinessObjectData.getStorageUnits().get(0).getStorage().getName()); assertEquals(testManifestFiles.size(), actualBusinessObjectData.getStorageUnits().get(0).getStorageFiles().size()); assertEquals(expectedAttributes, actualBusinessObjectData.getAttributes()); assertEquals(expectedParents, actualBusinessObjectData.getBusinessObjectDataParents()); }
theStorageFiles = ((this.storageFiles!= null)?this.getStorageFiles():null); strategy.appendField(locator, this, "storageFiles", buffer, theStorageFiles, (this.storageFiles!= null));
/** * Creates the appropriate business object data entries for an upload. * * @param businessObjectDataStatusCode the target business object data status. * * @return the upload single initiation response created during the upload flow. */ public UploadSingleInitiationResponse createUploadedFileData(String businessObjectDataStatusCode) { loggingHelper.setLogLevel(UploadDownloadHelperServiceImpl.class, LogLevel.OFF); // Create source and target business object formats database entities which are required to initiate an upload. createDatabaseEntitiesForUploadDownloadTesting(); // Initiate a file upload. UploadSingleInitiationResponse resultUploadSingleInitiationResponse = uploadDownloadService.initiateUploadSingle(createUploadSingleInitiationRequest()); // Complete the upload. uploadDownloadService.performCompleteUploadSingleMessage( resultUploadSingleInitiationResponse.getSourceBusinessObjectData().getStorageUnits().get(0).getStorageFiles().get(0).getFilePath()); // Update the target business object data status to valid. Normally this would happen as part of the completion request, but since the status update // happens asynchronously, this will not happen within a unit test context which is why we are setting it explicitly. businessObjectDataDao.getBusinessObjectDataByAltKey( businessObjectDataHelper.getBusinessObjectDataKey(resultUploadSingleInitiationResponse.getTargetBusinessObjectData())) .setStatus(businessObjectDataStatusDao.getBusinessObjectDataStatusByCode(businessObjectDataStatusCode)); resultUploadSingleInitiationResponse.getTargetBusinessObjectData().setStatus(businessObjectDataStatusCode); // Return the initiate upload single response. return resultUploadSingleInitiationResponse; }
@Test public void testPerformCompleteUploadSingleMessageSourceBusinessObjectDataStatusNotUploading() { uploadDownloadServiceTestHelper.createDatabaseEntitiesForUploadDownloadTesting(); UploadSingleInitiationResponse resultUploadSingleInitiationResponse = uploadDownloadService.initiateUploadSingle(uploadDownloadServiceTestHelper .createUploadSingleInitiationRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, NAMESPACE, BDEF_NAME_2, FORMAT_USAGE_CODE_2, FORMAT_FILE_TYPE_CODE_2, FORMAT_VERSION_2, FILE_NAME)); String filePath = resultUploadSingleInitiationResponse.getTargetBusinessObjectData().getStorageUnits().get(0).getStorageFiles().get(0).getFilePath(); // Create a business object data status. businessObjectDataStatusDaoTestHelper.createBusinessObjectDataStatusEntity(BDATA_STATUS); // Update the status of the source business object data so it would not be "UPLOADING". businessObjectDataStatusService.updateBusinessObjectDataStatus( businessObjectDataHelper.getBusinessObjectDataKey(resultUploadSingleInitiationResponse.getSourceBusinessObjectData()), new BusinessObjectDataStatusUpdateRequest(BDATA_STATUS)); // Try to complete the upload, when source business object data status is not "UPLOADING". UploadDownloadServiceImpl.CompleteUploadSingleMessageResult result = uploadDownloadService.performCompleteUploadSingleMessage(filePath); assertEquals(BDATA_STATUS, result.getSourceOldBusinessObjectDataStatus()); assertNull(result.getSourceNewBusinessObjectDataStatus()); assertEquals(BusinessObjectDataStatusEntity.UPLOADING, result.getTargetOldBusinessObjectDataStatus()); assertNull(result.getTargetNewBusinessObjectDataStatus()); }
@Test public void testPerformCompleteUploadSingleMessageTargetBusinessObjectDataStatusNotUploading() { uploadDownloadServiceTestHelper.createDatabaseEntitiesForUploadDownloadTesting(); UploadSingleInitiationResponse resultUploadSingleInitiationResponse = uploadDownloadService.initiateUploadSingle(uploadDownloadServiceTestHelper .createUploadSingleInitiationRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, NAMESPACE, BDEF_NAME_2, FORMAT_USAGE_CODE_2, FORMAT_FILE_TYPE_CODE_2, FORMAT_VERSION_2, FILE_NAME)); String filePath = resultUploadSingleInitiationResponse.getTargetBusinessObjectData().getStorageUnits().get(0).getStorageFiles().get(0).getFilePath(); // Create a business object data status. businessObjectDataStatusDaoTestHelper.createBusinessObjectDataStatusEntity(BDATA_STATUS); // Update the status of the target business object data so it would not be "UPLOADING". businessObjectDataStatusService.updateBusinessObjectDataStatus( businessObjectDataHelper.getBusinessObjectDataKey(resultUploadSingleInitiationResponse.getTargetBusinessObjectData()), new BusinessObjectDataStatusUpdateRequest(BDATA_STATUS)); // Try to complete the upload, when target business object data status is not "UPLOADING". UploadDownloadServiceImpl.CompleteUploadSingleMessageResult result = uploadDownloadService.performCompleteUploadSingleMessage(filePath); assertEquals(BusinessObjectDataStatusEntity.UPLOADING, result.getSourceOldBusinessObjectDataStatus()); assertNull(result.getSourceNewBusinessObjectDataStatus()); assertEquals(BDATA_STATUS, result.getTargetOldBusinessObjectDataStatus()); assertNull(result.getTargetNewBusinessObjectDataStatus()); }
String filePath = resultUploadSingleInitiationResponse.getTargetBusinessObjectData().getStorageUnits().get(0).getStorageFiles().get(0).getFilePath();