@Override public String getIdentifyingInformation(NotificationEventParamsDto notificationEventParams, BusinessObjectDataHelper businessObjectDataHelper) { if (notificationEventParams instanceof StorageUnitNotificationEventParamsDto) { StorageUnitNotificationEventParamsDto storageUnitNotificationEventParams = (StorageUnitNotificationEventParamsDto) notificationEventParams; return String.format("namespace: \"%s\", actionId: \"%s\" with " + businessObjectDataHelper.businessObjectDataKeyToString( businessObjectDataHelper.getBusinessObjectDataKey(storageUnitNotificationEventParams.getBusinessObjectData())) + ", storageName: \"%s\"", storageUnitNotificationEventParams.getStorageUnitNotificationRegistration().getNamespace().getCode(), storageUnitNotificationEventParams.getNotificationJobAction().getId(), storageUnitNotificationEventParams.getStorageName()); } else { throw new IllegalStateException( "Notification event parameters DTO passed to the method must be an instance of StorageUnitNotificationEventParamsDto."); } }
/** * Validates a list of storage file paths. This method makes sure that all storage file paths match the expected s3 key prefix value. * * @param storageFilePaths the storage file paths to be validated * @param s3KeyPrefix the S3 key prefix that storage file paths are expected to start with * @param businessObjectDataEntity the business object data entity * @param storageName the name of the storage that storage files are stored in */ public void validateStorageFilePaths(Collection<String> storageFilePaths, String s3KeyPrefix, BusinessObjectDataEntity businessObjectDataEntity, String storageName) { for (String storageFilePath : storageFilePaths) { Assert.isTrue(storageFilePath.startsWith(s3KeyPrefix), String .format("Storage file \"%s\" registered with business object data {%s} in \"%s\" storage does not match the expected S3 key prefix \"%s\".", storageFilePath, businessObjectDataHelper.businessObjectDataEntityAltKeyToString(businessObjectDataEntity), storageName, s3KeyPrefix)); } }
/** * Creates the business object data from the persisted entity. * * @param businessObjectDataEntity the newly persisted business object data entity. * * @return the business object data. */ public BusinessObjectData createBusinessObjectDataFromEntity(BusinessObjectDataEntity businessObjectDataEntity) { return createBusinessObjectDataFromEntity(businessObjectDataEntity, false, false); }
/** * Returns a list of primary and sub-partition values per specified business object data. * * @param businessObjectData the business object data * * @return the list of primary and sub-partition values */ public List<String> getPrimaryAndSubPartitionValues(BusinessObjectData businessObjectData) { return getPrimaryAndSubPartitionValues(getBusinessObjectDataKey(businessObjectData)); }
/** * Gets a list of matched partition filters per specified list of storage unit entities and a sample partition filter. * * @param storageUnitEntities the list of storage unit entities * @param samplePartitionFilter the sample partition filter * * @return the list of partition filters */ private List<List<String>> getPartitionFilters(List<StorageUnitEntity> storageUnitEntities, List<String> samplePartitionFilter) { List<List<String>> partitionFilters = new ArrayList<>(); for (StorageUnitEntity storageUnitEntity : storageUnitEntities) { BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(storageUnitEntity.getBusinessObjectData()); partitionFilters.add(businessObjectDataHelper.getPartitionFilter(businessObjectDataKey, samplePartitionFilter)); } return partitionFilters; }
@Override public void execute() throws Exception { BusinessObjectData businessObjectData = downloaderWebClient.preRegisterBusinessObjectData(uploaderInputManifestDto, StorageEntity.MANAGED_STORAGE, false); BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(businessObjectData); downloaderWebClient .addStorageFiles(businessObjectDataKey, uploaderInputManifestDto, getTestS3FileTransferRequestParamsDto(S3_TEST_PATH_V0 + "/"), StorageEntity.MANAGED_STORAGE); downloaderWebClient.updateBusinessObjectDataStatus(businessObjectDataKey, BusinessObjectDataStatusEntity.VALID); } });
.thenReturn(BDATA_FINAL_DESTROY_DELAY_IN_DAYS); when(businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey)).thenReturn(businessObjectDataEntity); when(businessObjectDataHelper.getDateFromString(PARTITION_VALUE)).thenReturn(primaryPartitionValueDate); when(herdDao.getCurrentTimestamp()).thenReturn(currentTimestamp); when(storageUnitDao.getStorageUnitsByStoragePlatformAndBusinessObjectData(StoragePlatformEntity.S3, businessObjectDataEntity)) when(businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity)).thenReturn(businessObjectDataKey); when(configurationHelper.getProperty(ConfigurationValue.S3_ENDPOINT)).thenReturn(S3_ENDPOINT); verify(businessObjectDataHelper).validateBusinessObjectDataKey(businessObjectDataKey, true, true); verify(configurationHelper).getRequiredProperty(ConfigurationValue.S3_OBJECT_DELETE_TAG_KEY); verify(configurationHelper).getRequiredProperty(ConfigurationValue.S3_OBJECT_DELETE_TAG_VALUE); verify(herdStringHelper).getConfigurationValueAsInteger(ConfigurationValue.BDATA_FINAL_DESTROY_DELAY_IN_DAYS); verify(businessObjectDataDaoHelper).getBusinessObjectDataEntity(businessObjectDataKey); verify(businessObjectDataHelper).getDateFromString(PARTITION_VALUE); verify(herdDao).getCurrentTimestamp(); verify(storageUnitDao).getStorageUnitsByStoragePlatformAndBusinessObjectData(StoragePlatformEntity.S3, businessObjectDataEntity); verify(storageUnitDaoHelper).updateStorageUnitStatus(storageUnitEntity, StorageUnitStatusEntity.DISABLING, StorageUnitStatusEntity.DISABLING); verify(businessObjectDataDaoHelper).updateBusinessObjectDataStatus(businessObjectDataEntity, BusinessObjectDataStatusEntity.DELETED); verify(businessObjectDataHelper).getBusinessObjectDataKey(businessObjectDataEntity); verify(configurationHelper).getProperty(ConfigurationValue.S3_ENDPOINT); verifyNoMoreInteractionsHelper();
@NamespacePermission(fields = "#businessObjectDataKey.namespace", permissions = NamespacePermissionEnum.READ) @Override public BusinessObjectDataVersions getBusinessObjectDataVersions(BusinessObjectDataKey businessObjectDataKey) { // Validate and trim the business object data key. businessObjectDataHelper.validateBusinessObjectDataKey(businessObjectDataKey, false, false); // Get the business object data versions based on the specified parameters. List<BusinessObjectDataEntity> businessObjectDataEntities = businessObjectDataDao.getBusinessObjectDataEntities(businessObjectDataKey); // Create the response. BusinessObjectDataVersions businessObjectDataVersions = new BusinessObjectDataVersions(); for (BusinessObjectDataEntity businessObjectDataEntity : businessObjectDataEntities) { BusinessObjectDataVersion businessObjectDataVersion = new BusinessObjectDataVersion(); BusinessObjectDataKey businessObjectDataVersionKey = businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity); businessObjectDataVersion.setBusinessObjectDataKey(businessObjectDataVersionKey); businessObjectDataVersion.setStatus(businessObjectDataEntity.getStatus().getCode()); businessObjectDataVersions.getBusinessObjectDataVersions().add(businessObjectDataVersion); } return businessObjectDataVersions; }
Date primaryPartitionValue = businessObjectDataHelper.getDateFromString(businessObjectDataEntity.getPartitionValue()); BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity); StoragePolicyKey storagePolicyKey = new StoragePolicyKey(storagePolicyEntity.getNamespace().getCode(), storagePolicyEntity.getName());
@Test public void testGetHivePartitionsMultiplePathsFound() { // Create a test business object data entity. BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper .createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, DATA_VERSION, true, BDATA_STATUS); List<SchemaColumn> autoDiscoverableSubPartitionColumns = getPartitionColumns(Arrays.asList("Column1", "column2")); List<String> partitionPaths = Arrays.asList("/COLUMN1=111/COLUMN2=222", "/column1=111/COLUMN2=222"); List<String> storageFilePaths = getStorageFilePaths(Arrays.asList(partitionPaths.get(0) + "/file.dat", partitionPaths.get(1) + "/file.dat")); try { hive13DdlGenerator .getHivePartitions(businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity), autoDiscoverableSubPartitionColumns, TEST_S3_KEY_PREFIX, storageFilePaths, businessObjectDataEntity, STORAGE_NAME); fail("Should throw an IllegalArgumentException when multiple locations exist for the same Hive partition."); } catch (IllegalArgumentException e) { assertEquals(String.format("Found two different locations for the same Hive partition. " + "Storage: {%s}, business object data: {%s}, S3 key prefix: {%s}, path[1]: {%s}, path[2]: {%s}", STORAGE_NAME, businessObjectDataHelper.businessObjectDataEntityAltKeyToString(businessObjectDataEntity), TEST_S3_KEY_PREFIX, partitionPaths.get(0), partitionPaths.get(1)), e.getMessage()); } }
when(mockBusinessObjectDataHelper.createBusinessObjectDataKeyFromStorageUnitKey(businessObjectDataStorageUnitKey)).thenReturn(businessObjectDataKey); when(mockBusinessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey)).thenReturn(businessObjectDataEntity); when(mockBusinessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity)).thenReturn(businessObjectDataKey); when(businessObjectDataEntity.getLatestVersion()).thenReturn(true); when(mockBusinessObjectDataDao.getBusinessObjectDataMaxVersion(businessObjectDataKey)).thenReturn(BUSINESS_OBJECT_DATA_MAX_VERSION); verify(mockBusinessObjectDataHelper).createBusinessObjectDataKeyFromStorageUnitKey(businessObjectDataStorageUnitKey); verify(mockBusinessObjectDataDaoHelper).getBusinessObjectDataEntity(businessObjectDataKey);
Date primaryPartitionValue = businessObjectDataHelper.getDateFromString(businessObjectDataEntity.getPartitionValue()); businessObjectDataEntity.getPartitionValue(), businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey))); "Business object data fails retention threshold check for retention type \"%s\" with retention period of %d days. " + "Business object data: {%s}", retentionType, retentionPeriodInDays, businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey))); businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey))); "Business object data fails retention threshold check for retention type \"%s\" with retention expiration date %s. " + "Business object data: {%s}", retentionType, businessObjectDataEntity.getRetentionExpiration(), businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey)));
/** * Validate that business object data status is supported by the storage policy feature. * * @param businessObjectDataEntity the business object data entity * @param businessObjectDataKey the business object data key */ private void validateBusinessObjectData(BusinessObjectDataEntity businessObjectDataEntity, BusinessObjectDataKey businessObjectDataKey) { Assert.isTrue(StoragePolicySelectorServiceImpl.SUPPORTED_BUSINESS_OBJECT_DATA_STATUSES.contains(businessObjectDataEntity.getStatus().getCode()), String .format("Business object data status \"%s\" is not supported by the storage policy feature. Business object data: {%s}", businessObjectDataEntity.getStatus().getCode(), businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey))); }
when(mockBusinessObjectDataHelper.createBusinessObjectDataKeyFromStorageUnitKey(businessObjectDataStorageUnitKey)).thenReturn(businessObjectDataKey); when(mockBusinessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey)).thenReturn(businessObjectDataEntity); when(businessObjectDataEntity.getStorageUnits()).thenReturn(storageUnitEntities); when(mockBusinessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey)) .thenReturn(businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey)); businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey) + "}"))); verify(mockBusinessObjectDataHelper).createBusinessObjectDataKeyFromStorageUnitKey(businessObjectDataStorageUnitKey); verify(mockBusinessObjectDataDaoHelper).getBusinessObjectDataEntity(businessObjectDataKey); verify(businessObjectDataEntity).getStorageUnits(); verify(mockBusinessObjectDataHelper).businessObjectDataKeyToString(businessObjectDataKey);
@NamespacePermission(fields = "#businessObjectDataKey.namespace", permissions = NamespacePermissionEnum.WRITE) @Override public BusinessObjectData updateBusinessObjectDataAttributes(BusinessObjectDataKey businessObjectDataKey, BusinessObjectDataAttributesUpdateRequest businessObjectDataAttributesUpdateRequest) { // Validate and trim the business object data key. businessObjectDataHelper.validateBusinessObjectDataKey(businessObjectDataKey, true, true); // Validate the update request. Assert.notNull(businessObjectDataAttributesUpdateRequest, "A business object data attributes update request must be specified."); Assert.notNull(businessObjectDataAttributesUpdateRequest.getAttributes(), "A list of business object data attributes must be specified."); List<Attribute> attributes = businessObjectDataAttributesUpdateRequest.getAttributes(); // Validate attributes. This is also going to trim the attribute names. attributeHelper.validateAttributes(attributes); // Retrieve the business object data and ensure it exists. BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey); // Validate attributes against attribute definitions. attributeDaoHelper.validateAttributesAgainstBusinessObjectDataAttributeDefinitions(attributes, businessObjectDataEntity.getBusinessObjectFormat().getAttributeDefinitions()); // Update the attributes. attributeDaoHelper.updateBusinessObjectDataAttributes(businessObjectDataEntity, attributes); // Persist and refresh the entity. businessObjectDataEntity = businessObjectDataDao.saveAndRefresh(businessObjectDataEntity); // Create and return the business object data object from the persisted entity. return businessObjectDataHelper.createBusinessObjectDataFromEntity(businessObjectDataEntity); }
@Test public void getStorageUnitByStorageNameStorageUnitNoExists() { String testStorageName = "I_DO_NOT_EXIST"; // Try to get a non-existing storage unit. try { businessObjectDataHelper.getStorageUnitByStorageName( new BusinessObjectData(INTEGER_VALUE, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_KEY, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, NO_STORAGE_UNITS, NO_ATTRIBUTES, NO_BUSINESS_OBJECT_DATA_PARENTS, NO_BUSINESS_OBJECT_DATA_CHILDREN, NO_BUSINESS_OBJECT_DATA_STATUS_HISTORY, NO_RETENTION_EXPIRATION_DATE), testStorageName); fail("Should throw a IllegalStateException when storage unit does not exist."); } catch (IllegalStateException e) { assertEquals(String.format("Business object data has no storage unit with storage name \"%s\".", testStorageName), e.getMessage()); } }
/** * Returns a string representation of the alternate key values for the business object data entity. * * @param businessObjectDataEntity the business object data entity * * @return the string representation of the alternate key values for the business object data entity */ public String businessObjectDataEntityAltKeyToString(BusinessObjectDataEntity businessObjectDataEntity) { BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectDataEntity.getBusinessObjectFormat(); return businessObjectDataKeyToString(businessObjectFormatEntity.getBusinessObjectDefinition().getNamespace().getCode(), businessObjectFormatEntity.getBusinessObjectDefinition().getName(), businessObjectFormatEntity.getUsage(), businessObjectFormatEntity.getFileType().getCode(), businessObjectFormatEntity.getBusinessObjectFormatVersion(), businessObjectDataEntity.getPartitionValue(), getSubPartitionValues(businessObjectDataEntity), businessObjectDataEntity.getVersion()); }
.format("Storage unit status is \"%s\", but must be \"%s\". Storage: {%s}, business object data: {%s}", storageUnitEntity.getStatus().getCode(), StorageUnitStatusEntity.DISABLING, businessObjectDataDestroyDto.getStorageName(), businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey))); return businessObjectDataHelper.createBusinessObjectDataFromEntity(businessObjectDataEntity);
jsonHelper.objectToJson(businessObjectDataHelper.createBusinessObjectDataKeyFromStorageUnitKey(storageUnitKey)), runtimeException);
/** * Fires business object data status changed notifications. * * @param businessObjectDataEntities list of business object data that were created. */ private void processBusinessObjectDataStatusChangeNotificationEvents(List<BusinessObjectDataEntity> businessObjectDataEntities) { // Convert entities to key object List<BusinessObjectDataKey> registeredBusinessObjectDataKeys = new ArrayList<>(); for (BusinessObjectDataEntity businessObjectDataEntity : businessObjectDataEntities) { BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.createBusinessObjectDataKeyFromEntity(businessObjectDataEntity); registeredBusinessObjectDataKeys.add(businessObjectDataKey); } // Fire notifications on the keys for (BusinessObjectDataKey businessObjectDataKey : registeredBusinessObjectDataKeys) { sqsNotificationEventService.processBusinessObjectDataStatusChangeNotificationEvent(businessObjectDataKey, UNREGISTERED_STATUS, null); } }