/** * Prevents the use of data store references containing the backslash character. * * @param reference the data store reference to be validated * @return the supplied reference, if valid * @throws DataStoreException if the supplied reference is invalid */ private String validateReference(final String reference) throws DataStoreException { if (reference.contains("\\")) { throw new DataStoreException("Invalid reference - contains the backslash character"); } return reference; }
/** * Return the byte size of the data identified by the reference. * * @param reference a complete reference to be interpreted by the DataStore implementation * @return the byte size * @throws DataStoreException */ @Override public long size(String reference) throws DataStoreException { if (!dataMap.containsKey(reference)) { throw new DataStoreException("Failed to retrieve data, the asset does not exist."); } return dataMap.get(reference).length; }
/** * Provide a stream to get data identified by the reference. * * @param reference a complete reference to be interpreted by the DataStore implementation * @return * @throws DataStoreException */ @Override public InputStream retrieve(String reference) throws DataStoreException { numRetrieveRequests.incrementAndGet(); if (!dataMap.containsKey(reference)) { errors.incrementAndGet(); throw new DataStoreException("Failed to retrieve data, the asset does not exist."); } return new ByteArrayInputStream(dataMap.get(reference)); }
/** * Prevent a caller trying to "break out" of the root dataStorePath by performing a full path resolution of the reference. * * @param reference the data store reference relative to dataStorePath to resolve * @return the resolved path, if valid * @throws DataStoreException if the reference is invalid or cannot be resolved * @throws InvalidPathException if the reference cannot be converted to a Path */ private Path verifyReference(final String reference) throws DataStoreException { Path p = dataStorePath.resolve(reference).normalize(); if (!p.startsWith(dataStorePath)) { throw new DataStoreException("Invalid reference"); } return p; }
/** * Check there are 2 components to this name and that they are both valid UUIDs. * @throws DataStoreException if this is not a valid CafStoreReference */ private void validate() throws DataStoreException { if ( size() != 2 ) { throw new DataStoreException("Invalid reference, must consist of container id and asset id only"); } for(String component:this){ Matcher matcher = pattern.matcher(component); if(!matcher.find()){ throw new DataStoreException("Invalid reference due to invalid characters"); } } } }
/** * @throws DataStoreException if the reference is found but cannot be accessed or retrieved * @throws InvalidPathException if the reference cannot be converted to a Path */ @Override public String store(byte[] data, String partialReference) throws DataStoreException { try { Path ref = getStoreReference(partialReference); Files.write(ref, data); return dataStorePath.relativize(ref).toString().replace('\\', '/'); } catch (IOException e) { errors.incrementAndGet(); throw new DataStoreException("Failed to get output stream for store", e); } }
/** * Store data from a stream in the key given by the partial reference. * * @param dataStream the stream of data which will be read and stored in the in memory data store. * @param partialReference the partial reference, which the data will be stored relative to. * @return absolute reference to the stored data which can be used to retrieve. * @throws DataStoreException if the data store cannot service the request. */ @Override public String store(InputStream dataStream, String partialReference) throws DataStoreException { try { return store(IOUtils.toByteArray(dataStream), partialReference); } catch (IOException e) { errors.incrementAndGet(); throw new DataStoreException("Could not store input stream.", e); } }
/** * @throws DataStoreException if the reference is found but cannot be accessed or retrieved * @throws InvalidPathException if the reference cannot be converted to a Path */ @Override public String store(final InputStream dataStream, final String partialReference) throws DataStoreException { try { Path ref = getStoreReference(partialReference); Files.copy(dataStream, ref); return dataStorePath.relativize(ref).toString().replace('\\', '/'); } catch (IOException e) { errors.incrementAndGet(); throw new DataStoreException("Failed to get output stream for store", e); } }
/** * Delete a file. * * @param reference the file to be deleted. * @throws DataStoreException if the reference cannot be accessed or deleted */ @Override public void delete(String reference) throws DataStoreException { Objects.requireNonNull(reference); try { numDx.incrementAndGet(); LOG.debug("Deleting {}", reference); Path path = FileSystems.getDefault().getPath(dataStorePath.toString(), reference); Files.delete(path); } catch (IOException | SecurityException | InvalidPathException e) { errors.incrementAndGet(); throw new DataStoreException("Failed to delete reference", e); } }
/** * @throws DataStoreException if the reference is found but cannot be accessed or retrieved * @throws InvalidPathException if the reference cannot be converted to a Path */ @Override public String store(Path dataPath, String partialReference) throws DataStoreException { try { Path ref = getStoreReference(partialReference); Files.copy(dataPath, ref); return dataStorePath.relativize(ref).toString().replace('\\', '/'); } catch (IOException e) { errors.incrementAndGet(); throw new DataStoreException("Failed to get output stream for store", e); } }
@Override public OutputStream store(final String partialReference, final Consumer<String> setReferenceFunction) throws DataStoreException { Objects.requireNonNull(setReferenceFunction); try { final Path ref = getStoreReference(partialReference); final String reference = dataStorePath.relativize(ref).toString().replace('\\', '/'); final OutputStream fos = Files.newOutputStream(ref); final BufferedOutputStream bos = new BufferedOutputStream(fos, outputBufferSize); return new ProxyOutputStream(bos) { @Override public void close() throws IOException { super.close(); setReferenceFunction.accept(reference); } }; } catch (final IOException e) { errors.incrementAndGet(); throw new DataStoreException("Failed to get output stream for store", e); } }
/** * Store data from a local file in the key given by the partial reference. * * @param dataPath path to a file on the local filesystem to store in the in-memory data store. * @param partialReference the partial reference, which the data will be stored relative to. * @return absolute reference to the stored data which can be used to retrieve. * @throws DataStoreException if the data store cannot service the request. */ @Override public String store(Path dataPath, String partialReference) throws DataStoreException { try { return store(Files.readAllBytes(dataPath), partialReference); } catch (IOException e) { errors.incrementAndGet(); throw new DataStoreException("Could not store file path.", e); } }
/** * Determine the directory for the data store, and create it if necessary. */ public FileSystemDataStore(final FileSystemDataStoreConfiguration config) throws DataStoreException { dataStorePath = FileSystems.getDefault().getPath(config.getDataDir()); if (!Files.exists(dataStorePath)) { try { Files.createDirectory(dataStorePath); } catch (IOException e) { throw new DataStoreException("Cannot create data store directory", e); } } outputBufferSize = getOutputBufferSize(config); LOG.debug("Initialised"); }
/** * @throws ReferenceNotFoundException if the requested reference does not exist in the file system * @throws DataStoreException if the reference is found but cannot be accessed or retrieved * @throws InvalidPathException if the reference cannot be converted to a Path */ @Override public long size(final String reference) throws DataStoreException { Objects.requireNonNull(reference); try { return Files.size(checkReferenceExists(verifyReference(reference))); } catch (IOException e) { errors.incrementAndGet(); throw new DataStoreException("Failed to get data size", e); } }
/** * {@inheritDoc} Read a file from disk in the data directory. * * @throws ReferenceNotFoundException if the requested reference does not exist in the file system * @throws DataStoreException if the reference is found but cannot be accessed or retrieved * @throws InvalidPathException if the reference cannot be converted to a Path */ @Override public InputStream retrieve(final String reference) throws DataStoreException { Objects.requireNonNull(reference); try { numRx.incrementAndGet(); LOG.debug("Requesting {}", reference); return Files.newInputStream(checkReferenceExists(verifyReference(reference))); } catch (IOException e) { errors.incrementAndGet(); throw new DataStoreException("Failed to retrieve data", e); } }
@Override public final ManagedDataStore getDataStore(final ConfigurationSource configurationSource) throws DataStoreException { try { return new FileSystemDataStore(configurationSource.getConfiguration(FileSystemDataStoreConfiguration.class)); } catch (ConfigurationException e) { throw new DataStoreException("Cannot create data store", e); } } }
@Override public String store(InputStream inputStream, String partialReference) throws DataStoreException { try ( FileBackedOutputStream fileBackedOutputStream = new FileBackedOutputStream(FILE_THRESHOLD, true) ) { try { ByteStreams.copy(inputStream, fileBackedOutputStream); return store(fileBackedOutputStream.asByteSource(), partialReference); } finally { fileBackedOutputStream.reset(); } } catch (IOException ex) { errors.incrementAndGet(); throw new DataStoreException("Could not store input stream.", ex); } }
private static String getDataStoreReferenceForFile(FileObject file) throws DataStoreException, IOException { try{ File f = new File(file.getName().getPath()); return dataStore.store(f.toPath(), dataDir); } catch(Exception e){ throw new DataStoreException("Failure storing file. File path: "+file.getName().getPath(), e); } }
@Override public final ManagedDataStore getDataStore(final ConfigurationSource configurationSource) throws DataStoreException { try { StorageServiceDataStoreConfiguration storageServiceDataStoreConfiguration = configurationSource.getConfiguration(StorageServiceDataStoreConfiguration.class); KeycloakClient keycloakClient = storageServiceDataStoreConfiguration.getAuthenticationConfiguration() != null ? new KeycloakClient(storageServiceDataStoreConfiguration.getAuthenticationConfiguration()) : null; return new StorageServiceDataStore(configurationSource.getConfiguration(StorageServiceDataStoreConfiguration.class), keycloakClient); } catch (ConfigurationException e) { throw new DataStoreException("Cannot create object data store", e); } } }
@Override public long size(String reference) throws DataStoreException { LOG.debug("Received size request for {}", reference); // Parse incoming reference. Extract container/asset identifiers as well as any delegation ticket provided. ReferenceComponents refComponents = ReferenceComponents.parseReference(reference); reference = refComponents.getReference(); String delegationTicket = refComponents.getNamedValue(DELEGATION_TICKET_NAMED_PARAMETER); CafStoreReference ref = new CafStoreReference(reference); try { GetAssetMetadataRequest getAssetMetadataRequest = new GetAssetMetadataRequest(accessToken, ref.getContainer(), ref.getAsset()); if (delegationTicket != null) { getAssetMetadataRequest.setDelegationTicket(delegationTicket); } return callStorageService(c -> c.getAssetMetadata(getAssetMetadataRequest)).getSize(); } catch (IOException | StorageClientException | StorageServiceException | StorageServiceConnectException e) { errors.incrementAndGet(); throw new DataStoreException("Failed to get data size for reference " + reference, e); } }