private byte[] readBytes(final int length, final FSDataInputStream reader) throws StoreException { try { final byte[] bytes = new byte[length]; int bytesRead = 0; while (bytesRead < length && bytesRead > -1) { bytesRead += reader.read(bytes, bytesRead, length - bytesRead); } return bytes; } catch (final IOException e) { throw new StoreException(e.getMessage()); } }
private long getLatestSnapshot(final String rootDir) throws StoreException { long latestSnapshot = 0L; try { for (final FileStatus status : fs.listStatus(new Path(rootDir))) { final long currentSnapshot = Long.parseLong(status.getPath().getName()); if (latestSnapshot < currentSnapshot) { latestSnapshot = currentSnapshot; } } } catch (final IOException e) { throw new StoreException(e.getMessage()); } return latestSnapshot; }
@Override public Connector getConnection() throws StoreException { try { mockConnector = mockAccumulo.getConnector(AccumuloProperties.USER, PASSWORD_TOKEN); } catch (final AccumuloException | AccumuloSecurityException e) { throw new StoreException(e.getMessage(), e); } return mockConnector; }
public void writeColumns(final String group, final String rootDir, final FileSystem fs) throws StoreException { try { for (final Map.Entry<String, ColumnIndex> columnIndexEntry : columnToIndex.entrySet()) { final String column = columnIndexEntry.getKey(); final ColumnIndex colIndex = columnIndexEntry.getValue(); final String indexDir = ParquetStore.getGroupDirectory(group, column, rootDir) + "/"; final Path path = new Path(indexDir + ParquetStoreConstants.INDEX); colIndex.write(fs.create(path)); } } catch (final IOException e) { throw new StoreException(e.getMessage()); } }
ElementWithPropertiesRecordReader(final String keyPackageClass, final Schema schema, final View view) throws StoreException, SchemaException, SerialisationException, UnsupportedEncodingException { super(); final AccumuloKeyPackage keyPackage; try { keyPackage = Class.forName(keyPackageClass).asSubclass(AccumuloKeyPackage.class).newInstance(); } catch (final InstantiationException | IllegalAccessException | ClassNotFoundException e) { throw new StoreException("Unable to construct an instance of key package: " + keyPackageClass, e); } keyPackage.setSchema(schema); this.converter = keyPackage.getKeyConverter(); this.view = view; }
protected <O> O doGet(final URL url, final TypeReference<O> outputTypeReference, final Context context) throws StoreException { final Invocation.Builder request = createRequest(null, url, context); final Response response; try { response = request.get(); } catch (final Exception e) { throw new StoreException("Request failed to execute via url " + url.toExternalForm(), e); } return handleResponse(response, outputTypeReference); }
protected <O> O doPost(final URL url, final Object body, final TypeReference<O> outputType, final Context context) throws StoreException { try { return doPost(url, new String(JSONSerialiser.serialise(body), CommonConstants.UTF_8), outputType, context); } catch (final SerialisationException | UnsupportedEncodingException e) { throw new StoreException("Unable to serialise body of request into json.", e); } }
protected <O> O doPost(final URL url, final String jsonBody, final TypeReference<O> clazz, final Context context) throws StoreException { final Invocation.Builder request = createRequest(jsonBody, url, context); final Response response; try { response = request.post(Entity.json(jsonBody)); } catch (final Exception e) { throw new StoreException("Failed to execute post via " + "the Gaffer URL " + url.toExternalForm(), e); } return handleResponse(response, clazz); }
private byte[] serialiseObject(final Object value) throws StoreException { final String objectType = value.getClass().getSimpleName(); if ("Long".equals(objectType)) { return BytesUtils.longToBytes((long) value); } else if ("Integer".equals(objectType)) { return BytesUtils.intToBytes((int) value); } else if ("Boolean".equals(objectType)) { return BytesUtils.booleanToBytes((boolean) value); } else if ("Float".equals(objectType)) { return BytesUtils.intToBytes(Float.floatToIntBits((float) value)); } else if ("String".equals(objectType)) { return StringUtil.toBytes((String) value); } else if ("byte[]".equals(objectType)) { return (byte[]) value; } else { throw new StoreException("Cannot serialise objects of type " + objectType); } }
/** * Returns the {@link org.apache.accumulo.core.security.Authorizations} of * the current user * * @param connection the connection to an accumulo instance * @return The accumulo Authorisations of the current user specified in the properties file * @throws StoreException if the table could not be found or other table/security issues */ public static Authorizations getCurrentAuthorizations(final Connector connection) throws StoreException { try { return connection.securityOperations().getUserAuthorizations(connection.whoami()); } catch (final AccumuloException | AccumuloSecurityException e) { throw new StoreException(e.getMessage(), e); } }
private void loadIndex() throws StoreException { final String rootDir = getDataDir(); try { if (fs.exists(new Path(rootDir))) { graphIndex = new GraphIndex(); final long snapshot = getLatestSnapshot(rootDir); graphIndex.readGroups(schemaUtils, rootDir + "/" + snapshot, fs); graphIndex.setSnapshotTimestamp(snapshot); } } catch (final IOException e) { throw new StoreException(e.getMessage()); } }
public static void setLocalityGroups(final AccumuloStore store) throws StoreException { final String tableName = store.getTableName(); Map<String, Set<Text>> localityGroups = new HashMap<>(); for (final String group : store.getSchema().getGroups()) { HashSet<Text> localityGroup = new HashSet<>(); localityGroup.add(new Text(group)); localityGroups.put(group, localityGroup); } LOGGER.info("Setting locality groups on table {}", tableName); try { store.getConnection().tableOperations().setLocalityGroups(tableName, localityGroups); } catch (final AccumuloException | AccumuloSecurityException | TableNotFoundException e) { throw new StoreException(e.getMessage(), e); } }
public void readColumns(final String group, final String rootDir, final FileSystem fs, final String[] columns) throws StoreException { try { for (final String column : columns) { final String indexDir = ParquetStore.getGroupDirectory(group, column, rootDir) + "/"; final Path path = new Path(indexDir + ParquetStoreConstants.INDEX); if (fs.exists(path)) { final ColumnIndex colIndex = new ColumnIndex(); colIndex.read(fs.open(path)); if (colIndex.getIterator().hasNext()) { add(column, colIndex); } } } } catch (final IOException e) { throw new StoreException(e.getMessage()); } }
private void cleanUp() throws StoreException { String dataDir = ""; try { dataDir = getDataDir(); deleteFolder(dataDir, getFS()); } catch (final IOException e) { throw new StoreException("Exception deleting folder: " + dataDir, e); } }
/** * Creates a connection to an accumulo instance using the provided * parameters * * @param instanceName the instance name * @param zookeepers the zoo keepers * @param userName the user name * @param password the password * @return A connection to an accumulo instance * @throws StoreException failure to create an accumulo connection */ public static Connector getConnector(final String instanceName, final String zookeepers, final String userName, final String password) throws StoreException { final Instance instance = new ZooKeeperInstance(instanceName, zookeepers); try { return instance.getConnector(userName, new PasswordToken(password)); } catch (final AccumuloException | AccumuloSecurityException e) { throw new StoreException("Failed to create accumulo connection", e); } }
/** * This method takes a store and the name of an iterator to be * removed. * * @param store the accumulo store * @param iteratorName the name of the iterator update * @throws StoreException if any issues occur when updating the iterator */ public static void removeIterator(final AccumuloStore store, final String iteratorName) throws StoreException { try { if (store.getConnection().tableOperations().listIterators(store.getTableName()).containsKey(iteratorName)) { store.getConnection() .tableOperations() .removeIterator(store.getTableName(), iteratorName, EnumSet.of(IteratorScope.majc, IteratorScope.minc, IteratorScope.scan)); } } catch (final AccumuloSecurityException | AccumuloException | TableNotFoundException | StoreException e) { throw new StoreException("Unable remove iterator with Name: " + iteratorName, e); } }
/** * This method takes a store and the name of an iterator to be * updated. The store's configured * {@link uk.gov.gchq.gaffer.accumulostore.key.IteratorSettingFactory} factory will be * used to create the new iterator in the removed one's place * * @param store the accumulo store * @param iteratorName the name of the iterator update * @throws StoreException if any issues occur when updating the iterator */ public static void updateIterator(final AccumuloStore store, final String iteratorName) throws StoreException { try { updateIterator(store, iteratorName, store.getKeyPackage().getIteratorFactory().getIteratorSetting(store, iteratorName)); } catch (final IteratorSettingException e) { throw new StoreException(e.getMessage(), e); } }
/** * This method can be used to attach an iterator to the table in use by the * store instance. * * @param store the accumulo store * @param iteratorSetting the iterator setting to add. * @throws StoreException if any issues occur adding an iterator setting */ public static void addIterator(final AccumuloStore store, final IteratorSetting iteratorSetting) throws StoreException { if (null != iteratorSetting) { try { store.getConnection().tableOperations().attachIterator(store.getTableName(), iteratorSetting); } catch (final AccumuloSecurityException | AccumuloException | TableNotFoundException e) { throw new StoreException("Add iterator with Name: " + iteratorSetting.getName(), e); } } TableUtils.setLocalityGroups(store); }
/** * This should be used if a gaffer version upgrade causes the aggregator * iterator to be removed from a table * * @param store the accumulo store * @param iteratorName the iterator name * @throws StoreException if any issues occur adding an aggregator iterator */ public static void addIterator(final AccumuloStore store, final String iteratorName) throws StoreException { if ((!AccumuloStoreConstants.VALIDATOR_ITERATOR_NAME.equals(iteratorName) || store.getProperties().getEnableValidatorIterator()) && (store.getSchema().isAggregationEnabled())) { try { addIterator(store, store.getKeyPackage() .getIteratorFactory() .getIteratorSetting(store, iteratorName)); } catch (final IteratorSettingException e) { throw new StoreException(e.getMessage(), e); } } }
public void read(final FSDataInputStream reader) throws StoreException { try { while (reader.available() > 0) { final int numOfCols = reader.readInt(); Object[] min = new Object[numOfCols]; for (int i = 0; i < numOfCols; i++) { final int colTypeLength = reader.readInt(); final byte[] colType = readBytes(colTypeLength, reader); final int minLength = reader.readInt(); final byte[] minBytes = readBytes(minLength, reader); min[i] = deserialiseColumn(colType, minBytes); } final int filePathLength = reader.readInt(); final byte[] filePath = readBytes(filePathLength, reader); add(new MinValuesWithPath(min, StringUtil.toString(filePath))); } reader.close(); } catch (final IOException e) { throw new StoreException(e.getMessage()); } }