private long getLatestSnapshot(final String rootDir) throws StoreException { long latestSnapshot = 0L; try { for (final FileStatus status : fs.listStatus(new Path(rootDir))) { final long currentSnapshot = Long.parseLong(status.getPath().getName()); if (latestSnapshot < currentSnapshot) { latestSnapshot = currentSnapshot; } } } catch (final IOException e) { throw new StoreException(e.getMessage()); } return latestSnapshot; }
private int getNumAccumuloSplits(final AccumuloStore store) { int numberTabletServers; try { numberTabletServers = store.getTabletServers().size(); LOGGER.debug("Number of region servers is {}", numberTabletServers); } catch (final StoreException e) { LOGGER.error("Exception thrown getting number of tablet servers: {}", e.getMessage()); throw new RuntimeException(e.getMessage(), e); } return numberTabletServers - 1; } }
private byte[] readBytes(final int length, final FSDataInputStream reader) throws StoreException { try { final byte[] bytes = new byte[length]; int bytesRead = 0; while (bytesRead < length && bytesRead > -1) { bytesRead += reader.read(bytes, bytesRead, length - bytesRead); } return bytes; } catch (final IOException e) { throw new StoreException(e.getMessage()); } }
@Override public int getExpectedNumberOfSplits(final Store store) { final AccumuloStore accumuloStore = (AccumuloStore) store; int numberTabletServers; try { numberTabletServers = accumuloStore.getTabletServers().size(); LOGGER.info("Number of tablet servers is {}", numberTabletServers); } catch (final StoreException e) { LOGGER.error("Exception thrown getting number of tablet servers: {}", e.getMessage()); throw new RuntimeException(e.getMessage(), e); } return numberTabletServers - 1; }
@Override public Connector getConnection() throws StoreException { try { mockConnector = mockAccumulo.getConnector(AccumuloProperties.USER, PASSWORD_TOKEN); } catch (final AccumuloException | AccumuloSecurityException e) { throw new StoreException(e.getMessage(), e); } return mockConnector; }
private void generateSplitsFromSampleData(final SampleDataForSplitPoints operation, final AccumuloStore store) throws OperationException { try { if (store.getTabletServers().size() < 2) { LOGGER.warn("There is only 1 tablet server so no split points will be calculated."); return; } } catch (final StoreException e) { throw new OperationException(e.getMessage(), e); } final SampleDataAndCreateSplitsFileTool sampleTool = new SampleDataAndCreateSplitsFileTool(new AccumuloSampleDataForSplitPointsJobFactory(), operation, store); try { ToolRunner.run(sampleTool, new String[0]); } catch (final Exception e) { throw new OperationException(e.getMessage(), e); } LOGGER.info("Finished calculating splits"); } }
public void writeColumns(final String group, final String rootDir, final FileSystem fs) throws StoreException { try { for (final Map.Entry<String, ColumnIndex> columnIndexEntry : columnToIndex.entrySet()) { final String column = columnIndexEntry.getKey(); final ColumnIndex colIndex = columnIndexEntry.getValue(); final String indexDir = ParquetStore.getGroupDirectory(group, column, rootDir) + "/"; final Path path = new Path(indexDir + ParquetStoreConstants.INDEX); colIndex.write(fs.create(path)); } } catch (final IOException e) { throw new StoreException(e.getMessage()); } }
@Override public Tuple4<String, String, ColumnIndex, OperationException> call() { try { final FileSystem fs = FileSystem.get(new Configuration()); if (fs.exists(new Path(directoryPath))) { final FileStatus[] files = fs.listStatus(new Path(directoryPath), path1 -> path1.getName().endsWith(".parquet")); for (final FileStatus file : files) { final String firstColumn = paths[0]; final Builder<String, Seq<String>> seqBuilder = Seq$.MODULE$.newBuilder(); final int numberOfColumns = paths.length; for (int i = 1; i < numberOfColumns; i++) { seqBuilder.$plus$eq(paths[i]); } final Dataset<Row> fileData = spark.read().parquet(file.getPath().toString()).select(firstColumn, seqBuilder.result()); try { final Row minRow = fileData.head(); columnIndex.add(generateGafferObjectsIndex(minRow, file.getPath().getName())); } catch (final NoSuchElementException ignored) { // ignore as dataframe was empty } } } } catch (final IOException e) { return new Tuple4<>(group, column, null, new OperationException("IOException generating the index files", e)); } catch (final StoreException e) { return new Tuple4<>(group, column, null, new OperationException(e.getMessage())); } return new Tuple4<>(group, column, columnIndex, null); }
ElementWithPropertiesRecordReader(final String keyPackageClass, final Schema schema, final View view) throws StoreException, SchemaException, SerialisationException, UnsupportedEncodingException { super(); final AccumuloKeyPackage keyPackage; try { keyPackage = Class.forName(keyPackageClass).asSubclass(AccumuloKeyPackage.class).newInstance(); } catch (final InstantiationException | IllegalAccessException | ClassNotFoundException e) { throw new StoreException("Unable to construct an instance of key package: " + keyPackageClass, e); } keyPackage.setSchema(schema); this.converter = keyPackage.getKeyConverter(); this.view = view; }
@Override public JobDetail executeJob(final OperationChain<?> operationChain, final Context context) throws OperationException { final URL url = getProperties().getGafferUrl("graph/jobs"); try { return doPost(url, operationChain, new TypeReferenceImpl.JobDetail(), context); } catch (final StoreException e) { throw new OperationException(e.getMessage(), e); } }
protected <O> O doGet(final URL url, final TypeReference<O> outputTypeReference, final Context context) throws StoreException { final Invocation.Builder request = createRequest(null, url, context); final Response response; try { response = request.get(); } catch (final Exception e) { throw new StoreException("Request failed to execute via url " + url.toExternalForm(), e); } return handleResponse(response, outputTypeReference); }
public <O> O executeOpChainViaUrl(final OperationChain<O> opChain, final Context context) throws OperationException { final String opChainJson; try { opChainJson = new String(JSONSerialiser.serialise(opChain), CommonConstants.UTF_8); } catch (final UnsupportedEncodingException | SerialisationException e) { throw new OperationException("Unable to serialise operation chain into JSON.", e); } final URL url = getProperties().getGafferUrl("graph/operations/execute"); try { return doPost(url, opChainJson, opChain.getOutputTypeReference(), context); } catch (final StoreException e) { throw new OperationException(e.getMessage(), e); } }
protected <O> O doPost(final URL url, final Object body, final TypeReference<O> outputType, final Context context) throws StoreException { try { return doPost(url, new String(JSONSerialiser.serialise(body), CommonConstants.UTF_8), outputType, context); } catch (final SerialisationException | UnsupportedEncodingException e) { throw new StoreException("Unable to serialise body of request into json.", e); } }
throw new RuntimeException(e.getMessage(), e);
protected <O> O doPost(final URL url, final String jsonBody, final TypeReference<O> clazz, final Context context) throws StoreException { final Invocation.Builder request = createRequest(jsonBody, url, context); final Response response; try { response = request.post(Entity.json(jsonBody)); } catch (final Exception e) { throw new StoreException("Failed to execute post via " + "the Gaffer URL " + url.toExternalForm(), e); } return handleResponse(response, clazz); }
throw new OperationException("IOException: Failed to connect to the file system", e); } catch (final StoreException e) { throw new OperationException(e.getMessage(), e);
@Override public void initialise(final String graphId, final Schema schema, final StoreProperties properties) throws StoreException { if (!(properties instanceof ParquetStoreProperties)) { throw new StoreException("ParquetStore must be initialised with properties of class ParquetStoreProperties"); } final ParquetStoreProperties parquetStoreProperties = (ParquetStoreProperties) properties; if (null == parquetStoreProperties.getDataDir()) { throw new StoreException("The ParquetStoreProperties must contain a non-null data directory (" + ParquetStoreProperties.DATA_DIR + ")"); } if (null == parquetStoreProperties.getTempFilesDir()) { throw new StoreException("The ParquetStoreProperties must contain a non-null temporary data directory (" + ParquetStoreProperties.TEMP_FILES_DIR + ")"); } super.initialise(graphId, schema, parquetStoreProperties); try { fs = FileSystem.get(new Configuration()); } catch (final IOException e) { throw new StoreException("Could not connect to the file system", e); } schemaUtils = new SchemaUtils(getSchema()); loadIndex(); }
throw new OperationException("IOException: Failed to connect to the file system", e); } catch (final StoreException e) { throw new OperationException(e.getMessage(), e);
private byte[] serialiseObject(final Object value) throws StoreException { final String objectType = value.getClass().getSimpleName(); if ("Long".equals(objectType)) { return BytesUtils.longToBytes((long) value); } else if ("Integer".equals(objectType)) { return BytesUtils.intToBytes((int) value); } else if ("Boolean".equals(objectType)) { return BytesUtils.booleanToBytes((boolean) value); } else if ("Float".equals(objectType)) { return BytesUtils.intToBytes(Float.floatToIntBits((float) value)); } else if ("String".equals(objectType)) { return StringUtil.toBytes((String) value); } else if ("byte[]".equals(objectType)) { return (byte[]) value; } else { throw new StoreException("Cannot serialise objects of type " + objectType); } }
/** * Returns the {@link org.apache.accumulo.core.security.Authorizations} of * the current user * * @param connection the connection to an accumulo instance * @return The accumulo Authorisations of the current user specified in the properties file * @throws StoreException if the table could not be found or other table/security issues */ public static Authorizations getCurrentAuthorizations(final Connector connection) throws StoreException { try { return connection.securityOperations().getUserAuthorizations(connection.whoami()); } catch (final AccumuloException | AccumuloSecurityException e) { throw new StoreException(e.getMessage(), e); } }