private int getNumAccumuloSplits(final AccumuloStore store) { int numberTabletServers; try { numberTabletServers = store.getTabletServers().size(); LOGGER.debug("Number of region servers is {}", numberTabletServers); } catch (final StoreException e) { LOGGER.error("Exception thrown getting number of tablet servers: {}", e.getMessage()); throw new RuntimeException(e.getMessage(), e); } return numberTabletServers - 1; } }
@Override public int getExpectedNumberOfSplits(final Store store) { final AccumuloStore accumuloStore = (AccumuloStore) store; int numberTabletServers; try { numberTabletServers = accumuloStore.getTabletServers().size(); LOGGER.info("Number of tablet servers is {}", numberTabletServers); } catch (final StoreException e) { LOGGER.error("Exception thrown getting number of tablet servers: {}", e.getMessage()); throw new RuntimeException(e.getMessage(), e); } return numberTabletServers - 1; }
private void generateSplitsFromSampleData(final SampleDataForSplitPoints operation, final AccumuloStore store) throws OperationException { try { if (store.getTabletServers().size() < 2) { LOGGER.warn("There is only 1 tablet server so no split points will be calculated."); return; } } catch (final StoreException e) { throw new OperationException(e.getMessage(), e); } final SampleDataAndCreateSplitsFileTool sampleTool = new SampleDataAndCreateSplitsFileTool(new AccumuloSampleDataForSplitPointsJobFactory(), operation, store); try { ToolRunner.run(sampleTool, new String[0]); } catch (final Exception e) { throw new OperationException(e.getMessage(), e); } LOGGER.info("Finished calculating splits"); } }
@Override public Tuple4<String, String, ColumnIndex, OperationException> call() { try { final FileSystem fs = FileSystem.get(new Configuration()); if (fs.exists(new Path(directoryPath))) { final FileStatus[] files = fs.listStatus(new Path(directoryPath), path1 -> path1.getName().endsWith(".parquet")); for (final FileStatus file : files) { final String firstColumn = paths[0]; final Builder<String, Seq<String>> seqBuilder = Seq$.MODULE$.newBuilder(); final int numberOfColumns = paths.length; for (int i = 1; i < numberOfColumns; i++) { seqBuilder.$plus$eq(paths[i]); } final Dataset<Row> fileData = spark.read().parquet(file.getPath().toString()).select(firstColumn, seqBuilder.result()); try { final Row minRow = fileData.head(); columnIndex.add(generateGafferObjectsIndex(minRow, file.getPath().getName())); } catch (final NoSuchElementException ignored) { // ignore as dataframe was empty } } } } catch (final IOException e) { return new Tuple4<>(group, column, null, new OperationException("IOException generating the index files", e)); } catch (final StoreException e) { return new Tuple4<>(group, column, null, new OperationException(e.getMessage())); } return new Tuple4<>(group, column, columnIndex, null); }
@Override public JobDetail executeJob(final OperationChain<?> operationChain, final Context context) throws OperationException { final URL url = getProperties().getGafferUrl("graph/jobs"); try { return doPost(url, operationChain, new TypeReferenceImpl.JobDetail(), context); } catch (final StoreException e) { throw new OperationException(e.getMessage(), e); } }
public <O> O executeOpChainViaUrl(final OperationChain<O> opChain, final Context context) throws OperationException { final String opChainJson; try { opChainJson = new String(JSONSerialiser.serialise(opChain), CommonConstants.UTF_8); } catch (final UnsupportedEncodingException | SerialisationException e) { throw new OperationException("Unable to serialise operation chain into JSON.", e); } final URL url = getProperties().getGafferUrl("graph/operations/execute"); try { return doPost(url, opChainJson, opChain.getOutputTypeReference(), context); } catch (final StoreException e) { throw new OperationException(e.getMessage(), e); } }
throw new RuntimeException(e.getMessage(), e);
throw new OperationException("IOException: Failed to connect to the file system", e); } catch (final StoreException e) { throw new OperationException(e.getMessage(), e);
throw new OperationException("IOException: Failed to connect to the file system", e); } catch (final StoreException e) { throw new OperationException(e.getMessage(), e);