@Override public Dataset<GenericRecord> run() { return Datasets.load(datasetUri); } });
private View<Record> load(ProcessContext context, FlowFile file) { String uri = context.getProperty(KITE_DATASET_URI) .evaluateAttributeExpressions(file) .getValue(); return Datasets.load(uri, Record.class); } }
return Datasets.load(uri).getDataset().getDescriptor().getSchema(); } else if ("resource".equals(uri.getScheme())) { try (InputStream in = Resources.getResource(uri.getSchemeSpecificPart()).openStream()) {
private SavePolicy(Context context) { String uri = context.getString(CONFIG_KITE_ERROR_DATASET_URI); Preconditions.checkArgument(uri != null, "Must set " + CONFIG_KITE_ERROR_DATASET_URI + " when " + CONFIG_FAILURE_POLICY + "=save"); if (Datasets.exists(uri)) { dataset = Datasets.load(uri, AvroFlumeEvent.class); } else { DatasetDescriptor descriptor = new DatasetDescriptor.Builder() .schema(AvroFlumeEvent.class) .build(); dataset = Datasets.create(uri, descriptor, AvroFlumeEvent.class); } nEventsHandled = 0; }
@Override public Dataset<GenericRecord> run() { return Datasets.load(uri); } });
@Override public Dataset<GenericRecord> run() { return Datasets.load(datasetUri); } });
@Override public void run() { Dataset<Record> ds = Datasets.<Record, Dataset<Record>> load("dataset:unknown:/tmp/data/test", Record.class); } });
@Override public void run() { Dataset<Object> ds = Datasets .<Object, Dataset<Object>>load("dataset:hbase:" + zk + "/nosuchdataset", Object.class); } }
@Override public void run() { Dataset<Object> ds = Datasets.<Object, Dataset<Object>> load("dataset:hdfs://" + hdfsAuth + "/tmp/data/ns/nosuchdataset", Object.class); } });
@Override public void run() { Dataset<Object> ds = Datasets.<Object, Dataset<Object>> load("dataset:hdfs://" + hdfsAuth + "/test", Object.class); } });
@Override public void run() { Dataset<Object> ds = Datasets.<Object, Dataset<Object>> load("dataset:hdfs://" + hdfsAuth + "/tmp/data/nosuchnamespace/test", Object.class); } });
@Override public void run() { Dataset<Record> ds = Datasets.<Record, Dataset<Record>> load("dataset:file:/tmp/data/ns/nosuchdataset", Record.class); } });
@Override public void run() { Dataset<Record> ds = Datasets.<Record, Dataset<Record>> load("dataset:file:/test", Record.class); } });
@SuppressWarnings("deprecation") private static <E> View<E> load(JobContext jobContext) { Configuration conf = Hadoop.JobContext.getConfiguration.invoke(jobContext); Class<E> type = getType(jobContext); String outputUri = conf.get(KITE_OUTPUT_URI); return Datasets.<E, View<E>>load(outputUri, type); }
@SuppressWarnings("deprecation") private static <E> View<E> load(JobContext jobContext) { Configuration conf = Hadoop.JobContext.getConfiguration.invoke(jobContext); Class<E> type = getType(jobContext); String outputUri = conf.get(KITE_OUTPUT_URI); return Datasets.<E, View<E>>load(outputUri, type); }
@Test(expected=DatasetIOException.class) public void testCannotFindHDFS() throws Exception { // do not set the default configuration that the loader will use Datasets.load("dataset:hdfs:/tmp/datasets/ns/strings"); } }
private View<Record> load(ProcessContext context, FlowFile file) { String uri = context.getProperty(KITE_DATASET_URI) .evaluateAttributeExpressions(file) .getValue(); return Datasets.load(uri, Record.class); } }
@Test public void testLoadWithoutType() { Dataset<GenericRecord> expected = mock(Dataset.class); when(repo.load("ns", "test", GenericRecord.class)).thenReturn(expected); URI datasetUri = new URIBuilder(repoUri, "ns", "test").build(); Dataset<GenericRecord> ds = Datasets.load(datasetUri); verify(repo).load("ns", "test", GenericRecord.class); verifyNoMoreInteractions(repo); verifyNoMoreInteractions(expected); Assert.assertEquals(expected, ds); }
@Test public void testLoadStringUri() { Dataset<Object> expected = mock(Dataset.class); when(repo.load("ns", "test", Object.class)).thenReturn(expected); URI datasetUri = new URIBuilder(repoUri, "ns", "test").build(); Dataset<Object> ds = Datasets.<Object, Dataset<Object>> load(datasetUri, Object.class); verify(repo).load("ns", "test", Object.class); verifyNoMoreInteractions(repo); verifyNoMoreInteractions(expected); Assert.assertEquals(expected, ds); }
@Test public void testMissingHDFSAuthority() { try { Datasets.load("dataset:hdfs:/tmp/data/ns/test", Object.class); Assert.fail("Shouldn't be able to connect to HDFS"); } catch (DatasetIOException e) { Assert.assertTrue("Should have helpful error message", e.getMessage().contains("make sure the default hdfs URI is configured")); } }