/** * Initializes the Mapper, and sets input parameters for the job * * @param job the job to set the properties for * @param dataStore the datastore as the input * @param outKeyClass Map output key class * @param outValueClass Map output value class * @param mapperClass the mapper class extending GoraMapper * @param reuseObjects whether to reuse objects in serialization * @param <K1> Map input key class * @param <V1> Map input value class * @param <K2> Map output key class * @param <V2> Map output value class * @throws IOException if there is an error initializing the Map job */ @SuppressWarnings({ "rawtypes" }) public static <K1, V1 extends Persistent, K2, V2> void initMapperJob( Job job, DataStore<K1,V1> dataStore, Class<K2> outKeyClass, Class<V2> outValueClass, Class<? extends GoraMapper> mapperClass, boolean reuseObjects) throws IOException { initMapperJob(job, dataStore.newQuery(), outKeyClass, outValueClass, mapperClass, reuseObjects); }
/** * Initializes the Mapper, and sets input parameters for the job * * @param job the job to set the properties for * @param dataStore the datastore as the input * @param outKeyClass Map output key class * @param outValueClass Map output value class * @param mapperClass the mapper class extending GoraMapper * @param reuseObjects whether to reuse objects in serialization * @param <K1> Map input key class * @param <V1> Map input value class * @param <K2> Map output key class * @param <V2> Map output value class * @throws IOException if there is an error initializing the Map job */ @SuppressWarnings({ "rawtypes" }) public static <K1, V1 extends Persistent, K2, V2> void initMapperJob( Job job, DataStore<K1,V1> dataStore, Class<K2> outKeyClass, Class<V2> outValueClass, Class<? extends GoraMapper> mapperClass, boolean reuseObjects) throws IOException { initMapperJob(job, dataStore.newQuery(), outKeyClass, outValueClass, mapperClass, reuseObjects); }
/** This method illustrates delete by query call */ private void deleteByQuery(long startKey, long endKey) throws Exception { //Constructs a query from the dataStore. The matching rows to this query will be deleted Query<Long, Pageview> query = dataStore.newQuery(); //set the properties of query query.setStartKey(startKey); query.setEndKey(endKey); dataStore.deleteByQuery(query); log.info("pageviews with keys between {} and {} are deleted.", startKey, endKey); }
/** * This method illustrates delete by query call */ private void deleteByQuery(long startKey, long endKey) throws Exception { //Constructs a query from the dataStore. The matching rows to this query will be deleted Query<Long, Pageview> query = cacheStore.newQuery(); //set the properties of query query.setStartKey(startKey); query.setEndKey(endKey); cacheStore.deleteByQuery(query); log.info("pageviews with keys between {} and {} are deleted.", startKey, endKey); }
/** * Sets the input parameters for the job * * @param job the job to set the properties for * @param dataStoreClass the datastore class * @param inKeyClass Map input key class * @param inValueClass Map input value class * @param reuseObjects whether to reuse objects in serialization * @param <K1> key class for {@link org.apache.gora.query.Query} * @param <V1> object value key class for {@link org.apache.gora.query.Query} * @throws IOException if there is an error configuring the input parameters for the job */ public static <K1, V1 extends Persistent> void setInput( Job job, Class<? extends DataStore<K1,V1>> dataStoreClass, Class<K1> inKeyClass, Class<V1> inValueClass, boolean reuseObjects) throws IOException { DataStore<K1,V1> store = DataStoreFactory.getDataStore(dataStoreClass , inKeyClass, inValueClass, job.getConfiguration()); setInput(job, store.newQuery(), reuseObjects); } }
/** * Sets the input parameters for the job * * @param job the job to set the properties for * @param dataStoreClass the datastore class * @param inKeyClass Map input key class * @param inValueClass Map input value class * @param reuseObjects whether to reuse objects in serialization * @param <K1> key class for {@link org.apache.gora.query.Query} * @param <V1> object value key class for {@link org.apache.gora.query.Query} * @throws IOException if there is an error configuring the input parameters for the job */ public static <K1, V1 extends Persistent> void setInput( Job job, Class<? extends DataStore<K1,V1>> dataStoreClass, Class<K1> inKeyClass, Class<V1> inValueClass, boolean reuseObjects) throws IOException { DataStore<K1,V1> store = DataStoreFactory.getDataStore(dataStoreClass , inKeyClass, inValueClass, job.getConfiguration()); setInput(job, store.newQuery(), reuseObjects); } }
/** * Initializes a {@link JavaPairRDD} from given Spark context, Hadoop * configuration and data store. * * @param sparkContext * Spark context * @param conf * Hadoop configuration * @param dataStore * Data store * @return initialized rdd */ public JavaPairRDD<K, V> initialize(JavaSparkContext sparkContext, Configuration conf, DataStore<K, V> dataStore) { GoraMapReduceUtils.setIOSerializations(conf, true); try { IOUtils .storeToConf(dataStore.newQuery(), conf, GoraInputFormat.QUERY_KEY); } catch (IOException ioex) { throw new RuntimeException(ioex.getMessage()); } return sparkContext.newAPIHadoopRDD(conf, GoraInputFormat.class, clazzK, clazzV); }
/** * Initializes a {@link JavaPairRDD} from given Spark context, Hadoop * configuration and data store. * * @param sparkContext * Spark context * @param conf * Hadoop configuration * @param dataStore * Data store * @return initialized rdd */ public JavaPairRDD<K, V> initialize(JavaSparkContext sparkContext, Configuration conf, DataStore<K, V> dataStore) { GoraMapReduceUtils.setIOSerializations(conf, true); try { IOUtils .storeToConf(dataStore.newQuery(), conf, GoraInputFormat.QUERY_KEY); } catch (IOException ioex) { throw new RuntimeException(ioex.getMessage()); } return sparkContext.newAPIHadoopRDD(conf, GoraInputFormat.class, clazzK, clazzV); }
/** Queries and prints a single pageview object */ private void query(long key) throws Exception { //Queries are constructed from the data store Query<Long, Pageview> query = dataStore.newQuery(); query.setKey(key); Result<Long, Pageview> result = query.execute(); //Actually executes the query. // alternatively dataStore.execute(query); can be used printResult(result); }
/** * Queries and prints pageview object that have keys between startKey and endKey */ private void query(long startKey, long endKey) throws Exception { Query<Long, Pageview> query = cacheStore.newQuery(); //set the properties of query query.setStartKey(startKey); query.setEndKey(endKey); Result<Long, Pageview> result = query.execute(); printResult(result); }
/** Queries and prints pageview object that have keys between startKey and endKey*/ private void query(long startKey, long endKey) throws Exception { Query<Long, Pageview> query = dataStore.newQuery(); //set the properties of query query.setStartKey(startKey); query.setEndKey(endKey); Result<Long, Pageview> result = query.execute(); printResult(result); }
public static void testGetPartitions(DataStore<String, WebPage> store) throws Exception { createWebPageData(store); testGetPartitions(store, store.newQuery()); }
public static void testQueryWebPageEmptyResults(DataStore<String, WebPage> store) throws Exception { createWebPageData(store); //query empty results Query<String, WebPage> query = store.newQuery(); query.setStartKey("aa"); query.setEndKey("ab"); assertEmptyResults(query); //query empty results for one key query = store.newQuery(); query.setKey("aa"); assertEmptyResults(query); }
public static void testQueryWebPages(DataStore<String, WebPage> store) throws Exception { Query<String, WebPage> query = store.newQuery(); Result<String, WebPage> result = query.execute(); int i=0; while(result.next()) { WebPage page = result.get(); DataStoreTestUtil.assertWebPage(page, URL_INDEXES.get(page.getUrl().toString())); i++; } assertEquals(i, URLS.length); }
Query<String, WebPage> query = store.newQuery(); if(setStartKeys) query.setStartKey(sortedUrls.get(i));
/** * Tests by running the {@link org.apache.gora.examples.mapreduce.QueryCounter} * mapreduce job */ public static void testCountQuery(DataStore<String, WebPage> dataStore, Configuration conf) throws Exception { ((DataStoreBase<String, WebPage>)dataStore).setConf(conf); //create input WebPageDataCreator.createWebPageData(dataStore); QueryCounter<String,WebPage> counter = new QueryCounter<>(conf); Query<String,WebPage> query = dataStore.newQuery(); List<Field> fields = WebPage.SCHEMA$.getFields(); String[] fieldNames = new String[fields.size() - 1]; for(int i = 0; i< fieldNames.length; i++){ fieldNames[i] = fields.get(i+1).name(); } query.setFields(fieldNames); dataStore.close(); //run the job log.info("running count query job"); long result = counter.countQuery(query); log.info("finished count query job"); //assert results assertEquals(WebPageDataCreator.URLS.length, result); }
public static void testTruncateSchema(DataStore<String, WebPage> dataStore) throws Exception { dataStore.createSchema(); WebPageDataCreator.createWebPageData(dataStore); dataStore.truncateSchema(); assertEmptyResults(dataStore.newQuery()); }
public static void testDeleteSchema(DataStore<String, WebPage> dataStore) throws Exception { dataStore.createSchema(); WebPageDataCreator.createWebPageData(dataStore); dataStore.deleteSchema(); dataStore.createSchema(); assertEmptyResults(dataStore.newQuery()); }
public static void testDelete(DataStore<String, WebPage> store) throws Exception { WebPageDataCreator.createWebPageData(store); //delete one by one int deletedSoFar = 0; for(String url : URLS) { assertTrue(store.delete(url)); store.flush(); //assert that it is actually deleted assertNull(store.get(url)); //assert that other records are not deleted assertNumResults(store.newQuery(), URLS.length - ++deletedSoFar); } }
private static void testQueryWebPageSingleKey(DataStore<String, WebPage> store , String[] fields) throws Exception { createWebPageData(store); for(int i=0; i<URLS.length; i++) { Query<String, WebPage> query = store.newQuery(); query.setFields(fields); query.setKey(URLS[i]); Result<String, WebPage> result = query.execute(); assertTrue(result.next()); WebPage page = result.get(); assertWebPage(page, i); assertFalse(result.next()); } }