/** Stores the pageview object with the given key */ private void storePageview(long key, Pageview pageview) throws Exception { log.info("Storing Pageview in: " + dataStore.toString()); dataStore.put(key, pageview); }
/** * Stores the pageview object with the given key */ private void storePageview(long key, Pageview pageview, boolean isCacheEnabled) throws Exception { if (!isCacheEnabled) { log.info("Storing Pageview in: " + dataStore.toString()); dataStore.put(key, pageview); } else { log.info("Storing Pageview in: " + dataStore.toString()); cacheStore.put(key, pageview); } }
@Override public void write(K key, T value) throws IOException, InterruptedException { try{ store.put(key, (Persistent) value); counter.increment(); if (counter.isModulo()) { LOG.info("Flushing the datastore after " + counter.getRecordsNumber() + " records"); store.flush(); } }catch(Exception e){ LOG.warn("Exception at GoraRecordWriter.class while writing to datastore: {}", e.getMessage()); LOG.warn("Trace: {}", e.getStackTrace()); throw new RuntimeException(e); } } }
@Override public void write(Cache.Entry<? extends K, ? extends T> entry) throws CacheWriterException { dataStore.put(entry.getKey(), entry.getValue()); LOG.info("Written data bean to persistent datastore on key {}.", entry.getKey().toString()); }
@Override public void write(K key, T value) throws IOException, InterruptedException { try{ store.put(key, (Persistent) value); counter.increment(); if (counter.isModulo()) { LOG.info("Flushing the datastore after " + counter.getRecordsNumber() + " records"); store.flush(); } }catch(Exception e){ LOG.warn("Exception at GoraRecordWriter.class while writing to datastore: {}", e.getMessage()); LOG.warn("Trace: {}", e.getStackTrace()); throw new RuntimeException(e); } } }
@Override public void write(Cache.Entry<? extends K, ? extends T> entry) throws CacheWriterException { try { dataStore.put(entry.getKey(), entry.getValue()); LOG.info("Written data bean to persistent datastore on key {}.", entry.getKey().toString()); } catch (GoraException e) { throw new CacheWriterException(e); } }
public static void testAutoCreateSchema(DataStore<String,Employee> dataStore) throws Exception { //should not throw exception dataStore.put("foo", createEmployee()); }
public static <T extends CharSequence> void populateEmployeeStore(DataStore<T, Employee> dataStore, int n) throws GoraException { for(int i=0; i<n; i++) { Employee e = createEmployee(i); dataStore.put((T)e.getSsn(),e); } dataStore.flush(); }
public static byte[] testPutBytes(DataStore<String, WebPage> store) throws Exception { store.createSchema(); WebPage page = WebPage.newBuilder().build(); page.setUrl(new Utf8("http://example.com")); byte[] contentBytes = "example content in example.com".getBytes(Charset.defaultCharset()); ByteBuffer buff = ByteBuffer.wrap(contentBytes); page.setContent(buff); store.put("com.example/http", page); store.close(); return contentBytes; }
public static void testPutArray(DataStore<String, WebPage> store) throws Exception { store.createSchema(); WebPage page = WebPage.newBuilder().build(); String[] tokens = {"example", "content", "in", "example.com"}; page.setParsedContent(new ArrayList<CharSequence>()); for(String token: tokens) { page.getParsedContent().add(new Utf8(token)); } store.put("com.example/http", page); store.close(); }
public static void testPutMixedMapTypes(DataStore<String, WebPage> store) throws GoraException { WebPage webpage = createWebPage(); webpage.getByteData().put(new Utf8("byteData"), ByteBuffer.wrap(ByteUtils.toBytes("hello map"))); webpage.getStringData().put(new Utf8("stringData"), "hello map"); store.createSchema(); store.put(webpage.getUrl().toString(), webpage); store.flush(); assertNotNull(store.get(webpage.getUrl().toString())); }
public static void testEmptyUpdateEmployee(DataStore<String, Employee> dataStore) throws Exception { dataStore.createSchema(); long ssn = 1234567890L; String ssnStr = Long.toString(ssn); long now = System.currentTimeMillis(); Employee employee = dataStore.newPersistent(); employee.setName(new Utf8("John Doe")); employee.setDateOfBirth(now - 20L * YEAR_IN_MS); employee.setSalary(100000); employee.setSsn(new Utf8(ssnStr)); dataStore.put(employee.getSsn().toString(), employee); dataStore.flush(); employee = dataStore.get(ssnStr); dataStore.put(ssnStr, employee); dataStore.flush(); employee = dataStore.newPersistent(); dataStore.put(Long.toString(ssn + 1), employee); dataStore.flush(); employee = dataStore.get(Long.toString(ssn + 1)); assertNull(employee); }
public static void testGetEmployee(DataStore<String, Employee> dataStore) throws Exception { dataStore.createSchema(); Employee employee = DataStoreTestUtil.createEmployee(); String ssn = employee.getSsn().toString(); dataStore.put(ssn, employee); dataStore.flush(); Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$)); assertEqualEmployeeObjects(employee, after); }
public static void testGetEmployeeRecursive(DataStore<String, Employee> dataStore) throws Exception { Employee employee = DataStoreTestUtil.createEmployee(); Employee boss = DataStoreTestUtil.createBoss(); employee.setBoss(boss); String ssn = employee.getSsn().toString(); dataStore.put(ssn, employee); dataStore.flush(); Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$)); assertEqualEmployeeObjects(employee, after); }
public static void testGetEmployee3UnionField(DataStore<String, Employee> dataStore) throws Exception { Employee employee = DataStoreTestUtil.createEmployee(); employee.setBoss(new Utf8("Real boss")) ; String ssn = employee.getSsn().toString(); dataStore.put(ssn, employee); dataStore.flush(); Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$)); assertEqualEmployeeObjects(employee, after); assertEquals("Real boss", after.getBoss().toString()) ; }
public static void testPutMap(DataStore<String, WebPage> store) throws Exception { store.createSchema(); WebPage page = WebPage.newBuilder().build(); page.setUrl(new Utf8("http://example.com")); page.getOutlinks().put(new Utf8("http://example2.com"), new Utf8("anchor2")); page.getOutlinks().put(new Utf8("http://example3.com"), new Utf8("anchor3")); page.getOutlinks().put(new Utf8("http://example3.com"), new Utf8("anchor4")); store.put("com.example/http", page); store.close(); }
public static void testGetEmployeeNested(DataStore<String, Employee> dataStore) throws Exception { Employee employee = DataStoreTestUtil.createEmployee(); WebPage webpage = new BeanFactoryImpl<>(String.class,WebPage.class).newPersistent() ; webpage.setUrl(new Utf8("url..")) ; webpage.setContent(ByteBuffer.wrap("test content".getBytes(Charset.defaultCharset()))) ; webpage.setParsedContent(new ArrayList<CharSequence>()); Metadata metadata = new BeanFactoryImpl<>(String.class,Metadata.class).newPersistent(); webpage.setMetadata(metadata) ; employee.setWebpage(webpage) ; String ssn = employee.getSsn().toString(); dataStore.put(ssn, employee); dataStore.flush(); Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$)); assertEqualEmployeeObjects(employee, after); assertEqualWebPageObjects(webpage, after.getWebpage()); }
@Test public void testPutGet() throws Exception { String key = "org.apache.gora:http:/"; DataStore<String, WebPage> store = new MemStore<>(); assumeTrue(store.get(key, new String[0]) == null); store.put(key, WebPage.newBuilder().build()); assertNotNull(store.get(key, new String[0])); store.close(); }
public static void testGetEmployeeDoubleRecursive(DataStore<String, Employee> dataStore) throws Exception { Employee employee = DataStoreTestUtil.createEmployee(); Employee boss = DataStoreTestUtil.createBoss(); Employee uberBoss = DataStoreTestUtil.createBoss(); uberBoss.setName(new Utf8("Überboss")) ; boss.setBoss(uberBoss) ; employee.setBoss(boss) ; String ssn = employee.getSsn().toString(); dataStore.put(ssn, employee); dataStore.flush(); Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$)); assertEqualEmployeeObjects(employee, after); }
public static void testPutNested(DataStore<String, WebPage> store) throws Exception { String revUrl = "foo.com:http/"; String url = "http://foo.com/"; store.createSchema(); WebPage page = WebPage.newBuilder().build(); Metadata metadata = Metadata.newBuilder().build(); metadata.setVersion(1); metadata.getData().put(new Utf8("foo"), new Utf8("baz")); page.setMetadata(metadata); page.setUrl(new Utf8(url)); store.put(revUrl, page); store.flush(); page = store.get(revUrl); metadata = page.getMetadata(); assertNotNull(metadata); assertEquals(1, metadata.getVersion().intValue()); assertEquals(new Utf8("baz"), metadata.getData().get(new Utf8("foo"))); }