@Override public void flush() throws GoraException { persistentDataStore.flush(); LOG.info("JCache Gora datastore flushed successfully."); }
@Override public void flush() { persistentDataStore.flush(); LOG.info("JCache Gora datastore flushed successfully."); }
/** * Deletes the pageview with the given line number */ private void delete(long lineNum) throws Exception { cacheStore.delete(lineNum); cacheStore.flush(); //write changes may need to be flushed before //they are committed log.info("pageview with key: {} deleted", lineNum); }
/**Deletes the pageview with the given line number */ private void delete(long lineNum) throws Exception { dataStore.delete(lineNum); dataStore.flush(); //write changes may need to be flushed before //they are committed log.info("pageview with key: {} deleted", lineNum); }
/** * <p> * If the {@link org.apache.gora.store.DataStore} instance is not null, we * execute a {@link org.apache.gora.store.DataStore#flush()}. This forces * the write caches to be flushed. DataStore implementations may optimize * their writing by deferring the actual put / delete operations until * this moment. * </p> * <p>Otherwise, we utilize {@link org.apache.gora.store.DataStore#close()} * which closes the DataStore. This should release any resources held by * the implementation, so that the instance is ready for GC. All other * DataStore methods cannot be used after this method was called. * Subsequent calls of this method are ignored. * </p> * @see org.apache.hadoop.chukwa.datacollection.writer.ChukwaWriter#close() */ @Override public void close() throws WriterException { if (chunkStore != null) { chunkStore.flush(); } else { chunkStore.close(); } log.debug("Gora datastore successfully closed."); }
@Override public void write(K key, T value) throws IOException, InterruptedException { try{ store.put(key, (Persistent) value); counter.increment(); if (counter.isModulo()) { LOG.info("Flushing the datastore after " + counter.getRecordsNumber() + " records"); store.flush(); } }catch(Exception e){ LOG.warn("Exception at GoraRecordWriter.class while writing to datastore: {}", e.getMessage()); LOG.warn("Trace: {}", e.getStackTrace()); throw new RuntimeException(e); } } }
@Override public void write(K key, T value) throws IOException, InterruptedException { try{ store.put(key, (Persistent) value); counter.increment(); if (counter.isModulo()) { LOG.info("Flushing the datastore after " + counter.getRecordsNumber() + " records"); store.flush(); } }catch(Exception e){ LOG.warn("Exception at GoraRecordWriter.class while writing to datastore: {}", e.getMessage()); LOG.warn("Trace: {}", e.getStackTrace()); throw new RuntimeException(e); } } }
public static <T extends CharSequence> void populateEmployeeStore(DataStore<T, Employee> dataStore, int n) throws GoraException { for(int i=0; i<n; i++) { Employee e = createEmployee(i); dataStore.put((T)e.getSsn(),e); } dataStore.flush(); }
public static void testPutMixedMapTypes(DataStore<String, WebPage> store) throws GoraException { WebPage webpage = createWebPage(); webpage.getByteData().put(new Utf8("byteData"), ByteBuffer.wrap(ByteUtils.toBytes("hello map"))); webpage.getStringData().put(new Utf8("stringData"), "hello map"); store.createSchema(); store.put(webpage.getUrl().toString(), webpage); store.flush(); assertNotNull(store.get(webpage.getUrl().toString())); }
public static void testEmptyUpdateEmployee(DataStore<String, Employee> dataStore) throws Exception { dataStore.createSchema(); long ssn = 1234567890L; String ssnStr = Long.toString(ssn); long now = System.currentTimeMillis(); Employee employee = dataStore.newPersistent(); employee.setName(new Utf8("John Doe")); employee.setDateOfBirth(now - 20L * YEAR_IN_MS); employee.setSalary(100000); employee.setSsn(new Utf8(ssnStr)); dataStore.put(employee.getSsn().toString(), employee); dataStore.flush(); employee = dataStore.get(ssnStr); dataStore.put(ssnStr, employee); dataStore.flush(); employee = dataStore.newPersistent(); dataStore.put(Long.toString(ssn + 1), employee); dataStore.flush(); employee = dataStore.get(Long.toString(ssn + 1)); assertNull(employee); }
public static void testGetEmployee(DataStore<String, Employee> dataStore) throws Exception { dataStore.createSchema(); Employee employee = DataStoreTestUtil.createEmployee(); String ssn = employee.getSsn().toString(); dataStore.put(ssn, employee); dataStore.flush(); Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$)); assertEqualEmployeeObjects(employee, after); }
public static void testGetEmployee3UnionField(DataStore<String, Employee> dataStore) throws Exception { Employee employee = DataStoreTestUtil.createEmployee(); employee.setBoss(new Utf8("Real boss")) ; String ssn = employee.getSsn().toString(); dataStore.put(ssn, employee); dataStore.flush(); Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$)); assertEqualEmployeeObjects(employee, after); assertEquals("Real boss", after.getBoss().toString()) ; }
public static void testGetEmployeeRecursive(DataStore<String, Employee> dataStore) throws Exception { Employee employee = DataStoreTestUtil.createEmployee(); Employee boss = DataStoreTestUtil.createBoss(); employee.setBoss(boss); String ssn = employee.getSsn().toString(); dataStore.put(ssn, employee); dataStore.flush(); Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$)); assertEqualEmployeeObjects(employee, after); }
page.setContent(ByteBuffer.wrap("content".getBytes(Charset.defaultCharset()))); inStore.put("key1", page); inStore.flush();
public static void testGetEmployeeNested(DataStore<String, Employee> dataStore) throws Exception { Employee employee = DataStoreTestUtil.createEmployee(); WebPage webpage = new BeanFactoryImpl<>(String.class,WebPage.class).newPersistent() ; webpage.setUrl(new Utf8("url..")) ; webpage.setContent(ByteBuffer.wrap("test content".getBytes(Charset.defaultCharset()))) ; webpage.setParsedContent(new ArrayList<CharSequence>()); Metadata metadata = new BeanFactoryImpl<>(String.class,Metadata.class).newPersistent(); webpage.setMetadata(metadata) ; employee.setWebpage(webpage) ; String ssn = employee.getSsn().toString(); dataStore.put(ssn, employee); dataStore.flush(); Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$)); assertEqualEmployeeObjects(employee, after); assertEqualWebPageObjects(webpage, after.getWebpage()); }
public static void testDelete(DataStore<String, WebPage> store) throws Exception { WebPageDataCreator.createWebPageData(store); //delete one by one int deletedSoFar = 0; for(String url : URLS) { assertTrue(store.delete(url)); store.flush(); //assert that it is actually deleted assertNull(store.get(url)); //assert that other records are not deleted assertNumResults(store.newQuery(), URLS.length - ++deletedSoFar); } }
public static void testUpdateWebPageRemoveField(DataStore<String, WebPage> dataStore) throws Exception { dataStore.createSchema(); String[] urls = {"http://a.com/a", "http://b.com/b", "http://c.com/c", "http://d.com/d", "http://e.com/e", "http://f.com/f", "http://g.com/g" }; String header = "header"; String[] headers = { "firstHeader", "secondHeader", "thirdHeader", "fourthHeader", "fifthHeader", "sixthHeader" }; for (String url : urls) { WebPage webPage = WebPage.newBuilder().build(); webPage.setUrl(new Utf8(url)); webPage.setHeaders(new HashMap<CharSequence, CharSequence>()); for (int j = 0; j < headers.length; j++) { webPage.getHeaders().put(new Utf8(header + j), new Utf8(headers[j])); } dataStore.put(webPage.getUrl().toString(), webPage); } dataStore.flush(); // nullable map field removal test for (String url : urls) { WebPage webPage = dataStore.get(url); webPage.setHeaders(null); dataStore.put(webPage.getUrl().toString(), webPage); } dataStore.flush(); for (String url : urls) { WebPage webPage = dataStore.get(url); assertNull(webPage.getHeaders()); } }
public static void testGetEmployeeDoubleRecursive(DataStore<String, Employee> dataStore) throws Exception { Employee employee = DataStoreTestUtil.createEmployee(); Employee boss = DataStoreTestUtil.createBoss(); Employee uberBoss = DataStoreTestUtil.createBoss(); uberBoss.setName(new Utf8("Überboss")) ; boss.setBoss(uberBoss) ; employee.setBoss(boss) ; String ssn = employee.getSsn().toString(); dataStore.put(ssn, employee); dataStore.flush(); Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$)); assertEqualEmployeeObjects(employee, after); }
public static void testGetEmployeeWithFields(DataStore<String, Employee> dataStore) throws Exception { Employee employee = DataStoreTestUtil.createEmployee(); WebPage webpage = createWebPage(); employee.setWebpage(webpage); Employee boss = createBoss(); employee.setBoss(boss); String ssn = employee.getSsn().toString(); dataStore.put(ssn, employee); dataStore.flush(); String[] fields = AvroUtils.getPersistentFieldNames(employee); for(Set<String> subset : StringUtils.powerset(fields)) { if(subset.isEmpty()) continue; Employee after = dataStore.get(ssn, subset.toArray(new String[subset.size()])); Employee expected = Employee.newBuilder().build(); for(String field:subset) { int index = expected.getSchema().getField(field).pos(); expected.put(index, employee.get(index)); } assertEqualEmployeeObjects(expected, after); } }
public static void testPutNested(DataStore<String, WebPage> store) throws Exception { String revUrl = "foo.com:http/"; String url = "http://foo.com/"; store.createSchema(); WebPage page = WebPage.newBuilder().build(); Metadata metadata = Metadata.newBuilder().build(); metadata.setVersion(1); metadata.getData().put(new Utf8("foo"), new Utf8("baz")); page.setMetadata(metadata); page.setUrl(new Utf8(url)); store.put(revUrl, page); store.flush(); page = store.get(revUrl); metadata = page.getMetadata(); assertNotNull(metadata); assertEquals(1, metadata.getVersion().intValue()); assertEquals(new Utf8("baz"), metadata.getData().get(new Utf8("foo"))); }