/** * Configure {@link DataLoader} using properties used to configure the * database. * * @param database * The database. */ public DataLoader(final AbstractTripleStore database) { this(database.getProperties(), database , System.out); }
/** * Configure {@link DataLoader} using properties used to configure the * database. * * @param database * The database. */ public DataLoader(final AbstractTripleStore database) { this(database.getProperties(), database , System.out); }
protected TempTripleStore createTempTripleStore() { // log.info("creating temp triple store for owl:sameAs backchainer"); // System.err.println("creating temp triple store for owl:sameAs backchainer"); final Properties props = db.getProperties(); // do not store terms props.setProperty(AbstractTripleStore.Options.LEXICON, "false"); // only store the SPO index props.setProperty(AbstractTripleStore.Options.ONE_ACCESS_PATH, "true"); // @todo MikeP : test w/ SPO bloom filter enabled and see if this improves performance. props.setProperty(AbstractTripleStore.Options.BLOOM_FILTER, "false"); return new TempTripleStore(db.getIndexManager().getTempStore(), props, db); }
protected TempTripleStore createTempTripleStore() { // log.info("creating temp triple store for owl:sameAs backchainer"); // System.err.println("creating temp triple store for owl:sameAs backchainer"); final Properties props = db.getProperties(); // do not store terms props.setProperty(AbstractTripleStore.Options.LEXICON, "false"); // only store the SPO index props.setProperty(AbstractTripleStore.Options.ONE_ACCESS_PATH, "true"); // @todo MikeP : test w/ SPO bloom filter enabled and see if this improves performance. props.setProperty(AbstractTripleStore.Options.BLOOM_FILTER, "false"); return new TempTripleStore(db.getIndexManager().getTempStore(), props, db); }
/** * Test where an error in a source file SHOULD be ignored because we have * specified {@link DataLoader.Options#IGNORE_INVALID_FILES}. * * @see BLZG-1531 (Add option to make the DataLoader robust to files that * cause rio to throw a fatal exception) */ public void test_DataLoader_ignoreFailures02() throws IOException { final AbstractTripleStore store = getStore(); try { final Properties properties = new Properties(store.getProperties()); properties.setProperty(DataLoader.Options.IGNORE_INVALID_FILES, "true"); final DataLoader dataLoader = new DataLoader(properties, store); final String resource = "com/bigdata/rdf/store/sample-data-bad.ttl"; final String baseURL = new File(resource).toURI().toString(); dataLoader.loadData(new String[] { resource }, new String[] { baseURL }, new RDFFormat[] { RDFFormat.TURTLE }); } finally { store.__tearDownUnitTest(); } }
/** * Test durable queues using {@link CommitEnum#Batch} and {@link ClosureEnum#None} */ public void test_durableQueues02_batchCommit_noClosure() throws IOException { final AbstractTripleStore store = getStore(); try { final Properties properties = new Properties(store.getProperties()); // enable durable queues. properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true"); // Batch commit. properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Batch.name()); properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.None.name()); final DataLoader dataLoader = new DataLoader(properties, store); doDurableQueueTest(dataLoader); } finally { store.__tearDownUnitTest(); } }
/** * Test durable queues using {@link CommitEnum#Batch} and {@link ClosureEnum#None} */ public void test_durableQueues02_batchCommit_noClosure() throws IOException { final AbstractTripleStore store = getStore(); try { final Properties properties = new Properties(store.getProperties()); // enable durable queues. properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true"); // Batch commit. properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Batch.name()); properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.None.name()); final DataLoader dataLoader = new DataLoader(properties, store); doDurableQueueTest(dataLoader); } finally { store.__tearDownUnitTest(); } }
/** * Test durable queues using {@link CommitEnum#Incremental} and {@link ClosureEnum#Batch} */ public void test_durableQueues01_incrementalCommit_batchClosure() throws IOException { final AbstractTripleStore store = getStore(); try { final Properties properties = new Properties(store.getProperties()); // enable durable queues. properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true"); // Incremental commit. properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Incremental.name()); properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.Batch.name()); final DataLoader dataLoader = new DataLoader(properties, store); doDurableQueueTest(dataLoader); } finally { store.__tearDownUnitTest(); } }
/** * Test durable queues using {@link CommitEnum#Batch} and {@link ClosureEnum#Batch} */ public void test_durableQueues02_batchCommit_batchClosure() throws IOException { final AbstractTripleStore store = getStore(); try { final Properties properties = new Properties(store.getProperties()); // enable durable queues. properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true"); // Batch commit. properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Batch.name()); properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.Batch.name()); final DataLoader dataLoader = new DataLoader(properties, store); doDurableQueueTest(dataLoader); } finally { store.__tearDownUnitTest(); } }
/** * Test durable queues using {@link CommitEnum#Batch} and {@link ClosureEnum#Incremental} */ public void test_durableQueues02_batchCommit_incrementalClosure() throws IOException { final AbstractTripleStore store = getStore(); try { final Properties properties = new Properties(store.getProperties()); // enable durable queues. properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true"); // Batch commit. properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Batch.name()); properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.Incremental.name()); final DataLoader dataLoader = new DataLoader(properties, store); doDurableQueueTest(dataLoader); } finally { store.__tearDownUnitTest(); } }
/** * Test durable queues using {@link CommitEnum#Incremental} and {@link ClosureEnum#None} */ public void test_durableQueues01_incrementalCommit_noClosure() throws IOException { final AbstractTripleStore store = getStore(); try { final Properties properties = new Properties(store.getProperties()); // enable durable queues. properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true"); // Incremental commit. properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Incremental.name()); properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.None.name()); final DataLoader dataLoader = new DataLoader(properties, store); doDurableQueueTest(dataLoader); } finally { store.__tearDownUnitTest(); } }
/** * Test durable queues using {@link CommitEnum#Incremental} and {@link ClosureEnum#None} */ public void test_durableQueues01_incrementalCommit_noClosure() throws IOException { final AbstractTripleStore store = getStore(); try { final Properties properties = new Properties(store.getProperties()); // enable durable queues. properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true"); // Incremental commit. properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Incremental.name()); properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.None.name()); final DataLoader dataLoader = new DataLoader(properties, store); doDurableQueueTest(dataLoader); } finally { store.__tearDownUnitTest(); } }
/** * Test durable queues using {@link CommitEnum#Incremental} and {@link ClosureEnum#Batch} */ public void test_durableQueues01_incrementalCommit_batchClosure() throws IOException { final AbstractTripleStore store = getStore(); try { final Properties properties = new Properties(store.getProperties()); // enable durable queues. properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true"); // Incremental commit. properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Incremental.name()); properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.Batch.name()); final DataLoader dataLoader = new DataLoader(properties, store); doDurableQueueTest(dataLoader); } finally { store.__tearDownUnitTest(); } }
/** * Test durable queues using {@link CommitEnum#Batch} and {@link ClosureEnum#Incremental} */ public void test_durableQueues02_batchCommit_incrementalClosure() throws IOException { final AbstractTripleStore store = getStore(); try { final Properties properties = new Properties(store.getProperties()); // enable durable queues. properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true"); // Batch commit. properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Batch.name()); properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.Incremental.name()); final DataLoader dataLoader = new DataLoader(properties, store); doDurableQueueTest(dataLoader); } finally { store.__tearDownUnitTest(); } }
/** * Test durable queues using {@link CommitEnum#Batch} and {@link ClosureEnum#Batch} */ public void test_durableQueues02_batchCommit_batchClosure() throws IOException { final AbstractTripleStore store = getStore(); try { final Properties properties = new Properties(store.getProperties()); // enable durable queues. properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true"); // Batch commit. properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Batch.name()); properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.Batch.name()); final DataLoader dataLoader = new DataLoader(properties, store); doDurableQueueTest(dataLoader); } finally { store.__tearDownUnitTest(); } }
/** * Test durable queues using {@link CommitEnum#Batch} and {@link ClosureEnum#Incremental} */ public void test_durableQueues02_batchCommit_incrementalClosure() throws IOException { final AbstractTripleStore store = getStore(); try { final Properties properties = new Properties(store.getProperties()); // enable durable queues. properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true"); // Batch commit. properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Batch.name()); properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.Incremental.name()); final DataLoader dataLoader = new DataLoader(properties, store); doDurableQueueTest(dataLoader); } finally { store.__tearDownUnitTest(); } }
/** * Test durable queues using {@link CommitEnum#Incremental} and {@link ClosureEnum#Batch} */ public void test_durableQueues01_incrementalCommit_batchClosure() throws IOException { final AbstractTripleStore store = getStore(); try { final Properties properties = new Properties(store.getProperties()); // enable durable queues. properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true"); // Incremental commit. properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Incremental.name()); properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.Batch.name()); final DataLoader dataLoader = new DataLoader(properties, store); doDurableQueueTest(dataLoader); } finally { store.__tearDownUnitTest(); } }
/** * Test durable queues using {@link CommitEnum#Batch} and {@link ClosureEnum#Batch} */ public void test_durableQueues02_batchCommit_batchClosure() throws IOException { final AbstractTripleStore store = getStore(); try { final Properties properties = new Properties(store.getProperties()); // enable durable queues. properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true"); // Batch commit. properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Batch.name()); properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.Batch.name()); final DataLoader dataLoader = new DataLoader(properties, store); doDurableQueueTest(dataLoader); } finally { store.__tearDownUnitTest(); } }
/** * Test durable queues using {@link CommitEnum#Batch} and {@link ClosureEnum#None} */ public void test_durableQueues02_batchCommit_noClosure() throws IOException { final AbstractTripleStore store = getStore(); try { final Properties properties = new Properties(store.getProperties()); // enable durable queues. properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true"); // Batch commit. properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Batch.name()); properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.None.name()); final DataLoader dataLoader = new DataLoader(properties, store); doDurableQueueTest(dataLoader); } finally { store.__tearDownUnitTest(); } }
/** * Test durable queues using {@link CommitEnum#Incremental} and {@link ClosureEnum#None} */ public void test_durableQueues01_incrementalCommit_noClosure() throws IOException { final AbstractTripleStore store = getStore(); try { final Properties properties = new Properties(store.getProperties()); // enable durable queues. properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true"); // Incremental commit. properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Incremental.name()); properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.None.name()); final DataLoader dataLoader = new DataLoader(properties, store); doDurableQueueTest(dataLoader); } finally { store.__tearDownUnitTest(); } }