/** * Create a table with an 'id' column full of integers and a * last_modified column with timestamps. */ private void createTimestampTable(String tableName, int insertRows, Timestamp baseTime) throws SQLException { SqoopOptions options = new SqoopOptions(); options.setConnectString(SOURCE_DB_URL); HsqldbManager manager = new HsqldbManager(options); Connection c = manager.getConnection(); PreparedStatement s = null; try { s = c.prepareStatement("CREATE TABLE " + manager.escapeTableName(tableName) + "(id INT NOT NULL, " + "last_modified TIMESTAMP)"); s.executeUpdate(); c.commit(); insertIdTimestampRows(tableName, 0, insertRows, baseTime); } finally { s.close(); } }
@Test public void testAppendWithTimestamp() throws Exception { // Create a table with data in it; import it. // Then add more data, verify that only the incremental data is pulled. final String TABLE_NAME = "appendTimestamp"; Timestamp thePast = new Timestamp(System.currentTimeMillis() - 100); createTimestampTable(TABLE_NAME, 10, thePast); List<String> args = getArgListForTable(TABLE_NAME, false, false); args.add("--append"); createJob(TABLE_NAME, args); runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 10); // Add some more rows. long importWasBefore = System.currentTimeMillis(); Thread.sleep(50); long rowsAddedTime = System.currentTimeMillis() - 5; assertTrue(rowsAddedTime > importWasBefore); assertTrue(rowsAddedTime < System.currentTimeMillis()); insertIdTimestampRows(TABLE_NAME, 10, 20, new Timestamp(rowsAddedTime)); // Import only those rows. runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 20); }
assertTrue(rowsAddedTime < System.currentTimeMillis()); insertIdTimestampRows(TABLE_NAME, 0, 10, new Timestamp(rowsAddedTime)); assertTrue(rowsAddedTime > importWasBefore); assertTrue(rowsAddedTime < System.currentTimeMillis()); insertIdTimestampRows(TABLE_NAME, 10, 20, new Timestamp(rowsAddedTime));
@Test public void testTimestampBoundary() throws Exception { // Run an import, and then insert rows with the last-modified timestamp // set to the exact time when the first import runs. Run a second import // and ensure that we pick up the new data. long now = System.currentTimeMillis(); final String TABLE_NAME = "boundaryTimestamp"; Timestamp thePast = new Timestamp(now - 100); createTimestampTable(TABLE_NAME, 10, thePast); Timestamp firstJobTime = new Timestamp(now); InstrumentHsqldbManager.setCurrentDbTimestamp(firstJobTime); // Configure the job to use the instrumented Hsqldb manager. Configuration conf = newConf(); conf.set(ConnFactory.FACTORY_CLASS_NAMES_KEY, InstrumentHsqldbManagerFactory.class.getName()); List<String> args = getArgListForTable(TABLE_NAME, false, false); args.add("--append"); createJob(TABLE_NAME, args, conf); runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 10); // Add some more rows with the timestamp equal to the job run timestamp. insertIdTimestampRows(TABLE_NAME, 10, 20, firstJobTime); assertRowCount(TABLE_NAME, 20); // Run a second job with the clock advanced by 100 ms. Timestamp secondJobTime = new Timestamp(now + 100); InstrumentHsqldbManager.setCurrentDbTimestamp(secondJobTime); // Import only those rows. runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 20); }
@Test public void testIncrementalAppendTimestamp() throws Exception { // Run an import, and then insert rows with the last-modified timestamp // set to the exact time when the first import runs. Run a second import // and ensure that we pick up the new data. long now = System.currentTimeMillis(); final String TABLE_NAME = "incrementalAppendTimestamp"; Timestamp thePast = new Timestamp(now - 100); createTimestampTable(TABLE_NAME, 10, thePast); Timestamp firstJobTime = new Timestamp(now); InstrumentHsqldbManager.setCurrentDbTimestamp(firstJobTime); // Configure the job to use the instrumented Hsqldb manager. Configuration conf = newConf(); conf.set(ConnFactory.FACTORY_CLASS_NAMES_KEY, InstrumentHsqldbManagerFactory.class.getName()); List<String> args = getArgListForTable(TABLE_NAME, false, true, true); createJob(TABLE_NAME, args, conf); runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 10); // Add some more rows with the timestamp equal to the job run timestamp. insertIdTimestampRows(TABLE_NAME, 10, 20, firstJobTime); assertRowCount(TABLE_NAME, 20); // Run a second job with the clock advanced by 100 ms. Timestamp secondJobTime = new Timestamp(now + 100); InstrumentHsqldbManager.setCurrentDbTimestamp(secondJobTime); // Import only those rows. runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 20); } @Test