@Test public void testLockRecovery() throws Exception { DirLock lock1 = DirLock.tryLock(fs, locksDir); // should pass Assert.assertNotNull(lock1); DirLock lock2 = DirLock.takeOwnershipIfStale(fs, locksDir, LOCK_EXPIRY_SEC); // should fail Assert.assertNull(lock2); Thread.sleep(LOCK_EXPIRY_SEC * 1000 + 500); // wait for lock to expire Assert.assertTrue(fs.exists(lock1.getLockFile())); DirLock lock3 = DirLock.takeOwnershipIfStale(fs, locksDir, LOCK_EXPIRY_SEC); // should pass now Assert.assertNotNull(lock3); Assert.assertTrue(fs.exists(lock3.getLockFile())); lock3.release(); Assert.assertFalse(fs.exists(lock3.getLockFile())); lock1.release(); // should not throw }
/** Get a lock on file if not already locked * * @param fs * @param dir the dir on which to get a lock * @return The lock object if it the lock was acquired. Returns null if the dir is already locked. * @throws IOException if there were errors */ public static DirLock tryLock(FileSystem fs, Path dir) throws IOException { Path lockFile = getDirLockFile(dir); try { FSDataOutputStream ostream = HdfsUtils.tryCreateFile(fs, lockFile); if (ostream != null) { LOG.debug("Thread ({}) Acquired lock on dir {}", threadInfo(), dir); ostream.close(); return new DirLock(fs, lockFile); } else { LOG.debug("Thread ({}) cannot lock dir {} as its already locked.", threadInfo(), dir); return null; } } catch (IOException e) { LOG.error("Error when acquiring lock on dir " + dir, e); throw e; } }
/** if the lock on the directory is stale, take ownership */ public static DirLock takeOwnershipIfStale(FileSystem fs, Path dirToLock, int lockTimeoutSec) { Path dirLockFile = getDirLockFile(dirToLock); long now = System.currentTimeMillis(); long expiryTime = now - (lockTimeoutSec * 1000); try { long modTime = fs.getFileStatus(dirLockFile).getModificationTime(); if (modTime <= expiryTime) { return takeOwnership(fs, dirLockFile); } return null; } catch (IOException e) { return null; } }
@Test public void testBasicLocking() throws Exception { // 1 grab lock DirLock lock = DirLock.tryLock(fs, locksDir); Assert.assertTrue(fs.exists(lock.getLockFile())); // 2 try to grab another lock while dir is locked DirLock lock2 = DirLock.tryLock(fs, locksDir); // should fail Assert.assertNull(lock2); // 3 let go first lock lock.release(); Assert.assertFalse(fs.exists(lock.getLockFile())); // 4 try locking again lock2 = DirLock.tryLock(fs, locksDir); Assert.assertTrue(fs.exists(lock2.getLockFile())); lock2.release(); Assert.assertFalse(fs.exists(lock.getLockFile())); lock2.release(); // should be throw }
@Override public void run() { Thread.currentThread().setName("DirLockingThread-" + thdNum); DirLock lock = null; try { do { System.err.println("Trying lock " + getName()); lock = DirLock.tryLock(fs, dir); System.err.println("Acquired lock " + getName()); if (lock == null) { System.out.println("Retrying lock - " + getName()); } } while (lock == null && !Thread.currentThread().isInterrupted()); cleanExit = true; } catch (IOException e) { e.printStackTrace(); } finally { try { if (lock != null) { lock.release(); System.err.println("Released lock " + getName()); } } catch (IOException e) { e.printStackTrace(System.err); } } System.err.println("Thread exiting " + getName()); } // run()
/** Release lock on dir by deleting the lock file */ public void release() throws IOException { if (!fs.delete(lockFile, false)) { LOG.error("Thread {} could not delete dir lock {} ", threadInfo(), lockFile); } else { LOG.debug("Thread {} Released dir lock {} ", threadInfo(), lockFile); } }
private static DirLock takeOwnership(FileSystem fs, Path dirLockFile) throws IOException { if (fs instanceof DistributedFileSystem) { if (!((DistributedFileSystem) fs).recoverLease(dirLockFile)) { LOG.warn("Unable to recover lease on dir lock file " + dirLockFile + " right now. Cannot transfer ownership. Will need to try later."); return null; } } // delete and recreate lock file if (fs.delete(dirLockFile, false)) { // returns false if somebody else already deleted it (to take ownership) FSDataOutputStream ostream = HdfsUtils.tryCreateFile(fs, dirLockFile); if (ostream != null) { ostream.close(); } return new DirLock(fs, dirLockFile); } return null; }
/** Release lock on dir by deleting the lock file */ public void release() throws IOException { if(!fs.delete(lockFile, false)) { LOG.error("Thread {} could not delete dir lock {} ", threadInfo(), lockFile); } else { LOG.debug("Thread {} Released dir lock {} ", threadInfo(), lockFile); } }
private static DirLock takeOwnership(FileSystem fs, Path dirLockFile) throws IOException { if(fs instanceof DistributedFileSystem) { if (!((DistributedFileSystem) fs).recoverLease(dirLockFile)) { LOG.warn("Unable to recover lease on dir lock file " + dirLockFile + " right now. Cannot transfer ownership. Will need to try later."); return null; } } // delete and recreate lock file if( fs.delete(dirLockFile, false) ) { // returns false if somebody else already deleted it (to take ownership) FSDataOutputStream ostream = HdfsUtils.tryCreateFile(fs, dirLockFile); if(ostream!=null) { ostream.close(); } return new DirLock(fs, dirLockFile); } return null; }
DirLock dirlock = DirLock.tryLock(hdfs, lockDirPath); if (dirlock == null) { dirlock = DirLock.takeOwnershipIfStale(hdfs, lockDirPath, lockTimeoutSec); if (dirlock == null) { LOG.debug("Spout {} could not take over ownership of DirLock for {}", spoutId, lockDirPath); dirlock.release(); LOG.debug("Released DirLock {}, SpoutID {} ", dirlock.getLockFile(), spoutId);
/** Get a lock on file if not already locked * * @param fs * @param dir the dir on which to get a lock * @return The lock object if it the lock was acquired. Returns null if the dir is already locked. * @throws IOException if there were errors */ public static DirLock tryLock(FileSystem fs, Path dir) throws IOException { Path lockFile = getDirLockFile(dir); try { FSDataOutputStream ostream = HdfsUtils.tryCreateFile(fs, lockFile); if (ostream!=null) { LOG.debug("Thread ({}) Acquired lock on dir {}", threadInfo(), dir); ostream.close(); return new DirLock(fs, lockFile); } else { LOG.debug("Thread ({}) cannot lock dir {} as its already locked.", threadInfo(), dir); return null; } } catch (IOException e) { LOG.error("Error when acquiring lock on dir " + dir, e); throw e; } }
/** if the lock on the directory is stale, take ownership */ public static DirLock takeOwnershipIfStale(FileSystem fs, Path dirToLock, int lockTimeoutSec) { Path dirLockFile = getDirLockFile(dirToLock); long now = System.currentTimeMillis(); long expiryTime = now - (lockTimeoutSec*1000); try { long modTime = fs.getFileStatus(dirLockFile).getModificationTime(); if(modTime <= expiryTime) { return takeOwnership(fs, dirLockFile); } return null; } catch (IOException e) { return null; } }
DirLock dirlock = DirLock.tryLock(hdfs, lockDirPath); if (dirlock == null) { dirlock = DirLock.takeOwnershipIfStale(hdfs, lockDirPath, lockTimeoutSec); if (dirlock == null) { LOG.debug("Spout {} could not take over ownership of DirLock for {}", spoutId, lockDirPath); dirlock.release(); LOG.debug("Released DirLock {}, SpoutID {} ", dirlock.getLockFile(), spoutId);