protected DelegationTokenIdentifier getTokenIdentifier(Token<DelegationTokenIdentifier> token) throws IOException { // turn bytes back into identifier for cache lookup ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier()); DataInputStream in = new DataInputStream(buf); DelegationTokenIdentifier id = createIdentifier(); id.readFields(in); return id; }
@Override public List<DelegationTokenIdentifier> getAllDelegationTokenIdentifiers() { String containerNode = rootNode + NODE_TOKENS; final List<String> nodes = zkGetChildren(containerNode); List<DelegationTokenIdentifier> result = new java.util.ArrayList<DelegationTokenIdentifier>( nodes.size()); for (String node : nodes) { DelegationTokenIdentifier id = new DelegationTokenIdentifier(); try { TokenStoreDelegationTokenSecretManager.decodeWritable(id, node); result.add(id); } catch (Exception e) { LOGGER.warn("Failed to decode token '{}'", node); } } return result; }
private String getTokenPath(DelegationTokenIdentifier tokenIdentifier) { try { return rootNode + NODE_TOKENS + "/" + TokenStoreDelegationTokenSecretManager.encodeWritable(tokenIdentifier); } catch (IOException ex) { throw new TokenStoreException("Failed to encode token identifier", ex); } }
if (lastMasterKeyUpdate + keyUpdateInterval < now) { try { rollMasterKeyExt(); lastMasterKeyUpdate = now; } catch (IOException e) { removeExpiredTokens(); lastTokenCacheCleanup = now;
/** * Extension of rollMasterKey to remove expired keys from store. * @throws IOException */ protected void rollMasterKeyExt() throws IOException { Map<Integer, DelegationKey> keys = reloadKeys(); int currentKeyId = super.currentId; HiveDelegationTokenSupport.rollMasterKey(TokenStoreDelegationTokenSecretManager.this); List<DelegationKey> keysAfterRoll = Arrays.asList(getAllKeys()); for (DelegationKey key : keysAfterRoll) { keys.remove(key.getKeyId()); if (key.getKeyId() == currentKeyId) { tokenStore.updateMasterKey(currentKeyId, encodeWritable(key)); } } for (DelegationKey expiredKey : keys.values()) { LOGGER.info("Removing expired key id={}", expiredKey.getKeyId()); tokenStore.removeMasterKey(expiredKey.getKeyId()); } }
/** * Synchronize master key updates / sequence generation for multiple nodes. * NOTE: {@Link AbstractDelegationTokenSecretManager} keeps currentKey private, so we need * to utilize this "hook" to manipulate the key through the object reference. * This .20S workaround should cease to exist when Hadoop supports token store. */ @Override protected void logUpdateMasterKey(DelegationKey key) throws IOException { int keySeq = this.tokenStore.addMasterKey(encodeWritable(key)); // update key with assigned identifier DelegationKey keyWithSeq = new DelegationKey(keySeq, key.getExpiryDate(), key.getKey()); String keyStr = encodeWritable(keyWithSeq); this.tokenStore.updateMasterKey(keySeq, keyStr); decodeWritable(key, keyStr); LOGGER.info("New master key with key id={}", key.getKeyId()); super.logUpdateMasterKey(key); }
@Override public long renewToken(Token<DelegationTokenIdentifier> token, String renewer) throws InvalidToken, IOException { // since renewal is KERBEROS authenticated token may not be cached final DelegationTokenIdentifier id = getTokenIdentifier(token); DelegationTokenInformation tokenInfo = this.tokenStore.getToken(id); if (tokenInfo == null) { throw new InvalidToken("token does not exist: " + id); // no token found } // ensure associated master key is available if (!super.allKeys.containsKey(id.getMasterKeyId())) { LOGGER.info("Unknown master key (id={}), (re)loading keys from token store.", id.getMasterKeyId()); reloadKeys(); } // reuse super renewal logic synchronized (this) { super.currentTokens.put(id, tokenInfo); try { return super.renewToken(token, renewer); } finally { super.currentTokens.remove(id); } } }
@Override public DelegationTokenIdentifier cancelToken(Token<DelegationTokenIdentifier> token, String canceller) throws IOException { DelegationTokenIdentifier id = getTokenIdentifier(token); LOGGER.info("Token cancelation requested for identifier: "+id); this.tokenStore.removeToken(id); return id; }
public void startDelegationTokenSecretManager(Configuration conf, Object rawStore, ServerMode smode) throws IOException{ long secretKeyInterval = conf.getLong(DELEGATION_KEY_UPDATE_INTERVAL_KEY, DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT); long tokenMaxLifetime = conf.getLong(DELEGATION_TOKEN_MAX_LIFETIME_KEY, DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT); long tokenRenewInterval = conf.getLong(DELEGATION_TOKEN_RENEW_INTERVAL_KEY, DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT); long tokenGcInterval = conf.getLong(DELEGATION_TOKEN_GC_INTERVAL, DELEGATION_TOKEN_GC_INTERVAL_DEFAULT); DelegationTokenStore dts = getTokenStore(conf); dts.init(rawStore, smode); secretManager = new TokenStoreDelegationTokenSecretManager(secretKeyInterval, tokenMaxLifetime, tokenRenewInterval, tokenGcInterval, dts); secretManager.startThreads(); }
/** * Extension of rollMasterKey to remove expired keys from store. * * @throws IOException */ protected void rollMasterKeyExt() throws IOException { Map<Integer, DelegationKey> keys = reloadKeys(); int currentKeyId = super.currentId; HiveDelegationTokenSupport.rollMasterKey(TokenStoreDelegationTokenSecretManager.this); List<DelegationKey> keysAfterRoll = Arrays.asList(getAllKeys()); for (DelegationKey key : keysAfterRoll) { keys.remove(key.getKeyId()); if (key.getKeyId() == currentKeyId) { tokenStore.updateMasterKey(currentKeyId, encodeWritable(key)); } } for (DelegationKey expiredKey : keys.values()) { LOGGER.info("Removing expired key id={}", expiredKey.getKeyId()); try { tokenStore.removeMasterKey(expiredKey.getKeyId()); } catch (Exception e) { LOGGER.error("Error removing expired key id={}", expiredKey.getKeyId(), e); } } }
/** * Synchronize master key updates / sequence generation for multiple nodes. * NOTE: {@Link AbstractDelegationTokenSecretManager} keeps currentKey private, so we need * to utilize this "hook" to manipulate the key through the object reference. * This .20S workaround should cease to exist when Hadoop supports token store. */ @Override protected void logUpdateMasterKey(DelegationKey key) throws IOException { int keySeq = this.tokenStore.addMasterKey(encodeWritable(key)); // update key with assigned identifier DelegationKey keyWithSeq = new DelegationKey(keySeq, key.getExpiryDate(), key.getKey()); String keyStr = encodeWritable(keyWithSeq); this.tokenStore.updateMasterKey(keySeq, keyStr); decodeWritable(key, keyStr); LOGGER.info("New master key with key id={}", key.getKeyId()); super.logUpdateMasterKey(key); }
@Override public long renewToken(Token<DelegationTokenIdentifier> token, String renewer) throws InvalidToken, IOException { // since renewal is KERBEROS authenticated token may not be cached final DelegationTokenIdentifier id = getTokenIdentifier(token); DelegationTokenInformation tokenInfo = this.tokenStore.getToken(id); if (tokenInfo == null) { throw new InvalidToken("token does not exist: " + id); // no token found } // ensure associated master key is available if (!super.allKeys.containsKey(id.getMasterKeyId())) { LOGGER.info("Unknown master key (id={}), (re)loading keys from token store.", id.getMasterKeyId()); reloadKeys(); } // reuse super renewal logic synchronized (this) { super.currentTokens.put(id, tokenInfo); try { return super.renewToken(token, renewer); } finally { super.currentTokens.remove(id); } } }
if (lastMasterKeyUpdate + keyUpdateInterval < now) { try { rollMasterKeyExt(); lastMasterKeyUpdate = now; } catch (IOException e) { removeExpiredTokens(); lastTokenCacheCleanup = now;
@Override public DelegationTokenIdentifier cancelToken(Token<DelegationTokenIdentifier> token, String canceller) throws IOException { DelegationTokenIdentifier id = getTokenIdentifier(token); LOGGER.info("Token cancelation requested for identifier: "+id); this.tokenStore.removeToken(id); return id; }
public void startDelegationTokenSecretManager(Configuration conf, Object rawStore, ServerMode smode) throws IOException{ long secretKeyInterval = conf.getLong(DELEGATION_KEY_UPDATE_INTERVAL_KEY, DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT); long tokenMaxLifetime = conf.getLong(DELEGATION_TOKEN_MAX_LIFETIME_KEY, DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT); long tokenRenewInterval = conf.getLong(DELEGATION_TOKEN_RENEW_INTERVAL_KEY, DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT); long tokenGcInterval = conf.getLong(DELEGATION_TOKEN_GC_INTERVAL, DELEGATION_TOKEN_GC_INTERVAL_DEFAULT); DelegationTokenStore dts = getTokenStore(conf); dts.init(rawStore, smode); secretManager = new TokenStoreDelegationTokenSecretManager(secretKeyInterval, tokenMaxLifetime, tokenRenewInterval, tokenGcInterval, dts); secretManager.startThreads(); }
/** * Extension of rollMasterKey to remove expired keys from store. * * @throws IOException */ protected void rollMasterKeyExt() throws IOException { Map<Integer, DelegationKey> keys = reloadKeys(); int currentKeyId = super.currentId; HiveDelegationTokenSupport.rollMasterKey(TokenStoreDelegationTokenSecretManager.this); List<DelegationKey> keysAfterRoll = Arrays.asList(getAllKeys()); for (DelegationKey key : keysAfterRoll) { keys.remove(key.getKeyId()); if (key.getKeyId() == currentKeyId) { tokenStore.updateMasterKey(currentKeyId, encodeWritable(key)); } } for (DelegationKey expiredKey : keys.values()) { LOGGER.info("Removing expired key id={}", expiredKey.getKeyId()); try { tokenStore.removeMasterKey(expiredKey.getKeyId()); } catch (Exception e) { LOGGER.error("Error removing expired key id={}", expiredKey.getKeyId(), e); } } }
private String getTokenPath(DelegationTokenIdentifier tokenIdentifier) { try { return rootNode + NODE_TOKENS + "/" + TokenStoreDelegationTokenSecretManager.encodeWritable(tokenIdentifier); } catch (IOException ex) { throw new TokenStoreException("Failed to encode token identifier", ex); } }
/** * Synchronize master key updates / sequence generation for multiple nodes. * NOTE: {@Link AbstractDelegationTokenSecretManager} keeps currentKey private, so we need * to utilize this "hook" to manipulate the key through the object reference. * This .20S workaround should cease to exist when Hadoop supports token store. */ @Override protected void logUpdateMasterKey(DelegationKey key) throws IOException { int keySeq = this.tokenStore.addMasterKey(encodeWritable(key)); // update key with assigned identifier DelegationKey keyWithSeq = new DelegationKey(keySeq, key.getExpiryDate(), key.getKey()); String keyStr = encodeWritable(keyWithSeq); this.tokenStore.updateMasterKey(keySeq, keyStr); decodeWritable(key, keyStr); LOGGER.info("New master key with key id={}", key.getKeyId()); super.logUpdateMasterKey(key); }
@Override public long renewToken(Token<DelegationTokenIdentifier> token, String renewer) throws InvalidToken, IOException { // since renewal is KERBEROS authenticated token may not be cached final DelegationTokenIdentifier id = getTokenIdentifier(token); DelegationTokenInformation tokenInfo = this.tokenStore.getToken(id); if (tokenInfo == null) { throw new InvalidToken("token does not exist: " + id); // no token found } // ensure associated master key is available if (!super.allKeys.containsKey(id.getMasterKeyId())) { LOGGER.info("Unknown master key (id={}), (re)loading keys from token store.", id.getMasterKeyId()); reloadKeys(); } // reuse super renewal logic synchronized (this) { super.currentTokens.put(id, tokenInfo); try { return super.renewToken(token, renewer); } finally { super.currentTokens.remove(id); } } }
if (lastMasterKeyUpdate + keyUpdateInterval < now) { try { rollMasterKeyExt(); lastMasterKeyUpdate = now; } catch (IOException e) { removeExpiredTokens(); lastTokenCacheCleanup = now;