/** * Read a collection {@link Token}s from a given file. * * @param tokenFilePath the token file path * @param configuration a {@link Configuration} object carrying Hadoop configuration properties * @return a collection of {@link Token}s * @throws IOException */ public static Collection<Token<? extends TokenIdentifier>> readTokensFromFile(Path tokenFilePath, Configuration configuration) throws IOException { return Credentials.readTokenStorageFile(tokenFilePath, configuration).getAllTokens(); }
/** * Obtain the collection of tokens associated with this user. * * @return an unmodifiable collection of tokens associated with user */ public Collection<Token<? extends TokenIdentifier>> getTokens() { synchronized (subject) { return Collections.unmodifiableCollection( new ArrayList<Token<?>>(getCredentialsInternal().getAllTokens())); } }
@Override public org.apache.hadoop.security.token.Token<? extends TokenIdentifier> selectDelegationToken(URL url, Credentials creds) { if (LOG.isDebugEnabled()) { LOG.debug("Looking for delegation token. creds: {}", creds.getAllTokens()); } // clientTokenProvider is either "this" or a load balancing instance. // if the latter, it will first look for the load balancer's uri // service followed by each sub-provider for backwards-compatibility. return clientTokenProvider.selectDelegationToken(creds); } };
private boolean containsKmsDt(UserGroupInformation ugi) throws IOException { // Add existing credentials from the UGI, since provider is cached. Credentials creds = ugi.getCredentials(); if (!creds.getAllTokens().isEmpty()) { LOG.debug("Searching for KMS delegation token in user {}'s credentials", ugi); return clientTokenProvider.selectDelegationToken(creds) != null; } return false; }
if (privateCredentials != null) { for (Credentials cred : privateCredentials) { Collection<Token<? extends TokenIdentifier>> allTokens = cred.getAllTokens(); if (allTokens != null) { for (Token<? extends TokenIdentifier> token : allTokens) {
protected static Token<?> selectDelegationToken(Credentials creds, Text service) { Token<?> token = creds.getToken(service); LOG.debug("selected by alias={} token={}", service, token); if (token != null && TOKEN_KIND.equals(token.getKind())) { return token; } token = TokenSelector.INSTANCE.selectToken(service, creds.getAllTokens()); LOG.debug("selected by service={} token={}", service, token); return token; }
/** * Obtain the tokens in credentials form associated with this user. * * @return Credentials of tokens associated with this user */ public Credentials getCredentials() { synchronized (subject) { Credentials creds = new Credentials(getCredentialsInternal()); Iterator<Token<?>> iter = creds.getAllTokens().iterator(); while (iter.hasNext()) { if (iter.next().isPrivate()) { iter.remove(); } } return creds; } }
private ByteBuffer getSecurityTokens() throws IOException { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); Closer closer = Closer.create(); try { DataOutputBuffer dataOutputBuffer = closer.register(new DataOutputBuffer()); credentials.writeTokenStorageToStream(dataOutputBuffer); // Remove the AM->RM token so that containers cannot access it Iterator<Token<?>> tokenIterator = credentials.getAllTokens().iterator(); while (tokenIterator.hasNext()) { Token<?> token = tokenIterator.next(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { tokenIterator.remove(); } } return ByteBuffer.wrap(dataOutputBuffer.getData(), 0, dataOutputBuffer.getLength()); } catch (Throwable t) { throw closer.rethrow(t); } finally { closer.close(); } }
for (Token<? extends TokenIdentifier> tokenForLog : credential.getAllTokens()) { LOG.debug("Obtained token info in credential: {} / {}", tokenForLog.toString(), tokenForLog.decodeIdentifier().getUser());
/** * Dump all tokens of a UGI. * @param ugi UGI to examine */ public void dumpTokens(UserGroupInformation ugi) { Collection<Token<? extends TokenIdentifier>> tokens = ugi.getCredentials().getAllTokens(); title("Token Count: %d", tokens.size()); for (Token<? extends TokenIdentifier> token : tokens) { println("Token %s", token.getKind()); } endln(); }
/** * {@inheritDoc} */ @Override public void doRenew(Map<String, String> credentials, Map<String, Object> topologyConf, final String topologyOwnerPrincipal) { List<String> confKeys = getConfigKeys(topologyConf); for (Pair<String, Credentials> cred : getCredentials(credentials, confKeys)) { try { Configuration configuration = getHadoopConfiguration(topologyConf, cred.getFirst()); Collection<Token<? extends TokenIdentifier>> tokens = cred.getSecond().getAllTokens(); if (tokens != null && !tokens.isEmpty()) { for (Token<? extends TokenIdentifier> token : tokens) { //We need to re-login some other thread might have logged into hadoop using // their credentials (e.g. AutoHBase might be also part of nimbu auto creds) login(configuration); long expiration = token.renew(configuration); LOG.info("HDFS delegation token renewed, new expiration time {}", expiration); } } else { LOG.debug("No tokens found for credentials, skipping renewal."); } } catch (Exception e) { LOG.warn("could not renew the credentials, one of the possible reason is tokens are beyond " + "renewal period so attempting to get new tokens.", e); populateCredentials(credentials, topologyConf, topologyOwnerPrincipal); } } }
@Override public void doRenew(Map<String, String> credentials, Map<String, Object> topologyConf, final String topologyOwnerPrincipal) { List<String> configKeys = getConfigKeys(topologyConf); for (Pair<String, Credentials> cred : getCredentials(credentials, configKeys)) { try { Configuration configuration = getHadoopConfiguration(topologyConf, cred.getFirst()); String hiveMetaStoreURI = getMetaStoreURI(configuration); String hiveMetaStorePrincipal = getMetaStorePrincipal(configuration); Collection<Token<? extends TokenIdentifier>> tokens = cred.getSecond().getAllTokens(); login(configuration); if (tokens != null && !tokens.isEmpty()) { for (Token<? extends TokenIdentifier> token : tokens) { long expiration = renewToken(token, hiveMetaStoreURI, hiveMetaStorePrincipal); LOG.info("Hive delegation token renewed, new expiration time {}", expiration); } } else { LOG.debug("No tokens found for credentials, skipping renewal."); } } catch (Exception e) { LOG.warn("could not renew the credentials, one of the possible reason is tokens are beyond " + "renewal period so attempting to get new tokens.", e); populateCredentials(credentials, topologyConf); } } }
if (LOG.isDebugEnabled()) { LOG.debug("Token not set, looking for delegation token. Creds:{}," + " size:{}", creds.getAllTokens(), creds.numberOfTokens()); if (!creds.getAllTokens().isEmpty()) { dToken = selectDelegationToken(url, creds); if (dToken != null) {
/** Print out a Credentials object. * @param creds the Credentials object to be printed out. * @param alias print only tokens matching alias (null matches all). * @param out print to this stream. * @throws IOException */ public static void printCredentials( Credentials creds, Text alias, PrintStream out) throws IOException { boolean tokenHeader = true; String fmt = "%-24s %-20s %-15s %-12s %s%n"; for (Token<?> token : creds.getAllTokens()) { if (matchAlias(token, alias)) { if (tokenHeader) { out.printf(fmt, "Token kind", "Service", "Renewer", "Exp date", "URL enc token"); out.println(StringUtils.repeat("-", 80)); tokenHeader = false; } AbstractDelegationTokenIdentifier id = (AbstractDelegationTokenIdentifier) token.decodeIdentifier(); out.printf(fmt, token.getKind(), token.getService(), (id != null) ? id.getRenewer() : NA_STRING, (id != null) ? formatDate(id.getMaxDate()) : NA_STRING, token.encodeToUrlString()); } } }
/** Append tokens from list of files in local filesystem, saving to last file. * @param tokenFiles list of local File objects. Last file holds the output. * @param fileFormat a string equal to FORMAT_PB or FORMAT_JAVA, for output * @param conf Configuration object passed along. * @throws IOException */ public static void appendTokenFiles( ArrayList<File> tokenFiles, String fileFormat, Configuration conf) throws IOException { Credentials newCreds = new Credentials(); File lastTokenFile = null; for (File tokenFile : tokenFiles) { lastTokenFile = tokenFile; Credentials creds = Credentials.readTokenStorageFile(tokenFile, conf); for (Token<?> token : creds.getAllTokens()) { newCreds.addToken(token.getService(), token); } } doFormattedWrite(lastTokenFile, fileFormat, newCreds, conf); }
public static void cancelTokens(State state) throws IOException, InterruptedException, TException { Preconditions.checkArgument(state.contains(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION), "Missing required property " + ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION); Preconditions.checkArgument(state.contains(ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_SUPER_USER), "Missing required property " + ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_SUPER_USER); Preconditions.checkArgument(state.contains(ConfigurationKeys.KERBEROS_REALM), "Missing required property " + ConfigurationKeys.KERBEROS_REALM); String superUser = state.getProp(ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_SUPER_USER); String keytabLocation = state.getProp(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION); String realm = state.getProp(ConfigurationKeys.KERBEROS_REALM); UserGroupInformation.loginUserFromKeytab(HostUtils.getPrincipalUsingHostname(superUser, realm), keytabLocation); UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); UserGroupInformation realUser = currentUser.getRealUser(); Credentials credentials = realUser.getCredentials(); for (Token<?> token : credentials.getAllTokens()) { if (token.getKind().equals(DelegationTokenIdentifier.HIVE_DELEGATION_KIND)) { log.info("Cancelling hive token"); HiveMetaStoreClient hiveClient = new HiveMetaStoreClient(new HiveConf()); hiveClient.cancelDelegationToken(token.encodeToUrlString()); } } }
/** Renew a token from a file in the local filesystem, matching alias. * @param tokenFile a local File object. * @param fileFormat a string equal to FORMAT_PB or FORMAT_JAVA, for output * @param alias renew only tokens matching alias; null matches all. * @param conf Configuration object passed along. * @throws IOException * @throws InterruptedException */ public static void renewTokenFile( File tokenFile, String fileFormat, Text alias, Configuration conf) throws IOException, InterruptedException { Credentials creds = Credentials.readTokenStorageFile(tokenFile, conf); for (Token<?> token : creds.getAllTokens()) { if (token.isManaged() && matchAlias(token, alias)) { long result = token.renew(conf); LOG.info("Renewed" + token.getKind() + ":" + token.getService() + " until " + formatDate(result)); } } doFormattedWrite(tokenFile, fileFormat, creds, conf); } }
/** Alias a token from a file and save back to file in the local filesystem. * @param tokenFile a local File object to hold the input and output. * @param fileFormat a string equal to FORMAT_PB or FORMAT_JAVA, for output * @param alias overwrite service field of fetched token with this text. * @param service only apply alias to tokens matching this service text. * @param conf Configuration object passed along. * @throws IOException */ public static void aliasTokenFile(File tokenFile, String fileFormat, Text alias, Text service, Configuration conf) throws Exception { Credentials newCreds = new Credentials(); Credentials creds = Credentials.readTokenStorageFile(tokenFile, conf); for (Token<?> token : creds.getAllTokens()) { newCreds.addToken(token.getService(), token); if (token.getService().equals(service)) { Token<?> aliasedToken = token.copyToken(); aliasedToken.setService(alias); newCreds.addToken(alias, aliasedToken); } } doFormattedWrite(tokenFile, fileFormat, newCreds, conf); }
credentials.writeTokenStorageToStream(dob); Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); LOG.info("Executing with tokens:"); while (iter.hasNext()) {
/** Remove a token from a file in the local filesystem, matching alias. * @param cancel cancel token as well as remove from file. * @param tokenFile a local File object. * @param fileFormat a string equal to FORMAT_PB or FORMAT_JAVA, for output * @param alias remove only tokens matching alias; null matches all. * @param conf Configuration object passed along. * @throws IOException * @throws InterruptedException */ public static void removeTokenFromFile(boolean cancel, File tokenFile, String fileFormat, Text alias, Configuration conf) throws IOException, InterruptedException { Credentials newCreds = new Credentials(); Credentials creds = Credentials.readTokenStorageFile(tokenFile, conf); for (Token<?> token : creds.getAllTokens()) { if (matchAlias(token, alias)) { if (token.isManaged() && cancel) { token.cancel(conf); LOG.info("Canceled " + token.getKind() + ":" + token.getService()); } } else { newCreds.addToken(token.getService(), token); } } doFormattedWrite(tokenFile, fileFormat, newCreds, conf); }