/** * Register an application with its secret specified as a byte buffer. */ public void registerApp(String appId, ByteBuffer shuffleSecret) { registerApp(appId, JavaUtils.bytesToString(shuffleSecret)); }
/** * Register an application with its secret specified as a byte buffer. */ public void registerApp(String appId, ByteBuffer shuffleSecret) { registerApp(appId, JavaUtils.bytesToString(shuffleSecret)); }
/** * Register an application with its secret specified as a byte buffer. */ public void registerApp(String appId, ByteBuffer shuffleSecret) { registerApp(appId, JavaUtils.bytesToString(shuffleSecret)); }
private void loadSecretsFromDb() throws IOException { secretsFile = initRecoveryDb(SECRETS_RECOVERY_FILE_NAME); // Make sure this is protected in case its not in the NM recovery dir FileSystem fs = FileSystem.getLocal(_conf); fs.mkdirs(new Path(secretsFile.getPath()), new FsPermission((short) 0700)); db = LevelDBProvider.initLevelDB(secretsFile, CURRENT_VERSION, mapper); logger.info("Recovery location is: " + secretsFile.getPath()); if (db != null) { logger.info("Going to reload spark shuffle data"); DBIterator itr = db.iterator(); itr.seek(APP_CREDS_KEY_PREFIX.getBytes(StandardCharsets.UTF_8)); while (itr.hasNext()) { Map.Entry<byte[], byte[]> e = itr.next(); String key = new String(e.getKey(), StandardCharsets.UTF_8); if (!key.startsWith(APP_CREDS_KEY_PREFIX)) { break; } String id = parseDbAppKey(key); ByteBuffer secret = mapper.readValue(e.getValue(), ByteBuffer.class); logger.info("Reloading tokens for app: " + id); secretManager.registerApp(id, secret); } } }
private void createSecretManager() throws IOException { secretManager = new ShuffleSecretManager(); secretsFile = initRecoveryDb(SECRETS_RECOVERY_FILE_NAME); // Make sure this is protected in case its not in the NM recovery dir FileSystem fs = FileSystem.getLocal(_conf); fs.mkdirs(new Path(secretsFile.getPath()), new FsPermission((short)0700)); db = LevelDBProvider.initLevelDB(secretsFile, CURRENT_VERSION, mapper); logger.info("Recovery location is: " + secretsFile.getPath()); if (db != null) { logger.info("Going to reload spark shuffle data"); DBIterator itr = db.iterator(); itr.seek(APP_CREDS_KEY_PREFIX.getBytes(StandardCharsets.UTF_8)); while (itr.hasNext()) { Map.Entry<byte[], byte[]> e = itr.next(); String key = new String(e.getKey(), StandardCharsets.UTF_8); if (!key.startsWith(APP_CREDS_KEY_PREFIX)) { break; } String id = parseDbAppKey(key); ByteBuffer secret = mapper.readValue(e.getValue(), ByteBuffer.class); logger.info("Reloading tokens for app: " + id); secretManager.registerApp(id, secret); } } }
private void createSecretManager() throws IOException { secretManager = new ShuffleSecretManager(); secretsFile = initRecoveryDb(SECRETS_RECOVERY_FILE_NAME); // Make sure this is protected in case its not in the NM recovery dir FileSystem fs = FileSystem.getLocal(_conf); fs.mkdirs(new Path(secretsFile.getPath()), new FsPermission((short)0700)); db = LevelDBProvider.initLevelDB(secretsFile, CURRENT_VERSION, mapper); logger.info("Recovery location is: " + secretsFile.getPath()); if (db != null) { logger.info("Going to reload spark shuffle data"); DBIterator itr = db.iterator(); itr.seek(APP_CREDS_KEY_PREFIX.getBytes(StandardCharsets.UTF_8)); while (itr.hasNext()) { Map.Entry<byte[], byte[]> e = itr.next(); String key = new String(e.getKey(), StandardCharsets.UTF_8); if (!key.startsWith(APP_CREDS_KEY_PREFIX)) { break; } String id = parseDbAppKey(key); ByteBuffer secret = mapper.readValue(e.getValue(), ByteBuffer.class); logger.info("Reloading tokens for app: " + id); secretManager.registerApp(id, secret); } } }
@Override public void initializeApplication(ApplicationInitializationContext context) { String appId = context.getApplicationId().toString(); try { ByteBuffer shuffleSecret = context.getApplicationDataForService(); logger.info("Initializing application {}", appId); if (isAuthenticationEnabled()) { AppId fullId = new AppId(appId); if (db != null) { byte[] key = dbAppKey(fullId); byte[] value = mapper.writeValueAsString(shuffleSecret).getBytes(StandardCharsets.UTF_8); db.put(key, value); } secretManager.registerApp(appId, shuffleSecret); } } catch (Exception e) { logger.error("Exception when initializing application {}", appId, e); } }
@Override public void initializeApplication(ApplicationInitializationContext context) { String appId = context.getApplicationId().toString(); try { ByteBuffer shuffleSecret = context.getApplicationDataForService(); if (isAuthenticationEnabled()) { AppId fullId = new AppId(appId); if (db != null) { byte[] key = dbAppKey(fullId); byte[] value = mapper.writeValueAsString(shuffleSecret).getBytes(StandardCharsets.UTF_8); db.put(key, value); } secretManager.registerApp(appId, shuffleSecret); } } catch (Exception e) { logger.error("Exception when initializing application {}", appId, e); } }
@Override public void initializeApplication(ApplicationInitializationContext context) { String appId = context.getApplicationId().toString(); try { ByteBuffer shuffleSecret = context.getApplicationDataForService(); if (isAuthenticationEnabled()) { AppId fullId = new AppId(appId); if (db != null) { byte[] key = dbAppKey(fullId); byte[] value = mapper.writeValueAsString(shuffleSecret).getBytes(StandardCharsets.UTF_8); db.put(key, value); } secretManager.registerApp(appId, shuffleSecret); } } catch (Exception e) { logger.error("Exception when initializing application {}", appId, e); } }
@Test public void testMultipleRegisters() { ShuffleSecretManager secretManager = new ShuffleSecretManager(); secretManager.registerApp(app1, pw1); assertEquals(pw1, secretManager.getSecretKey(app1)); secretManager.registerApp(app2, ByteBuffer.wrap(pw2.getBytes())); assertEquals(pw2, secretManager.getSecretKey(app2)); // now update the password for the apps and make sure it takes affect secretManager.registerApp(app1, pw1update); assertEquals(pw1update, secretManager.getSecretKey(app1)); secretManager.registerApp(app2, ByteBuffer.wrap(pw2update.getBytes())); assertEquals(pw2update, secretManager.getSecretKey(app2)); secretManager.unregisterApp(app1); assertNull(secretManager.getSecretKey(app1)); assertEquals(pw2update, secretManager.getSecretKey(app2)); secretManager.unregisterApp(app2); assertNull(secretManager.getSecretKey(app2)); assertNull(secretManager.getSecretKey(app1)); } }
@Test public void testMultipleRegisters() { ShuffleSecretManager secretManager = new ShuffleSecretManager(); secretManager.registerApp(app1, pw1); assertEquals(pw1, secretManager.getSecretKey(app1)); secretManager.registerApp(app2, ByteBuffer.wrap(pw2.getBytes())); assertEquals(pw2, secretManager.getSecretKey(app2)); // now update the password for the apps and make sure it takes affect secretManager.registerApp(app1, pw1update); assertEquals(pw1update, secretManager.getSecretKey(app1)); secretManager.registerApp(app2, ByteBuffer.wrap(pw2update.getBytes())); assertEquals(pw2update, secretManager.getSecretKey(app2)); secretManager.unregisterApp(app1); assertNull(secretManager.getSecretKey(app1)); assertEquals(pw2update, secretManager.getSecretKey(app2)); secretManager.unregisterApp(app2); assertNull(secretManager.getSecretKey(app2)); assertNull(secretManager.getSecretKey(app1)); } }
@Test public void testMultipleRegisters() { ShuffleSecretManager secretManager = new ShuffleSecretManager(); secretManager.registerApp(app1, pw1); assertEquals(pw1, secretManager.getSecretKey(app1)); secretManager.registerApp(app2, ByteBuffer.wrap(pw2.getBytes())); assertEquals(pw2, secretManager.getSecretKey(app2)); // now update the password for the apps and make sure it takes affect secretManager.registerApp(app1, pw1update); assertEquals(pw1update, secretManager.getSecretKey(app1)); secretManager.registerApp(app2, ByteBuffer.wrap(pw2update.getBytes())); assertEquals(pw2update, secretManager.getSecretKey(app2)); secretManager.unregisterApp(app1); assertNull(secretManager.getSecretKey(app1)); assertEquals(pw2update, secretManager.getSecretKey(app2)); secretManager.unregisterApp(app2); assertNull(secretManager.getSecretKey(app2)); assertNull(secretManager.getSecretKey(app1)); } }