/** * Obtain the tokens needed by the job and put them in the UGI * @param conf */ protected void initJobCredentialsAndUGI(Configuration conf) { try { this.currentUser = UserGroupInformation.getCurrentUser(); this.jobCredentials = ((JobConf)conf).getCredentials(); if (CryptoUtils.isEncryptedSpillEnabled(conf)) { int keyLen = conf.getInt( MRJobConfig.MR_ENCRYPTED_INTERMEDIATE_DATA_KEY_SIZE_BITS, MRJobConfig .DEFAULT_MR_ENCRYPTED_INTERMEDIATE_DATA_KEY_SIZE_BITS); KeyGenerator keyGen = KeyGenerator.getInstance(INTERMEDIATE_DATA_ENCRYPTION_ALGO); keyGen.init(keyLen); encryptedSpillKey = keyGen.generateKey().getEncoded(); } else { encryptedSpillKey = new byte[] {0}; } } catch (IOException e) { throw new YarnRuntimeException(e); } catch (NoSuchAlgorithmException e) { throw new YarnRuntimeException(e); } }
/** * Obtain the tokens needed by the job and put them in the UGI * @param conf */ protected void initJobCredentialsAndUGI(Configuration conf) { try { this.currentUser = UserGroupInformation.getCurrentUser(); this.jobCredentials = ((JobConf)conf).getCredentials(); if (CryptoUtils.isEncryptedSpillEnabled(conf)) { int keyLen = conf.getInt( MRJobConfig.MR_ENCRYPTED_INTERMEDIATE_DATA_KEY_SIZE_BITS, MRJobConfig .DEFAULT_MR_ENCRYPTED_INTERMEDIATE_DATA_KEY_SIZE_BITS); KeyGenerator keyGen = KeyGenerator.getInstance(INTERMEDIATE_DATA_ENCRYPTION_ALGO); keyGen.init(keyLen); encryptedSpillKey = keyGen.generateKey().getEncoded(); } else { encryptedSpillKey = new byte[] {0}; } } catch (IOException e) { throw new YarnRuntimeException(e); } catch (NoSuchAlgorithmException e) { throw new YarnRuntimeException(e); } }
/** * Obtain the tokens needed by the job and put them in the UGI * @param conf */ protected void initJobCredentialsAndUGI(Configuration conf) { try { this.currentUser = UserGroupInformation.getCurrentUser(); this.jobCredentials = ((JobConf)conf).getCredentials(); if (CryptoUtils.isEncryptedSpillEnabled(conf)) { int keyLen = conf.getInt( MRJobConfig.MR_ENCRYPTED_INTERMEDIATE_DATA_KEY_SIZE_BITS, MRJobConfig .DEFAULT_MR_ENCRYPTED_INTERMEDIATE_DATA_KEY_SIZE_BITS); KeyGenerator keyGen = KeyGenerator.getInstance(INTERMEDIATE_DATA_ENCRYPTION_ALGO); keyGen.init(keyLen); encryptedSpillKey = keyGen.generateKey().getEncoded(); } else { encryptedSpillKey = new byte[] {0}; } } catch (IOException e) { throw new YarnRuntimeException(e); } catch (NoSuchAlgorithmException e) { throw new YarnRuntimeException(e); } }
public static int cryptoPadding(Configuration conf) { // Sizeof(IV) + long(start-offset) return isEncryptedSpillEnabled(conf) ? CryptoCodec.getInstance(conf) .getCipherSuite().getAlgorithmBlockSize() + 8 : 0; }
public static int cryptoPadding(Configuration conf) { // Sizeof(IV) + long(start-offset) return isEncryptedSpillEnabled(conf) ? CryptoCodec.getInstance(conf) .getCipherSuite().getAlgorithmBlockSize() + 8 : 0; }
public static int cryptoPadding(Configuration conf) { // Sizeof(IV) + long(start-offset) return isEncryptedSpillEnabled(conf) ? CryptoCodec.getInstance(conf) .getCipherSuite().getAlgorithmBlockSize() + 8 : 0; }
public static int cryptoPadding(Configuration conf) { // Sizeof(IV) + long(start-offset) return isEncryptedSpillEnabled(conf) ? CryptoCodec.getInstance(conf) .getCipherSuite().getAlgorithmBlockSize() + 8 : 0; }
/** * Wraps a given FSDataOutputStream with a CryptoOutputStream. The size of the * data buffer required for the stream is specified by the * "mapreduce.job.encrypted-intermediate-data.buffer.kb" Job configuration * variable. * * @param conf * @param out * @return FSDataOutputStream * @throws IOException */ public static FSDataOutputStream wrapIfNecessary(Configuration conf, FSDataOutputStream out) throws IOException { if (isEncryptedSpillEnabled(conf)) { out.write(ByteBuffer.allocate(8).putLong(out.getPos()).array()); byte[] iv = createIV(conf); out.write(iv); if (LOG.isDebugEnabled()) { LOG.debug("IV written to Stream [" + Base64.encodeBase64URLSafeString(iv) + "]"); } return new CryptoFSDataOutputStream(out, CryptoCodec.getInstance(conf), getBufferSize(conf), getEncryptionKey(), iv); } else { return out; } }
/** * Wraps a given FSDataOutputStream with a CryptoOutputStream. The size of the * data buffer required for the stream is specified by the * "mapreduce.job.encrypted-intermediate-data.buffer.kb" Job configuration * variable. * * @param conf * @param out * @return FSDataOutputStream * @throws IOException */ public static FSDataOutputStream wrapIfNecessary(Configuration conf, FSDataOutputStream out) throws IOException { if (isEncryptedSpillEnabled(conf)) { out.write(ByteBuffer.allocate(8).putLong(out.getPos()).array()); byte[] iv = createIV(conf); out.write(iv); if (LOG.isDebugEnabled()) { LOG.debug("IV written to Stream [" + Base64.encodeBase64URLSafeString(iv) + "]"); } return new CryptoFSDataOutputStream(out, CryptoCodec.getInstance(conf), getBufferSize(conf), getEncryptionKey(), iv); } else { return out; } }
/** * Wraps a given FSDataOutputStream with a CryptoOutputStream. The size of the * data buffer required for the stream is specified by the * "mapreduce.job.encrypted-intermediate-data.buffer.kb" Job configuration * variable. * * @param conf * @param out * @return FSDataOutputStream * @throws IOException */ public static FSDataOutputStream wrapIfNecessary(Configuration conf, FSDataOutputStream out) throws IOException { if (isEncryptedSpillEnabled(conf)) { out.write(ByteBuffer.allocate(8).putLong(out.getPos()).array()); byte[] iv = createIV(conf); out.write(iv); if (LOG.isDebugEnabled()) { LOG.debug("IV written to Stream [" + Base64.encodeBase64URLSafeString(iv) + "]"); } return new CryptoFSDataOutputStream(out, CryptoCodec.getInstance(conf), getBufferSize(conf), getEncryptionKey(), iv); } else { return out; } }
/** * This method creates and initializes an IV (Initialization Vector) * * @param conf * @return byte[] * @throws IOException */ public static byte[] createIV(Configuration conf) throws IOException { CryptoCodec cryptoCodec = CryptoCodec.getInstance(conf); if (isEncryptedSpillEnabled(conf)) { byte[] iv = new byte[cryptoCodec.getCipherSuite().getAlgorithmBlockSize()]; cryptoCodec.generateSecureRandom(iv); return iv; } else { return null; } }
/** * This method creates and initializes an IV (Initialization Vector) * * @param conf * @return byte[] * @throws IOException */ public static byte[] createIV(Configuration conf) throws IOException { CryptoCodec cryptoCodec = CryptoCodec.getInstance(conf); if (isEncryptedSpillEnabled(conf)) { byte[] iv = new byte[cryptoCodec.getCipherSuite().getAlgorithmBlockSize()]; cryptoCodec.generateSecureRandom(iv); return iv; } else { return null; } }
/** * Wraps a given FSDataOutputStream with a CryptoOutputStream. The size of the * data buffer required for the stream is specified by the * "mapreduce.job.encrypted-intermediate-data.buffer.kb" Job configuration * variable. * * @param conf * @param out * @return FSDataOutputStream * @throws IOException */ public static FSDataOutputStream wrapIfNecessary(Configuration conf, FSDataOutputStream out) throws IOException { if (isEncryptedSpillEnabled(conf)) { out.write(ByteBuffer.allocate(8).putLong(out.getPos()).array()); byte[] iv = createIV(conf); out.write(iv); if (LOG.isDebugEnabled()) { LOG.debug("IV written to Stream [" + Base64.encodeBase64URLSafeString(iv) + "]"); } return new CryptoFSDataOutputStream(out, CryptoCodec.getInstance(conf), getBufferSize(conf), getEncryptionKey(), iv); } else { return out; } }
/** * This method creates and initializes an IV (Initialization Vector) * * @param conf * @return byte[] * @throws IOException */ public static byte[] createIV(Configuration conf) throws IOException { CryptoCodec cryptoCodec = CryptoCodec.getInstance(conf); if (isEncryptedSpillEnabled(conf)) { byte[] iv = new byte[cryptoCodec.getCipherSuite().getAlgorithmBlockSize()]; cryptoCodec.generateSecureRandom(iv); return iv; } else { return null; } }
/** * This method creates and initializes an IV (Initialization Vector) * * @param conf * @return byte[] * @throws IOException */ public static byte[] createIV(Configuration conf) throws IOException { CryptoCodec cryptoCodec = CryptoCodec.getInstance(conf); if (isEncryptedSpillEnabled(conf)) { byte[] iv = new byte[cryptoCodec.getCipherSuite().getAlgorithmBlockSize()]; cryptoCodec.generateSecureRandom(iv); return iv; } else { return null; } }
if (isEncryptedSpillEnabled(conf)) { int bufferSize = getBufferSize(conf); if (length > -1) {
if (isEncryptedSpillEnabled(conf)) { CryptoCodec cryptoCodec = CryptoCodec.getInstance(conf); int bufferSize = getBufferSize(conf);
if (isEncryptedSpillEnabled(conf)) { CryptoCodec cryptoCodec = CryptoCodec.getInstance(conf); int bufferSize = getBufferSize(conf);
if (isEncryptedSpillEnabled(conf)) { CryptoCodec cryptoCodec = CryptoCodec.getInstance(conf); int bufferSize = getBufferSize(conf);
if (isEncryptedSpillEnabled(conf)) { CryptoCodec cryptoCodec = CryptoCodec.getInstance(conf); int bufferSize = getBufferSize(conf);