congrats Icon
New! Announcing Tabnine Chat Beta
Learn More
Tabnine Logo
org.apache.gobblin.fork
Code IndexAdd Tabnine to your IDE (free)

How to use org.apache.gobblin.fork

Best Java code snippets using org.apache.gobblin.fork (Showing top 20 results out of 315)

origin: apache/incubator-gobblin

/**
 * Returns a {@link org.apache.gobblin.fork.CopyableSchema} wrapper around the given {@link Schema}.
 * {@inheritDoc}
 * @see org.apache.gobblin.converter.Converter#convertSchema(java.lang.Object, org.apache.gobblin.configuration.WorkUnitState)
 */
@Override
public CopyableSchema convertSchema(Schema inputSchema, WorkUnitState workUnit) throws SchemaConversionException {
 return new CopyableSchema(inputSchema);
}
origin: apache/incubator-gobblin

public RecordWithForkMap(ControlMessage<D> record, int activeBranchesForRecord) {
 this.record = record;
 this.forkMap = null;
 this.copiesLeft = activeBranchesForRecord;
 this.mustCopy = this.copiesLeft > 1;
 this.cloner = buildForkCloner();
}
origin: apache/incubator-gobblin

 @Override
 public boolean test(RecordWithForkMap<D> dRecordWithForkMap) {
  return dRecordWithForkMap.sendToBranch(this.forkIdx);
 }
}
origin: apache/incubator-gobblin

@Override
public GlobalMetadata<S> copy() throws CopyNotSupportedException {
 if (CopyHelper.isCopyable(schema)) {
  return new GlobalMetadata((S)CopyHelper.copy(schema));
 }
 throw new CopyNotSupportedException("Type is not copyable: " + schema.getClass().getName());
}
origin: apache/incubator-gobblin

/**
 * Copy this object if needed.
 * @param thing : this object that needs to be copied
 * @return: a possibly copied instance
 * @throws CopyNotSupportedException if thing needs to be copied but cannot be
 */
public static Object copy(Object thing) throws CopyNotSupportedException {
 if (!isCopyable(thing)) {
  throw new CopyNotSupportedException(thing.getClass().getName() + " cannot be copied. See Copyable");
 }
 if (thing instanceof Copyable) {
  return ((Copyable) thing).copy();
 }
 // Support for a few primitive types out of the box
 if (thing instanceof byte[]) {
  byte[] copy = new byte[((byte[]) thing).length];
  System.arraycopy(thing, 0, copy, 0, ((byte[]) thing).length);
  return copy;
 }
 // Assume that everything other type is immutable, not checking this again
 return thing;
}
origin: apache/incubator-gobblin

@Test
public void testCopyable()
  throws CopyNotSupportedException {
 Copyable c = mock(Copyable.class);
 Assert.assertTrue(CopyHelper.isCopyable(c));
 Object copy = new Object();
 when(c.copy()).thenReturn(copy);
 Assert.assertEquals(CopyHelper.copy(c), copy);
 Assert.assertEquals(CopyHelper.copy(c), copy);
}
origin: apache/incubator-gobblin

@Test
public void testByteArray()
  throws CopyNotSupportedException {
 int length = RANDOM.nextInt(200);
 byte[] bytes = new byte[length];
 RANDOM.nextBytes(bytes);
 Assert.assertTrue(CopyHelper.isCopyable(bytes));
 byte[] copiedBytes = (byte[]) CopyHelper.copy(bytes);
 Assert.assertTrue(copiedBytes != bytes, "Copied bytes reference should be different for every copy after that");
 Assert.assertEquals(copiedBytes, bytes, "Copied bytes value should be the same");
}
origin: apache/incubator-gobblin

@Override
public List<Boolean> forkSchema(WorkUnitState workUnitState, S input) {
 return this.embeddedForkOperator.forkSchema(workUnitState, input);
}
origin: apache/incubator-gobblin

@Override
public List<Boolean> forkDataRecordImpl(WorkUnitState workUnitState, D input) {
 return this.embeddedForkOperator.forkDataRecord(workUnitState, input);
}
origin: apache/incubator-gobblin

@Override
public int getBranches(WorkUnitState workUnitState) {
 return this.embeddedForkOperator.getBranches(workUnitState);
}
origin: apache/incubator-gobblin

 @Override
 public GenericRecord copy()
   throws CopyNotSupportedException {
  if (!(this.record instanceof GenericData.Record)) {
   throw new CopyNotSupportedException(
     "The record to make copy is not an instance of " + GenericData.Record.class.getName());
  }
  // Make a deep copy of the original record
  return new GenericData.Record((GenericData.Record) this.record, true);
 }
}
origin: apache/incubator-gobblin

@Override
public void init(WorkUnitState workUnitState) throws Exception {
 this.embeddedForkOperator.init(workUnitState);
 super.init(workUnitState, DecoratorUtils.resolveUnderlyingObject(this).getClass());
}
origin: apache/incubator-gobblin

@Override
protected StreamEntity<D> buildClone() {
 try {
  return new RecordEnvelope<>((D) CopyHelper.copy(_record), this, false);
 } catch (CopyNotSupportedException cnse) {
  throw new UnsupportedOperationException(cnse);
 }
}
origin: apache/incubator-gobblin

 /**
  * Returns a {@link org.apache.gobblin.fork.CopyableGenericRecord} wrapper around the given {@link GenericRecord}.
  * {@inheritDoc}
  * @see org.apache.gobblin.converter.Converter#convertRecord(java.lang.Object, java.lang.Object, org.apache.gobblin.configuration.WorkUnitState)
  */
 @Override
 public Iterable<CopyableGenericRecord> convertRecord(CopyableSchema outputSchema, GenericRecord inputRecord,
   WorkUnitState workUnit) throws DataConversionException {
  return new SingleRecordIterable<>(new CopyableGenericRecord(inputRecord));
 }
}
origin: apache/incubator-gobblin

@Override
public List<Boolean> forkDataRecord(WorkUnitState workUnitState, D input) {
 this.records.clear();
 for (int i = 0; i < getBranches(workUnitState); i++) {
  this.records.add(Boolean.TRUE);
 }
 return this.records;
}
origin: apache/incubator-gobblin

/**
 * Check if an object is copyable using the {@link #copy(Object)} method.
 * @param thing: the object that needs to be copied
 * @return: true if {@link CopyHelper} can copy this thing, false otherwise
 */
public static boolean isCopyable(Object thing) {
 if (
   (thing instanceof Copyable)
     || (thing instanceof byte[])
     || (isImmutableType(thing))
   ) {
  return true;
 }
 return false;
}
origin: apache/incubator-gobblin

@Override
public List<Boolean> forkSchema(WorkUnitState workUnitState, String input) {
 return Collections.nCopies(getBranches(workUnitState), true);
}
origin: apache/incubator-gobblin

@Override
public CopyableSchema convertSchema(String schema, WorkUnitState workUnit) {
 return new CopyableSchema(new Schema.Parser().parse(schema));
}
origin: apache/incubator-gobblin

public RecordWithForkMap(RecordEnvelope<D> record, List<Boolean> forkMap) {
 this.record = record;
 this.forkMap = Lists.newArrayList(forkMap);
 this.mustCopy = mustCopy(forkMap);
 this.copiesLeft = this.forkMap.stream().filter(x -> x).count();
 this.cloner = buildForkCloner();
}
origin: apache/incubator-gobblin

@Override
public List<Boolean> forkSchema(WorkUnitState workUnitState, S input) {
 this.schemas.clear();
 for (int i = 0; i < getBranches(workUnitState); i++) {
  this.schemas.add(Boolean.TRUE);
 }
 return this.schemas;
}
org.apache.gobblin.fork

Most used classes

  • ForkOperator
    An interface for fork operators that convert one input data record into multiple records. So essenti
  • CopyNotSupportedException
    A type of java.lang.Exceptions thrown when copying is not supported.
  • CopyHelper
    A helper class to copy things that may or may not be Copyable. Supports implementations for common p
  • Copyable
    An interface for classes that supports making copies of their instances.
  • Forker$ForkedStream
    An object containing the forked streams and a ConnectableFlowable used to connect the stream when al
  • CopyableGenericRecord,
  • CopyableSchema,
  • IdentityForkOperator,
  • Forker$ForkFilter,
  • Forker$RecordWithForkMap,
  • ForkerTest$MyFlowable,
  • ForkerTest$MyForkOperator
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now