/** * @deprecated Use {@link #getProp(String, String)} */ @Deprecated @Override protected String getProperty(String key, String def) { return getProp(key, def); }
/** * @deprecated Use {@link #getProp(String)} */ @Deprecated @Override protected String getProperty(String key) { return getProp(key); }
@VisibleForTesting GoogleAnalyticsUnsampledExtractor(WorkUnitState state, Extractor<S, D> actualExtractor, Analytics gaService) throws IOException { this.wuState = state; this.googleAnalyticsFormatter = DateTimeFormat.forPattern(DATE_FORMAT) .withZone(DateTimeZone.forID(state.getProp(SOURCE_TIMEZONE, DEFAULT_SOURCE_TIMEZONE))); this.watermarkFormatter = DateTimeFormat.forPattern(WATERMARK_INPUTFORMAT) .withZone(DateTimeZone.forID(state.getProp(SOURCE_TIMEZONE, DEFAULT_SOURCE_TIMEZONE))); this.actualExtractor = actualExtractor; this.gaService = gaService; this.nextWatermark = -1; }
/** * Get the actual high {@link Watermark} as a {@link JsonElement}. * * @return a {@link JsonElement} representing the actual high {@link Watermark}, * or {@code null} if the actual high {@link Watermark} is not set. */ public JsonElement getActualHighWatermark() { if (!contains(ConfigurationKeys.WORK_UNIT_STATE_ACTUAL_HIGH_WATER_MARK_KEY)) { return null; } return JSON_PARSER.parse(getProp(ConfigurationKeys.WORK_UNIT_STATE_ACTUAL_HIGH_WATER_MARK_KEY)); }
public TokenBasedFileInputStreamExtractor(WorkUnitState workUnitState, FileBasedHelper fsHelper) { super(workUnitState, fsHelper); Preconditions.checkArgument(this.fileDownloader instanceof TokenizedFileDownloader); this.token = workUnitState.getProp(TOKEN, TokenizedFileDownloader.DEFAULT_TOKEN); this.charSet = workUnitState.getProp(CHARSET, ConfigurationKeys.DEFAULT_CHARSET_ENCODING.name()); ((TokenizedFileDownloader) fileDownloader).setToken(token); ((TokenizedFileDownloader) fileDownloader).setCharset(charSet); } }
private List<GoogleWebmasterDataFetcher.Metric> getRequestedMetrics(WorkUnitState wuState) { List<GoogleWebmasterDataFetcher.Metric> metrics = new ArrayList<>(); String metricsString = wuState.getProp(GoogleWebMasterSource.KEY_REQUEST_METRICS); for (String metric : splitter.split(metricsString)) { metrics.add(GoogleWebmasterDataFetcher.Metric.valueOf(metric.toUpperCase())); } return metrics; } }
public ToAvroConverterBase<SI, JsonObject> init(WorkUnitState workUnit) { super.init(workUnit); this.ignoreFields = SPLITTER_ON_COMMA.splitToList(workUnit.getProp(ConfigurationKeys.CONVERTER_IGNORE_FIELDS, "")); return this; }
@Override public Converter<Class<String>, Class<String>, String, String> init(WorkUnitState workUnit) { this.pattern = Pattern.compile(Strings.nullToEmpty(workUnit.getProp( ForkOperatorUtils.getPropertyNameForBranch(workUnit, ConfigurationKeys.CONVERTER_STRING_FILTER_PATTERN)))); this.matcher = Optional.absent(); return this; }
public GoogleWebmasterExtractor(GoogleWebmasterClient gscClient, WorkUnitState wuState, long lowWatermark, long expectedHighWaterMark, Map<String, Integer> columnPositionMap, List<GoogleWebmasterFilter.Dimension> requestedDimensions, List<GoogleWebmasterDataFetcher.Metric> requestedMetrics, JsonArray schemaJson) throws IOException { this(wuState, lowWatermark, expectedHighWaterMark, columnPositionMap, requestedDimensions, requestedMetrics, schemaJson, createGoogleWebmasterDataFetchers(wuState.getProp(GoogleWebMasterSource.KEY_PROPERTY), gscClient, wuState)); }
public TaskState(WorkUnitState workUnitState) { // Since getWorkunit() returns an immutable WorkUnit object, // the WorkUnit object in this object is also immutable. super(workUnitState.getWorkunit(), workUnitState.getJobState(), workUnitState.getTaskBrokerNullable()); addAll(workUnitState); this.jobId = workUnitState.getProp(ConfigurationKeys.JOB_ID_KEY); this.taskId = workUnitState.getProp(ConfigurationKeys.TASK_ID_KEY); this.taskKey = workUnitState.getProp(ConfigurationKeys.TASK_KEY_KEY, "unknown_task_key"); this.taskAttemptId = Optional.fromNullable(workUnitState.getProp(ConfigurationKeys.TASK_ATTEMPT_ID_KEY)); this.setId(this.taskId); }
@Override public Converter<Class<String>, Class<String>, String, String> init(WorkUnitState workUnit) { String stringSplitterDelimiterKey = ForkOperatorUtils.getPropertyNameForBranch( workUnit, ConfigurationKeys.CONVERTER_STRING_SPLITTER_DELIMITER); Preconditions.checkArgument(workUnit.contains(stringSplitterDelimiterKey), "Cannot use " + this.getClass().getName() + " with out specifying " + ConfigurationKeys.CONVERTER_STRING_SPLITTER_DELIMITER); this.splitter = Splitter.on(workUnit.getProp(stringSplitterDelimiterKey)).omitEmptyStrings(); return this; }
@Override public String getSchema() throws IOException { // Source is responsible to set SOURCE_SCHEMA ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); IOUtils.copyBytes(fs.open( new Path(workUnitState.getProp(ConfigurationKeys.SOURCE_SCHEMA))), outputStream, 4096, false); String schema = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); workUnitState.setProp((ConfigurationKeys.CONVERTER_AVRO_SCHEMA_KEY), schema); return schema; }
@Override public Converter<Schema, Schema, GenericRecord, Object> init(WorkUnitState workUnit) { String fieldPathKey = ForkOperatorUtils.getPropertyNameForBranch(workUnit, ConfigurationKeys.CONVERTER_AVRO_EXTRACTOR_FIELD_PATH); Preconditions.checkArgument(workUnit.contains(fieldPathKey), "The converter " + this.getClass().getName() + " cannot be used without setting the property " + ConfigurationKeys.CONVERTER_AVRO_EXTRACTOR_FIELD_PATH); this.fieldLocation = workUnit.getProp(fieldPathKey); return this; }
/** * Get iterator from protocol specific api if is.specific.api.active is false * Get iterator from source specific api if is.specific.api.active is true * @return iterator */ private Iterator<D> getIterator() throws DataRecordException, IOException { if (Boolean.valueOf(this.workUnitState.getProp(ConfigurationKeys.SOURCE_QUERYBASED_IS_SPECIFIC_API_ACTIVE))) { return this.getRecordSetFromSourceApi(this.schema, this.entity, this.workUnit, this.predicateList); } return this.getRecordSet(this.schema, this.entity, this.workUnit, this.predicateList); }
/** * Get the current runtime state of the {@link WorkUnit}. * * @return {@link WorkingState} of the {@link WorkUnit} */ public WorkingState getWorkingState() { return WorkingState .valueOf(getProp(ConfigurationKeys.WORK_UNIT_WORKING_STATE_KEY, WorkingState.PENDING.toString())); }
@Override public Schema convertSchema(S schemaIn, WorkUnitState workUnit) throws SchemaConversionException { Preconditions.checkArgument(workUnit.contains(KafkaSource.TOPIC_NAME), "Must specify topic name."); String topic = workUnit.getProp(KafkaSource.TOPIC_NAME); try { return (Schema) this.schemaRegistry.getLatestSchema(topic); } catch (IOException | SchemaRegistryException e) { throw new SchemaConversionException(e); } }
@Override public Extractor<S, D> getExtractor(WorkUnitState state) throws IOException { Preconditions.checkArgument(state.contains(EXTRACTOR_TYPE), "Missing key " + EXTRACTOR_TYPE); try { ClassAliasResolver<KafkaExtractor> aliasResolver = new ClassAliasResolver<>(KafkaExtractor.class); Class<? extends KafkaExtractor> klazz = aliasResolver.resolveClass(state.getProp(EXTRACTOR_TYPE)); return GobblinConstructorUtils.invokeLongestConstructor(klazz, state); } catch (ReflectiveOperationException e) { throw new RuntimeException(e); } } }
public TestAvroExtractor(WorkUnitState workUnitState) throws IOException { this.state = workUnitState; this.recordList =getRecordFromFile(workUnitState.getProp(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL)); this.recordIterator = this.recordList.iterator(); }
@Override public Extractor<String, String> getExtractor(WorkUnitState workUnitState) { Extractor<String, String> extractor = super.getExtractor(workUnitState); if (workUnitState.getProp(ConfigurationKeys.DATASET_URN_KEY).endsWith("0")) { return new FaultyExtractor(workUnitState); } return extractor; } }
@Test public void passesRecordThrough() throws IOException { byte[] record = new byte[]{'a', 'b', 'c', 'd'}; dummyWriter.setExpectedRecord(record); writer.write(record); writer.commit(); Assert.assertEquals(dummyWriter.recordsWritten(), 1); Assert.assertNull(state.getProp(ConfigurationKeys.WRITER_METADATA_KEY)); }