@Override public Converter<String, JsonArray, String[], JsonObject> init(WorkUnitState workUnit) { super.init(workUnit); customOrder = workUnit.getPropAsList(CUSTOM_ORDERING, ""); if (!customOrder.isEmpty()) { LOG.info("Will use custom order to generate JSON from CSV: " + customOrder); } return this; }
public HiveTask(TaskContext taskContext) { super(taskContext); this.taskContext = taskContext; this.workUnitState = taskContext.getTaskState(); this.workUnit = new HiveWorkUnit(this.workUnitState.getWorkunit()); this.eventSubmitter = new EventSubmitter.Builder(this.metricContext, "gobblin.HiveTask") .build(); this.hiveExecutionQueries = Lists.newArrayList(); this.publishEntity = new QueryBasedHivePublishEntity(); try { this.hiveJdbcConnector = HiveJdbcConnector.newConnectorWithProps(this.workUnitState.getProperties()); } catch (SQLException se) { throw new RuntimeException("Error in creating JDBC Connector", se); } this.addFiles = this.workUnitState.getPropAsList(ADD_FILES, ""); this.addJars = this.workUnitState.getPropAsList(ADD_JARS, ""); this.setupQueries = Splitter.on(";").trimResults().omitEmptyStrings().splitToList(this.workUnitState.getProp(SETUP_QUERIES, "")); }
@Override public Converter<String, JsonArray, String, JsonObject> init(WorkUnitState workUnit) { super.init(workUnit); String pattern = workUnit.getProp(GROK_PATTERN); String patternsFile = workUnit.getProp(BASE_PATTERNS_FILE); this.nullStringRegexes = DatasetFilterUtils.getPatternsFromStrings(workUnit.getPropAsList(NULLSTRING_REGEXES, "")); InputStreamReader grokPatterns; try { if (patternsFile == null) { grokPatterns = new InputStreamReader(getClass().getResourceAsStream("/grok/grok-base-patterns"), "UTF8"); } else { grokPatterns = new InputStreamReader(new FileInputStream(patternsFile), "UTF8"); } grok = new Grok(); grok.addPatternFromReader(grokPatterns); grok.compile(pattern); } catch (GrokException | FileNotFoundException | UnsupportedEncodingException e) { throw new RuntimeException("Error initializing GROK: " + e); } return this; }
this.workUnit = workUnitState.getWorkunit(); this.filesToPull = Lists.newArrayList(workUnitState.getPropAsList(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL, "")); this.statusCount = this.workUnit.getPropAsInt(ConfigurationKeys.FILEBASED_REPORT_STATUS_ON_COUNT, ConfigurationKeys.DEFAULT_FILEBASED_REPORT_STATUS_ON_COUNT);
Assert.assertEquals(state.getPropAsList("list", "item1,item2").get(0), "item1"); Assert.assertEquals(state.getPropAsList("list", "item1,item2").get(1), "item2"); Assert.assertEquals(state.getPropAsLong("long", Long.MAX_VALUE), Long.MAX_VALUE); Assert.assertEquals(state.getPropAsInt("int", Integer.MAX_VALUE), Integer.MAX_VALUE); Assert.assertEquals(state.getPropAsList("list").get(0), "item1"); Assert.assertEquals(state.getPropAsList("list").get(1), "item2"); Assert.assertEquals(state.getPropAsLong("long"), Long.MAX_VALUE); Assert.assertEquals(state.getPropAsInt("int"), Integer.MAX_VALUE); Assert.assertNotEquals(state.getPropAsList("list", "item1,item2").get(0), "item1"); Assert.assertNotEquals(state.getPropAsList("list", "item1,item2").get(1), "item2"); Assert.assertNotEquals(state.getPropAsLong("long", Long.MAX_VALUE), Long.MAX_VALUE); Assert.assertNotEquals(state.getPropAsInt("int", Integer.MAX_VALUE), Integer.MAX_VALUE); Assert.assertNotEquals(state.getPropAsList("list").get(0), "item1"); Assert.assertNotEquals(state.getPropAsList("list").get(1), "item2"); Assert.assertNotEquals(state.getPropAsLong("long"), Long.MAX_VALUE); Assert.assertNotEquals(state.getPropAsInt("int"), Integer.MAX_VALUE); Assert.assertEquals(state.getPropAsList("list").get(0), "item3"); Assert.assertEquals(state.getPropAsList("list").get(1), "item4"); Assert.assertEquals(state.getPropAsLong("long"), Long.MIN_VALUE); Assert.assertEquals(state.getPropAsInt("int"), Integer.MIN_VALUE); Assert.assertEquals(state.getPropAsList("list").get(0), "item3"); Assert.assertEquals(state.getPropAsList("list").get(1), "item4"); Assert.assertEquals(state.getPropAsLong("long"), Long.MIN_VALUE); Assert.assertEquals(state.getPropAsInt("int"), Integer.MIN_VALUE);
.withZone(DateTimeZone.forID(wuState.getProp(SOURCE_TIMEZONE, DEFAULT_SOURCE_TIMEZONE))); Credential credential = new GoogleCommon.CredentialBuilder(wuState.getProp(SOURCE_CONN_PRIVATE_KEY), wuState.getPropAsList(API_SCOPES)) .fileSystemUri(wuState.getProp(PRIVATE_KEY_FILESYSTEM_URI)) .proxyUrl(wuState.getProp(SOURCE_CONN_USE_PROXY_URL))
@Override public Converter<String, JsonArray, String[], JsonObject> init(WorkUnitState workUnit) { super.init(workUnit); customOrder = workUnit.getPropAsList(CUSTOM_ORDERING, ""); if (!customOrder.isEmpty()) { LOG.info("Will use custom order to generate JSON from CSV: " + customOrder); } return this; }
public HiveTask(TaskContext taskContext) { super(taskContext); this.taskContext = taskContext; this.workUnitState = taskContext.getTaskState(); this.workUnit = new HiveWorkUnit(this.workUnitState.getWorkunit()); this.eventSubmitter = new EventSubmitter.Builder(this.metricContext, "gobblin.HiveTask") .build(); this.hiveExecutionQueries = Lists.newArrayList(); this.publishEntity = new QueryBasedHivePublishEntity(); try { this.hiveJdbcConnector = HiveJdbcConnector.newConnectorWithProps(this.workUnitState.getProperties()); } catch (SQLException se) { throw new RuntimeException("Error in creating JDBC Connector", se); } this.addFiles = this.workUnitState.getPropAsList(ADD_FILES, ""); this.addJars = this.workUnitState.getPropAsList(ADD_JARS, ""); this.setupQueries = Splitter.on(";").trimResults().omitEmptyStrings().splitToList(this.workUnitState.getProp(SETUP_QUERIES, "")); }
this.workUnit = workUnitState.getWorkunit(); this.filesToPull = Lists.newArrayList(workUnitState.getPropAsList(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL, "")); this.statusCount = this.workUnit.getPropAsInt(ConfigurationKeys.FILEBASED_REPORT_STATUS_ON_COUNT, ConfigurationKeys.DEFAULT_FILEBASED_REPORT_STATUS_ON_COUNT);
.withZone(DateTimeZone.forID(wuState.getProp(SOURCE_TIMEZONE, DEFAULT_SOURCE_TIMEZONE))); Credential credential = new GoogleCommon.CredentialBuilder(wuState.getProp(SOURCE_CONN_PRIVATE_KEY), wuState.getPropAsList(API_SCOPES)) .fileSystemUri(wuState.getProp(PRIVATE_KEY_FILESYSTEM_URI)) .proxyUrl(wuState.getProp(SOURCE_CONN_USE_PROXY_URL))