output = itr.next(); } else { throw new DataRecordException("Failed to get data from salesforce; REST response has no output"); throw new DataRecordException("Failed to get records from salesforce; error - " + e.getMessage(), e);
@Override public Iterator<DataRecord> getRecordSetFromSourceApi(String schema, String entity, WorkUnit workUnit, List<Predicate> predicateList) throws IOException { try { return getRecordSet(schema, entity, workUnit, predicateList); } catch (DataRecordException e) { e.printStackTrace(); return null; } } }
/** * Get a {@link JsonArray} containing the query results */ private JsonArray getRecordsForQuery(SalesforceConnector connector, String query) { try { String soqlQuery = SalesforceExtractor.getSoqlUrl(query); List<Command> commands = RestApiConnector.constructGetCommand(connector.getFullUri(soqlQuery)); CommandOutput<?, ?> response = connector.getResponse(commands); String output; Iterator<String> itr = (Iterator<String>) response.getResults().values().iterator(); if (itr.hasNext()) { output = itr.next(); } else { throw new DataRecordException("Failed to get data from salesforce; REST response has no output"); } return GSON.fromJson(output, JsonObject.class).getAsJsonArray("records"); } catch (RestApiClientException | RestApiProcessingException | DataRecordException e) { throw new RuntimeException("Fail to get data from salesforce", e); } }
throw new DataRecordException("Failed to get records from salesforce; error - " + e.getMessage(), e);
@Override public D readRecord(@Deprecated D reuse) throws DataRecordException, IOException { if (!this.isPullRequired()) { log.info("No more records to read"); return null; } D nextElement = null; try { if (isInitialPull()) { log.info("Initial pull"); if (shouldRemoveDataPullUpperBounds()) { this.removeDataPullUpperBounds(); } this.iterator = this.getIterator(); } if (this.iterator.hasNext()) { nextElement = this.iterator.next(); if (!this.iterator.hasNext()) { log.debug("Getting next pull"); this.iterator = this.getIterator(); if (this.iterator == null) { this.setFetchStatus(false); } } } } catch (Exception e) { throw new DataRecordException("Failed to get records using rest api; error - " + e.getMessage(), e); } return nextElement; }
resultset = itr.next(); } else { throw new DataRecordException("Failed to get source record count from database - Resultset has no records"); return recordSet.iterator(); } catch (Exception e) { throw new DataRecordException("Failed to get records from database; error - " + e.getMessage(), e);
@Override public Iterator<JsonElement> getRecordSet(String schema, String entity, WorkUnit workUnit, List<Predicate> predicateList) throws DataRecordException, IOException { Iterator<JsonElement> rs = null; List<Command> cmds; try { if (isFirstPull()) { this.log.info("Get data recordset using JDBC"); cmds = this.getDataMetadata(schema, entity, workUnit, predicateList); this.dataResponse = this.executePreparedSql(cmds); this.setFirstPull(false); } rs = this.getData(this.dataResponse); return rs; } catch (Exception e) { throw new DataRecordException("Failed to get record set using JDBC; error - " + e.getMessage(), e); } }
@Override public Iterator<JsonElement> getRecordSet(String schema, String entity, WorkUnit workUnit, List<Predicate> predicateList) throws DataRecordException { log.debug("Get data records using Rest Api"); Iterator<JsonElement> rs = null; List<Command> cmds; try { boolean success = true; if (this.connector.isConnectionClosed()) { success = this.connector.connect(); } if (!success) { throw new DataRecordException("Failed to connect."); } log.debug("Connected successfully."); if (this.getPullStatus() == false) { return null; } if (this.getNextUrl() == null) { cmds = this.getDataMetadata(schema, entity, workUnit, predicateList); } else { cmds = RestApiConnector.constructGetCommand(this.getNextUrl()); } CommandOutput<?, ?> response = this.connector.getResponse(cmds); rs = this.getData(response); return rs; } catch (Exception e) { throw new DataRecordException("Failed to get records using rest api; error - " + e.getMessage(), e); } }
throw new DataRecordException("Failed to get salesforce url for data records; error - " + e.getMessage(), e);
throw new DataRecordException("Could not decode Kafka record");
@Override public D readRecord(@Deprecated D reuse) throws DataRecordException, IOException { if (!this.isPullRequired()) { log.info("No more records to read"); return null; } D nextElement = null; try { if (isInitialPull()) { log.info("Initial pull"); if (shouldRemoveDataPullUpperBounds()) { this.removeDataPullUpperBounds(); } this.iterator = this.getIterator(); } if (this.iterator.hasNext()) { nextElement = this.iterator.next(); if (!this.iterator.hasNext()) { log.debug("Getting next pull"); this.iterator = this.getIterator(); if (this.iterator == null) { this.setFetchStatus(false); } } } } catch (Exception e) { throw new DataRecordException("Failed to get records using rest api; error - " + e.getMessage(), e); } return nextElement; }
@Override public Iterator<JsonElement> getRecordSet(String schema, String entity, WorkUnit workUnit, List<Predicate> predicateList) throws DataRecordException, IOException { Iterator<JsonElement> rs = null; List<Command> cmds; try { if (isFirstPull()) { this.log.info("Get data recordset using JDBC"); cmds = this.getDataMetadata(schema, entity, workUnit, predicateList); this.dataResponse = this.executePreparedSql(cmds); this.setFirstPull(false); } rs = this.getData(this.dataResponse); return rs; } catch (Exception e) { throw new DataRecordException("Failed to get record set using JDBC; error - " + e.getMessage(), e); } }
resultset = itr.next(); } else { throw new DataRecordException("Failed to get source record count from database - Resultset has no records"); return recordSet.iterator(); } catch (Exception e) { throw new DataRecordException("Failed to get records from database; error - " + e.getMessage(), e);
@Override public Iterator<JsonElement> getRecordSet(String schema, String entity, WorkUnit workUnit, List<Predicate> predicateList) throws DataRecordException { log.debug("Get data records using Rest Api"); Iterator<JsonElement> rs = null; List<Command> cmds; try { boolean success = true; if (this.connector.isConnectionClosed()) { success = this.connector.connect(); } if (!success) { throw new DataRecordException("Failed to connect."); } log.debug("Connected successfully."); if (this.getPullStatus() == false) { return null; } if (this.getNextUrl() == null) { cmds = this.getDataMetadata(schema, entity, workUnit, predicateList); } else { cmds = RestApiConnector.constructGetCommand(this.getNextUrl()); } CommandOutput<?, ?> response = this.connector.getResponse(cmds); rs = this.getData(response); return rs; } catch (Exception e) { throw new DataRecordException("Failed to get records using rest api; error - " + e.getMessage(), e); } }
throw new DataRecordException("Could not decode Kafka record");