@Signature public static Connection connect(String url) { return Jsoup.connect(url); }
private String getOpenId(String accessToken) throws IOException{ String url = openIdUri + accessToken; Document document = Jsoup.connect(url).get(); String resultText = document.text(); Matcher matcher = Pattern.compile("\"openid\":\"(.*?)\"").matcher(resultText); if (matcher.find()){ return matcher.group(1); } return null; }
private String vscoImageToURL(String url) throws IOException{ Document page = Jsoup.connect(url).userAgent(USER_AGENT) .get(); //create Elements filled only with Elements with the "meta" tag. Elements metaTags = page.getElementsByTag("meta"); String result = ""; for(Element metaTag : metaTags){ //find URL inside meta-tag with property of "og:image" if (metaTag.attr("property").equals("og:image")){ String givenURL = metaTag.attr("content"); givenURL = givenURL.replaceAll("\\?h=[0-9]+", "");//replace the "?h=xxx" tag at the end of the URL (where each x is a number) result = givenURL; LOGGER.debug("Found image URL: " + givenURL); break;//immediately stop after getting URL (there should only be 1 image to be downloaded) } } //Means website changed, things need to be fixed. if (result.isEmpty()){ LOGGER.error("Could not find image URL at: " + url); } return result; }
public static VideoBean getVideoFromWeipai(VideoBean video) throws Exception { Document dom = Jsoup.connect(video.getLongUrl()).get(); video.setIdStr(KeyGenerator.generateMD5(video.getShortUrl())); Elements divs = dom.select("div[class=video_img WscaleH]"); if (divs != null && divs.size() > 0) { video.setImage(divs.get(0).attr("data-url")); } divs = dom.select("video#video"); if (divs != null && divs.size() > 0) { video.setVideoUrl(divs.get(0).attr("src")); } return video; }
public static VideoBean getVideoFromSinaVideo(VideoBean video) throws Exception { Document dom = Jsoup.connect(video.getLongUrl()).get(); video.setIdStr(KeyGenerator.generateMD5(video.getShortUrl())); Elements divs = dom.select("video.video"); if (divs != null && divs.size() > 0) { String src = divs.get(0).attr("src"); src = src.replace("amp;", ""); video.setVideoUrl(src); } divs = dom.select("img.poster"); if (divs != null && divs.size() > 0) { video.setImage(divs.get(0).attr("src")); } return video; }
public static VideoBean getVideoFromMeipai(VideoBean video) throws Exception { Document dom = Jsoup.connect(video.getLongUrl()).get(); Elements divs = dom.select("div#mediaPlayer"); if (divs != null && divs.size() > 0) { Element div = divs.get(0); video.setVideoUrl(div.attr("data-video")); video.setImage(div.attr("data-poster")); } video.setIdStr(KeyGenerator.generateMD5(video.getShortUrl())); return video; }
private QQToken getToken(String tokenAccessApi) throws IOException{ Document document = Jsoup.connect(tokenAccessApi).get(); String tokenResult = document.text(); String[] results = tokenResult.split("&"); if (results.length == 3){ QQToken qqToken = new QQToken(); String accessToken = results[0].replace("access_token=", ""); int expiresIn = Integer.valueOf(results[1].replace("expires_in=", "")); String refreshToken = results[2].replace("refresh_token=", ""); qqToken.setAccessToken(accessToken); qqToken.setExpiresIn(expiresIn); qqToken.setRefresh_token(refreshToken); return qqToken; } return null; }
private JSONArray getPageUrls() { String postURL = "http://www.tsumino.com/Read/Load"; try { // This sessionId will expire and need to be replaced cookies.put("ASP.NET_SessionId","c4rbzccf0dvy3e0cloolmlkq"); Document doc = Jsoup.connect(postURL).data("q", getAlbumID()).userAgent(USER_AGENT).cookies(cookies).referrer("http://www.tsumino.com/Read/View/" + getAlbumID()).get(); String jsonInfo = doc.html().replaceAll("<html>","").replaceAll("<head></head>", "").replaceAll("<body>", "").replaceAll("</body>", "") .replaceAll("</html>", "").replaceAll("\n", ""); JSONObject json = new JSONObject(jsonInfo); return json.getJSONArray("reader_page_urls"); } catch (IOException e) { LOGGER.info(e); sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_ERRORED, "Unable to download album, please compete the captcha at http://www.tsumino.com/Read/Auth/" + getAlbumID() + " and try again"); return null; } }
public static Document getDocWithPC(String url) { try { return Jsoup.connect(url).userAgent(UA_PC).timeout(TIME_OUT).ignoreContentType(true).get(); } catch (IOException e) { log.error(ERROR_DESC + url); throw new AnyException(ERROR_DESC + url); } }
private String getImageLinkFromDLLink(String url) { try { Connection.Response response = Jsoup.connect(url) .userAgent(USER_AGENT) .timeout(10000) .cookies(cookies) .followRedirects(false) .execute(); String imageURL = response.header("Location"); LOGGER.info(imageURL); return imageURL; } catch (IOException e) { LOGGER.info("Got error message " + e.getMessage() + " trying to download " + url); return null; } }
public static Observable<BaseDanmakuParser> downloadXML(final String uri) { return Observable.create((Observable.OnSubscribe<BaseDanmakuParser>) subscriber -> { if (TextUtils.isEmpty(uri)) { subscriber.onNext(new BaseDanmakuParser() { @Override protected IDanmakus parse() { return new Danmakus(); } }); } ILoader loader = null; try { HttpConnection.Response rsp = (HttpConnection.Response) Jsoup.connect(uri).timeout(20000).execute(); InputStream stream = new ByteArrayInputStream(BiliDanmukuCompressionTools. decompressXML(rsp.bodyAsBytes())); loader = DanmakuLoaderFactory. create(DanmakuLoaderFactory.TAG_BILI); loader.load(stream); } catch (IOException | DataFormatException | IllegalDataException e) { e.printStackTrace(); } BaseDanmakuParser parser = new BiliDanmukuParser(); assert loader != null; IDataSource<?> dataSource = loader.getDataSource(); parser.load(dataSource); subscriber.onNext(parser); }).subscribeOn(Schedulers.io()); } }
private static Document getDocument(String strUrl) throws IOException { return Jsoup.connect(strUrl) .userAgent(USER_AGENT) .timeout(10 * 1000) .maxBodySize(0) .get(); }
public static Document getDocWithPhone(String url) { try { return Jsoup.connect(url).userAgent(UA_PHONE).timeout(TIME_OUT).ignoreContentType(true).validateTLSCertificates(false).get(); } catch (IOException e) { log.error(ERROR_DESC + url); throw new AnyException(ERROR_DESC + url); } }
public static Document getDocWithPhone(String url, String cookie) { try { return Jsoup.connect(url).userAgent(UA_PHONE).timeout(TIME_OUT).header("Cookie", cookie).ignoreContentType(true).get(); } catch (IOException e) { log.error(ERROR_DESC + url); throw new AnyException(ERROR_DESC + url); } }
/** * 获取片段播放的 key */ private String videoKey(String vid, String filename, String format) { try { Document document = Jsoup.connect(KEY_API).header("Cookie", COOKIE) .data("vid", vid).data("platform", PLATFORM) .data("otype", "json") .data("filename", filename).data("sdtfrom", SDTFROM) .data("format", format).data("guid", GUID).ignoreContentType(true).get(); String result = document.text().replace("QZOutputJson=", ""); System.out.println(result); result = result.substring(0, result.length() - 1); return JSONObject.parseObject(result).getString("key"); } catch (IOException e) { log.info("request tencent video part api error, vid : " + vid); throw new AnyException("request tencent api error, vid : " + vid); } } }
private Document requestAPI(String keyword) { try { return Jsoup.connect(api).userAgent(ua).ignoreContentType(true).data("wd", keyword).get(); } catch (IOException e) { throw new AnyException(ExceptionEnum.VIDEO_SEARCH_ERROR); } }
private void defaultSettings() { this.retries = Utils.getConfigInteger("download.retries", 1); connection = Jsoup.connect(this.url); connection.userAgent(AbstractRipper.USER_AGENT); connection.method(Method.GET); connection.timeout(TIMEOUT); connection.maxBodySize(0); }
/** * Prepare to submit this form. A Connection object is created with the request set up from the form values. You * can then set up other options (like user-agent, timeout, cookies), then execute it. * @return a connection prepared from the values of this form. * @throws IllegalArgumentException if the form's absolute action URL cannot be determined. Make sure you pass the * document's base URI when parsing. */ public Connection submit() { String action = hasAttr("action") ? absUrl("action") : baseUri(); Validate.notEmpty(action, "Could not determine a form action URL for submit. Ensure you set a base URI when parsing."); Connection.Method method = attr("method").toUpperCase().equals("POST") ? Connection.Method.POST : Connection.Method.GET; return Jsoup.connect(action) .data(formData()) .method(method); }
@Override public String workInBackground(Void... p) throws TaskException { try { AccountBean accountBean = AccountUtils.getLogedinAccount(); if (TextUtils.isEmpty(accountBean.getAccount()) || TextUtils.isEmpty(accountBean.getPassword())) throw new TaskException("", getString(R.string.account_fillaccount_faild)); String js = FileUtils.readAssetsFile("mobile.js", GlobalContext.getInstance()); js = js.replace("%username%", accountBean.getAccount()); js = js.replace("%password%", accountBean.getPassword()); Document dom = Jsoup.connect(url).get(); String html = dom.toString(); html = html.replace("</head>", js + "</head>"); return html; } catch (Exception e) { e.printStackTrace(); } throw new TaskException("", getString(R.string.account_fillaccount_faild)); }
/** * 调用腾讯接口,获取视频信息 */ private String videoInfo(String vid) { try { Document document = Jsoup.connect(VIDEO_API).header("Cookie", COOKIE) .data("vids", vid).data("platform", PLATFORM) .data("sdtfrom", SDTFROM) .data("format", "10209") .data("otype", "json").data("defn", "fhd") .data("defaultfmt", "fhd").data("guid", GUID).ignoreContentType(true).get(); String result = document.text().replace("QZOutputJson=", ""); return result.substring(0, result.length() - 1); } catch (IOException e) { log.info("request tencent api error, vid : " + vid); throw new AnyException("request tencent api error, vid : " + vid); } }