/** * Set startUrls of Spider.<br> * Prior to startUrls of Site. * * @param startRequests startRequests * @return this */ public Spider startRequest(List<Request> startRequests) { checkIfRunning(); this.startRequests = startRequests; return this; }
/** * set the downloader of spider * * @param downloader downloader * @return this * @see Downloader */ public Spider setDownloader(Downloader downloader) { checkIfRunning(); this.downloader = downloader; return this; }
public Spider setExecutorService(ExecutorService executorService) { checkIfRunning(); this.executorService = executorService; return this; }
/** * set pipelines for Spider * * @param pipelines pipelines * @return this * @see Pipeline * @since 0.4.1 */ public Spider setPipelines(List<Pipeline> pipelines) { checkIfRunning(); this.pipelines = pipelines; return this; }
/** * start with more than one threads * * @param threadNum threadNum * @return this */ public Spider thread(int threadNum) { checkIfRunning(); this.threadNum = threadNum; if (threadNum <= 0) { throw new IllegalArgumentException("threadNum should be more than one!"); } return this; }
/** * add a pipeline for Spider * * @param pipeline pipeline * @return this * @see Pipeline * @since 0.2.1 */ public Spider addPipeline(Pipeline pipeline) { checkIfRunning(); this.pipelines.add(pipeline); return this; }
/** * start with more than one threads * * @param executorService executorService to run the spider * @param threadNum threadNum * @return this */ public Spider thread(ExecutorService executorService, int threadNum) { checkIfRunning(); this.threadNum = threadNum; if (threadNum <= 0) { throw new IllegalArgumentException("threadNum should be more than one!"); } this.executorService = executorService; return this; }
/** * Set startUrls of Spider.<br> * Prior to startUrls of Site. * * @param startUrls startUrls * @return this */ public Spider startUrls(List<String> startUrls) { checkIfRunning(); this.startRequests = UrlUtils.convertToRequests(startUrls); return this; }
/** * set scheduler for Spider * * @param scheduler scheduler * @return this * @see Scheduler * @since 0.2.1 */ public Spider setScheduler(Scheduler scheduler) { checkIfRunning(); Scheduler oldScheduler = this.scheduler; this.scheduler = scheduler; if (oldScheduler != null) { Request request; while ((request = oldScheduler.poll(this)) != null) { this.scheduler.push(request, this); } } return this; }
/** * set the downloader of spider * * @param downloader downloader * @return this * @see Downloader */ public Spider setDownloader(Downloader downloader) { checkIfRunning(); this.downloader = downloader; return this; }
/** * Set startUrls of Spider.<br> * Prior to startUrls of Site. * * @param startRequests startRequests * @return this */ public Spider startRequest(List<Request> startRequests) { checkIfRunning(); this.startRequests = startRequests; return this; }
public Spider setExecutorService(ExecutorService executorService) { checkIfRunning(); this.executorService = executorService; return this; }
/** * set pipelines for Spider * * @param pipelines pipelines * @return this * @see Pipeline * @since 0.4.1 */ public Spider setPipelines(List<Pipeline> pipelines) { checkIfRunning(); this.pipelines = pipelines; return this; }
/** * add a pipeline for Spider * * @param pipeline pipeline * @return this * @see Pipeline * @since 0.2.1 */ public Spider addPipeline(Pipeline pipeline) { checkIfRunning(); this.pipelines.add(pipeline); return this; }
/** * start with more than one threads * * @param threadNum threadNum * @return this */ public Spider thread(int threadNum) { checkIfRunning(); this.threadNum = threadNum; if (threadNum <= 0) { throw new IllegalArgumentException("threadNum should be more than one!"); } return this; }
/** * start with more than one threads * * @param executorService executorService to run the spider * @param threadNum threadNum * @return this */ public Spider thread(ExecutorService executorService, int threadNum) { checkIfRunning(); this.threadNum = threadNum; if (threadNum <= 0) { throw new IllegalArgumentException("threadNum should be more than one!"); } this.executorService = executorService; return this; }
/** * Set startUrls of Spider.<br> * Prior to startUrls of Site. * * @param startUrls startUrls * @return this */ public Spider startUrls(List<String> startUrls) { checkIfRunning(); this.startRequests = UrlUtils.convertToRequests(startUrls); return this; }
/** * set scheduler for Spider * * @param scheduler scheduler * @return this * @see Scheduler * @since 0.2.1 */ public Spider setScheduler(Scheduler scheduler) { checkIfRunning(); Scheduler oldScheduler = this.scheduler; this.scheduler = scheduler; if (oldScheduler != null) { Request request; while ((request = oldScheduler.poll(this)) != null) { this.scheduler.push(request, this); } } return this; }