RegexRule regexRule = new RegexRule();
{
regexRule.addRule("http://bbs.gter.net/thread.*");
}
public Links visitAndGetNextLinks(Page page) {
String url = page.getUrl();
if (Pattern.matches("http://bbs.gter.net/forum-1033-.*", url)) {
Links nextLinks=new Links();
nextLinks.addAllFromDocument(page.getDoc(),regexRule);
System.out.println(nextLinks+"\n");
return nextLinks;
}else{
byte[] content=page.getContent();
try {
FileUtils.writeFileWithParent("/Users/gary/Dropbox/hooom/program/html"+id.incrementAndGet()+".txt", content);
System.out.println("save page "+page.getUrl());
} catch (IOException ex) {
ex.printStackTrace();
}
}
return null;
}