TianyaTest.java

package com.virjar.dungproxy.client.samples.webmagic;

import com.virjar.dungproxy.client.ippool.IpPoolHolder;
import com.virjar.dungproxy.client.ippool.config.DungProxyContext;
import com.virjar.dungproxy.client.ippool.strategy.impl.JSONFileAvProxyDumper;
import com.virjar.dungproxy.client.ippool.strategy.impl.WhiteListProxyStrategy;
import com.virjar.dungproxy.client.webmagic.DungProxyDownloader;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * Created by virjar on 17/2/20.
 */
public class TianyaTest implements PageProcessor {
    private static Site site = Site.me()// .setHttpProxy(new HttpHost("127.0.0.1",8888))
            .setRetryTimes(3) // 就我的经验,这个重试一般用处不大,他是httpclient内部重试
            .setTimeOut(30000)// 在使用代理的情况下,这个需要设置,可以考虑调大线程数目
            .setSleepTime(0)// 使用代理了之后,代理会通过切换IP来防止反扒。同时,使用代理本身qps降低了,所以这个可以小一些
            .setCycleRetryTimes(3)// 这个重试会换IP重试,是setRetryTimes的上一层的重试,不要怕三次重试解决一切问题。。
            .setUseGzip(true);

    public static void main(String[] args) {
        WhiteListProxyStrategy whiteListProxyStrategy = new WhiteListProxyStrategy();
        whiteListProxyStrategy.addAllHost("www.tianya.cn");
        JSONFileAvProxyDumper jsonFileAvProxyDumper = new JSONFileAvProxyDumper();
        jsonFileAvProxyDumper.setDumpFileName("availableProxy.json");
        DungProxyContext dungProxyContext = DungProxyContext.create().setNeedProxyStrategy(whiteListProxyStrategy)
                .setAvProxyDumper(jsonFileAvProxyDumper);
        dungProxyContext.getGroupBindRouter().buildCombinationRule("www.tianya.cn:.*tianya.*");
        IpPoolHolder.init(dungProxyContext);

        Spider.create(new TianyaTest())
                .addUrl("http://search.tianya.cn/bbs?q=%E5%9F%8B%E7%BA%BF%E5%8F%8C%E7%9C%BC%E7%9A%AE")
                .addUrl("http://search.tianya.cn/bbs?q=%E5%BE%AE%E5%88%9B%E5%8F%8C%E7%9C%BC%E7%9A%AE")
                .addUrl("http://bbs.tianya.cn/post-funinfo-5695400-1.shtml").setDownloader(new DungProxyDownloader())
                .thread(1).run();
    }

    @Override
    public void process(Page page) {

    }

    @Override
    public Site getSite() {
        return site;
    }
}