简单爬虫

1.首先是jar包

        <dependency>
          <groupId>org.jsoup</groupId>
          <artifactId>jsoup</artifactId>
          <version>1.11.1</version>
        </dependency>   

        <dependency>
            <groupId>commons-httpclient</groupId>
            <artifactId>commons-httpclient</artifactId>
            <version>3.1</version>
        </dependency>   

2.话不多说，直接上代码，使用HttpClient

// 抓取股票行情数据的
String baseUrl = "http://vip.stock.finance.sina.com.cn/quotes_service/view/CN_TransListV2.php?num=11&symbol=sh600004"+ "&rn=" + Math.round(new Date().getTime() / 60000);
        // 行情内容result一个字符串，切割切割就行
        String result = "";
        try {
            url=url.replaceAll(" ", "%20");
        GetMethod method = new GetMethod(url);
        HttpClient hc = new HttpClient();
        hc.executeMethod(method);
        InputStream resStream = method.getResponseBodyAsStream();
        result = FileUtil.readStream(resStream);
        } catch (Exception ex) {
            logger.error("获取腾讯实时详细行情信息失败。url=" + url, ex);
            return null;
        }

---

3.另一种使用jsoup是返回一个Document对象的

String url = "http://data.10jqka.com.cn/ipo/xgsgyzq/board/all/field/SSRQ/order/desc/ajax/1/";
        Connection conn = Jsoup
                .connect(url);
        // 修改http包中的header,伪装成浏览器进行抓取
        conn.header("User-Agent",
                "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, 
                like Gecko) Chrome/57.0.2987.98 Safari/537.36");
        Document doc = null;
        try {
            doc = conn.get();
        } catch (IOException e) {
            e.printStackTrace();
        }

        // 获取页数的链接
        Element table = doc.getElementById("maintable");// 获取表格
        Elements tr = table.getElementsByTag("tr");
        List<Node> tds = null;
        Calendar cal = Calendar.getInstance();
        cal.add(Calendar.DATE, 1);
        String tomorrowDate = new SimpleDateFormat("MM-dd").format(cal.getTime());
        List<SmsStockCodeDO> smsStockCodeDOlist = new ArrayList<SmsStockCodeDO>();
        SmsStockCodeDO smsStockCodeDO = null;
        for (int i = 0; i < tr.size(); i++) {
            tds = tr.get(i).childNodes();
            if (0 == tds.get(29).childNodeSize()) {
                continue;
            }
            if (tomorrowDate.equals(tds.get(29).childNodes().get(0).toString())) {
                // 多的逻辑就不写了
            }
        }