抓取36氨北极社区内容

private static void crawl36() {
		try {
			String urls = "http://www.36kr.com/topics/recent?page=1";
			String site = "http://www.36kr.com";
			HttpURLConnection con = getHttpURLConnection(urls);
			con.setRequestProperty(
					"User-Agent",
					"Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25");
			con.setRequestMethod("GET");
			String contentType = con.getContentType();
			if (contentType.indexOf("charset") == -1) {
				contentType = "GBK";
			} else {
				contentType = contentType.substring(
						contentType.indexOf("=") + 1, contentType.length());
			}
			Document document = Jsoup.parse(IOUtils.toString(
					con.getInputStream(), contentType));
			Elements elements = document.select(".infos");
			List<News> listNews = new ArrayList<News>();
			News news = null;
			for (Element element : elements) {
				Element elementUrl = element.select(".title").get(0);
				String url = elementUrl.getElementsByTag("a").attr("href");
				String title = elementUrl.getElementsByTag("a").text();
				String tag = element.select(".node").text();
				news = new News();
				news.setUrl(site + url);
				news.setTitle(title);
				news.setTag(tag);
				listNews.add(news);
			}
			System.out.println("抓取" + listNews.size() + "条");
			for (int i = 0; i < listNews.size(); i++) {
				System.out.print(listNews.get(i));
			}
		} catch (Exception e) {
		}
	}

待续