利用HttpURLConnection抓取网页取名

最新推荐文章于 2021-07-01 14:38:56 发布

z3h

最新推荐文章于 2021-07-01 14:38:56 发布

阅读量1.4k

点赞数

分类专栏： java技术文章标签： string import null url exception class

本文链接：https://blog.youkuaiyun.com/z3h/article/details/1796011

版权

java技术专栏收录该内容

18 篇文章

订阅专栏

本文介绍了一个使用Java HttpURLConnection与多线程技术抓取网页内容，并从中解析姓名及评分信息的小型示例程序。该程序通过指定网址抓取页面源码，再用正则表达式匹配特定字段。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

闲来无事，利用Java 的HttpURLConnection，使用多线程来抓取网页，计算名字的分数。

仅供娱乐。

程序如下

import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.net.HttpURLConnection;

import java.net.MalformedURLException;

import java.net.URL;

import org.apache.log4j.Logger;

public class XingMing {

static final Logger log = Logger.getLogger(XingMing. class );

public static String read(String urlStr) {

try {

URL url = new URL(urlStr);

HttpURLConnection connection = (HttpURLConnection) url

.openConnection();

connection.connect();

InputStream in = connection.getInputStream();

BufferedReader read = new BufferedReader( new InputStreamReader(in));

StringBuffer buf = new StringBuffer();

String line = null ;

while ((line = read.readLine()) != null ) {

buf.append(line);

}

return buf.toString();

} catch (MalformedURLException e) {

return null ;

} catch (IOException e) {

return null ;

}

public static String find(String str, String beginStr, String endStr) {

final int length = beginStr.length();

int index = str.indexOf(beginStr);

String result = null ;

if (index != -1) {

int index2 = str.indexOf(endStr, index + length);

if (index2 != -1) {

result = str.substring(index + beginStr.length(), index2);

}

return result;

}

public static String findName(String source, String name) {

// value= 我的姓名『 XX 』的分析：

return find(source, "value= 我的姓名『 " , " 』的分析 " );

}

public static String findScore(String source, String name) {

// 姓名评分：

// BT, 楷体 ">99.5

return find(

source,

" 姓名评分： " ,

"" );

}

public static void main(String[] args) throws IOException {

final char firstChar = ' 一 ' ;

final char lastChar = ' 龥 ' ;

// 最大开启 100 个线程，可以加快查询速度 .

int maxThread = 100;

int step = (lastChar - firstChar) / maxThread;

for ( int i = 0; i < maxThread; i++) {

char start = ( char ) (firstChar + i * step);

char end = ( char ) (firstChar + i * step + step - 1);

System. out .println( " 开启 " + (i + 1) + " 处理 :" + start + "-" + end

+ ( char ) (end + 1));

new CallThread(start, end).start();

}

static class CallThread extends Thread {

private char start ;

private char end ;

private String info ;

CallThread( char start, char end) {

this . start = start;

this . end = end;

this . info = this . start + "-" + this . end ;

}

public void run() {

// 姓

final char youname1 = ' 赵 ' ;

final String url = "http://www.xingming.net/cmjg-mz.asp?sex= 男 &youname1="

+ youname1 + "&youname2=" ;

String youname2;

String webinfo = null ;

for ( char i = start ; i <= end ; i++, webinfo = null ) {

// 名字规则自己取吧 .

// youname2 = " 良 " + i;

// youname2 = "" + i + i;

youname2 = i + " 敏 " ;

for ( int j = 0; j < 5 && webinfo == null ; j++) {

webinfo = XingMing.read(url + youname2);

}

if (webinfo == null ) {

log .warn( " 获取名字 [" + youname1 + youname2 + "] 失败 " );

continue ;

}

String webName = XingMing.findName(webinfo, "[" + youname2

+ "]" );

String webScore = XingMing.findScore(webinfo, "[" + youname2

+ "]" );

try {

if (Float.parseFloat(webScore) >= 90) {

System. out .println(youname2 + ":" + webName + ":"

+ webScore);

}

} catch (Exception e) {

}

log .info( this . info + ":" + webName + ":" + webScore);

if ((i - start ) % 100 == 0) {

System. out .println( this . info + " 处理了 " + (i - start ) + " 个 " );

}

System. out .println( this . info + " 结束了 ....." );

}