java ByteBufferStringOperation.java 非阻塞爬虫..

package common; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; public class ByteBufferStringOperation { public static String ByteBufferToString(ByteBuffer buffer) throws UnsupportedEncodingException { buffer.flip(); byte[] array = new byte[buffer.limit()]; for (int i = 0; i < buffer.limit(); i++) { array[i] = buffer.get(); } String content = new String(array, "utf8"); array = null; return content; } public static ByteBuffer StringToByteBuffer(String content, int size) throws UnsupportedEncodingException { if (size <= 0) { System.err.println("invalid size"); return null; } ByteBuffer buffer = ByteBuffer.allocate(size); buffer.put(content.getBytes("utf8")); return buffer; } public static void main(String[] args) { try { String jj = "你丫的"; ByteBuffer buffer = ByteBufferStringOperation.StringToByteBuffer( jj, 1024 * 256); String fuck; fuck = ByteBufferStringOperation.ByteBufferToString(buffer); System.out.println(fuck); } catch (UnsupportedEncodingException e) { // TODO Auto-generated catch block e.printStackTrace(); } } }

 

package client; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.nio.channels.SelectionKey; import java.nio.channels.Selector; import java.nio.channels.SocketChannel; import java.util.Iterator; import common.ByteBufferStringOperation; import common.ThreadParameterSettings; public class SocketConnectionThread extends Thread { Selector selector; String url; int port; public SocketConnectionThread(String url, int port) { this.url = url; this.port = port; try { selector = Selector.open(); } catch (IOException e) { e.printStackTrace(); } } private void ProcessKey(SelectionKey key) throws IOException { SocketChannel client = ((SocketChannel) key.channel()); if (key.isReadable()) { ByteBuffer in = ByteBuffer.allocate(1024*256); String content = new String(); int flag; while(( flag= client.read(in))>0) { content += ByteBufferStringOperation.ByteBufferToString(in); in.clear(); // 读入 } System.out.println("Reade "+flag); if (flag == -1) { // cancel 指的是,将它从selector那解除注册,放入canceled key set 中 // 在下一次轮询时被remove // 那为什么之前的在迭代器中 直接remove后,这里又作了一次remove呢? //我懂了:第一次remove是将其移出 触发了操作的set里面 //这一次移出,将是真正的移出selector in = null; key.cancel(); client.close(); } System.out.println("Client Recieved Msg: " + ByteBufferStringOperation.ByteBufferToString(in)); return; } if (key.isWritable()) { //为了防止无限写 if(key.attachment()!=null&&(Integer)key.attachment()==1) return ; String HttpHeader = new String(); HttpHeader = "GET / HTTP/1.1/r/n"; HttpHeader +="Accept:*/*/r/n"; // HttpHeader +="User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)/n"; HttpHeader +="Host:www.baidu.com/r/n"; HttpHeader +="Connection:Keep-Alive/r/n/r/n"; /*ByteBuffer out = ByteBufferStringOperation .StringToByteBuffer("你好,Server");*/ ByteBuffer out =ByteBufferStringOperation.StringToByteBuffer(HttpHeader); int writeAmount = client.write(out); // while (out.hasRemaining()) //为了防止无限写,附上一个key key.attach(new Integer(1)); //感兴趣的设置为读 key.interestOps(SelectionKey.OP_READ); System.err.println("finish writing : " + writeAmount); } // 否则 不做任何处理,继续等待 return; } private void Connect() throws Exception { for (int i = 0; i < ThreadParameterSettings.Thread_MAX_SOCKET; i++) { SocketChannel client = SocketChannel.open(); client.configureBlocking(false); // client. client.connect(new InetSocketAddress(url, port)); // 向选择器注册我们感兴趣的通道,关注:可读,可写 /*client.register(selector, SelectionKey.OP_READ | SelectionKey.OP_WRITE);*/ //先写，再读 client.register(selector, SelectionKey.OP_WRITE); // ByteBuffer out = ByteBuffer.allocate(1024 * 256); // 等待连接成功 while (!client.finishConnect()) { Thread.sleep(100); } } } @Override public void run() { try { Connect(); // 非线程安全性,只能移除 不能增加 // 包含的是至少有一个情况被触发的所有key的集合 while (true) { if (selector.select() > 0) { Iterator<SelectionKey> iterator = selector.selectedKeys() .iterator(); SelectionKey key; while (iterator.hasNext()) { key = iterator.next(); ProcessKey(key); } } } } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } } }

 

 

 

package client; import java.io.IOException; import common.ThreadParameterSettings; public class SpiderClient implements Runnable{ private SocketConnectionThread[] connectionPool = new SocketConnectionThread[ThreadParameterSettings.MAX_THREAD]; @Override public void run() { try { for (int i = 0; i <ThreadParameterSettings.MAX_THREAD; i++) { SocketConnectionThread socketThread = new SocketConnectionThread( "www.baidu.com",80); connectionPool[i] = socketThread; socketThread.start(); Thread.sleep(500); } while(true) { // TODO Some thing Thread.sleep(500); } } catch (InterruptedException e) { e.printStackTrace(); } } public static void main(String[] args) throws IOException { SpiderClient s = new SpiderClient(); new Thread(s).start(); } }