Hadoop RPC实现NIO通信client端分析

一、RPC.对Client的管理

1、管理包括：Client实例的获取，client的关闭

/**
	   * Construct a client-side proxy object with the default SocketFactory
	   * 
	   * @param protocol
	   * @param clientVersion
	   * @param addr
	   * @param conf
	   * @return a proxy instance
	   * @throws IOException
	   */
	  public static ProtocolInterface getProxy(
	      Class<? extends ProtocolInterface> protocol,
	      long clientVersion, InetSocketAddress addr, Configuration conf)
	      throws IOException {
	    return getProxy(protocol, clientVersion, addr, conf,
	        NetUtils.getDefaultSocketFactory(conf), 0);
	  }

	  public static ProtocolInterface getProxy(
	      Class<? extends ProtocolInterface> protocol,
	      long clientVersion, InetSocketAddress addr, Configuration conf, int rpcTimeout)
	      throws IOException {

	    return getProxy(protocol, clientVersion, addr, conf,
	        NetUtils.getDefaultSocketFactory(conf), rpcTimeout);
	  }

	  /**
	   * Stop this proxy and release its invoker's resource
	   * @param proxy the proxy to be stopped
	   */
	  public static void stopProxy(ProtocolInterface proxy) {
	    if (proxy!=null) {
	      ((Invoker)Proxy.getInvocationHandler(proxy)).close();
	    }
	  }

2、getProxy方法 new一个Invoker实例

/** Construct a client-side proxy object that implements the named protocol,
	   * talking to a server at the named address. */
	  public static ProtocolInterface getProxy(
	      Class<? extends ProtocolInterface> protocol,
	      long clientVersion, InetSocketAddress addr,
	      Configuration conf, SocketFactory factory, int rpcTimeout) throws IOException {

	    ProtocolInterface proxy =
	        (ProtocolInterface) Proxy.newProxyInstance(
	            protocol.getClassLoader(), new Class[] { protocol },
	            new Invoker(protocol, addr, conf, factory, rpcTimeout));
//	     serverVersion = proxy.getProtocolVersion(protocol.getName(), clientVersion);
	    /*if (serverVersion == clientVersion) {
	      return proxy;
	    } else {
	      throw new VersionMismatch(protocol.getName(), clientVersion, 
	                                serverVersion);
	    }*/
	    return proxy;
//	    return null;
	  }

3、在Invoker的构造函数中涉及了Client对象的缓存CLIENTS.getClient(,)

private static class Invoker implements InvocationHandler {
		private Client.ConnectionId remoteId;
		private Client client;
		private boolean isClosed = false;
		
		public Invoker(Class<?> protocol,
				InetSocketAddress address, Configuration conf,
				SocketFactory factory, int rpcTimeout) {
			this.remoteId = ConnectionId.getConnectionId(address,
					protocol, rpcTimeout, conf);
			this.client = CLIENTS.getClient(conf, factory);
		}
		@Override
		public Object invoke(Object proxy, Method method, Object[] args)
			throws Throwable {
			final boolean logDebug = LOG.isDebugEnabled();
			long startTime = 0;
			if (logDebug) {
			    startTime = System.currentTimeMillis();
			}

			 ObjectWritable value = (ObjectWritable)
			    client.call(new Invocation(method, args), remoteId);
			 if (logDebug) {
			    long callTime = System.currentTimeMillis() - startTime;
			    LOG.debug("Call: " + method.getName() + " " + callTime);
			  }
			  return value.get();
			  }
			    
		/* close the IPC client that's responsible for this invoker's RPCs */ 
	    synchronized private void close() {
	      if (!isClosed) {
	        isClosed = true;
	        CLIENTS.stopClient(client);
	      }
	    }	    

	}

4、现在看下getClient方法

/**
	     * Construct & cache an IPC client with the user-provided SocketFactory 
	     * if no cached client exists.
	     * 
	     * @param conf Configuration
	     * @return an IPC client
	     */
	    private synchronized Client getClient(Configuration conf,
	        SocketFactory factory) {
	      // Construct & cache client.  The configuration is only used for timeout,
	      // and Clients have connection pools.  So we can either (a) lose some
	      // connection pooling and leak sockets, or (b) use the same timeout for all
	      // configurations.  Since the IPC is usually intended globally, not
	      // per-job, we choose (a).
	      Client client = clients.get(factory);
	      if (client == null) {
	        client = new Client(ObjectWritable.class, conf, factory);
	        clients.put(factory, client);
	      } else {
	        client.incCount();
	      }
	      return client;
	    }

PRC中之所以保存了Client对象，因为一个Client对象可能被多个线程用到，因此在Client对象要有引用计数。计数方法：当从缓存中拿出一个Client时增加计数，stopClient时首先减少计数，如果发现计数为零才能从缓存中清除这个Client并断开Client的连接。

二、Client对连接的管理

Client缓存在RPC中并且以SocketFacoty作为key。Connection使用ConnectionId作为标识并缓存在Client中。ConnectionId含有远端地址，协议，用户票据(本文未考虑)信息。也就是说，同一个用户会使用同一个协议向同一远端发送多个Call会使用同一条连接。那么看来Connection有必要保存一个Call的列表了（addCall ,cleanupcalls），有必要建立连接了(setupIostreams)，要有个方法发送参数sendParm,有线程来接受回应(run, waitForWork, receiveReponse).

三、receiveResponse会不断从网路中读入信息（CallID+返回值 or 异常 or 致命错误），没收到信息touch方法要更新连接时间戳。

/* Receive a response.
	     * Because only one receiver, so no synchronization on in.
	     */
	    private void receiveResponse() {
	      if (shouldCloseConnection.get()) {
	        return;
	      }
	      touch();
	      
	      try {
	        int id = in.readInt();                    // try to read an id

	        if (LOG.isDebugEnabled())
	          LOG.debug(getName() + " got value #" + id);

	        Call call = calls.get(id);

	        int state = in.readInt();     // read call status
	        if (state == Status.SUCCESS.state) {
	          Writable value = ReflectionUtils.newInstance(valueClass, conf);
	          ((ObjectWritable)value).setConf(conf);
	          value.readFields(in);                 // read value
	          call.setValue(value);
	          calls.remove(id);
	        } else if (state == Status.ERROR.state) {
	          call.setException(new RemoteException(WritableUtils.readString(in),
	                                                WritableUtils.readString(in)));
	          calls.remove(id);
	        } else if (state == Status.FATAL.state) {
	          // Close the connection
	          markClosed(new RemoteException(WritableUtils.readString(in), 
	                                         WritableUtils.readString(in)));
	        }
	      } catch (IOException e) {
	        markClosed(e);
	      }
	    }

四、Call的调用过程

/** Make a call, passing <code>param</code>, to the IPC server defined by
	   * <code>remoteId</code>, returning the value.  
	   * Throws exceptions if there are network problems or if the remote code 
	   * threw an exception. */
	  public Writable call(Writable param, ConnectionId remoteId)  
	                       throws InterruptedException, IOException {
	    Call call = new Call(param);
	    Connection connection = getConnection(remoteId, call);
	    connection.sendParam(call);                 // send the parameter
	    boolean interrupted = false;
	    synchronized (call) {
	      while (!call.done) {
	        try {
	          call.wait();                           // wait for the result
	        } catch (InterruptedException ie) {
	          // save the fact that we were interrupted
	          interrupted = true;
	        }
	      }

	      if (interrupted) {
	        // set the interrupt flag now that we are done waiting
	        Thread.currentThread().interrupt();
	      }

	      if (call.error != null) {
	        if (call.error instanceof RemoteException) {
	          call.error.fillInStackTrace();
	          throw call.error;
	        } else { // local exception
	          // use the connection because it will reflect an ip change, unlike
	          // the remoteId
	          throw wrapException(connection.getRemoteAddress(), call.error);
	        }
	      } else {
	        return call.value;
	      }
	    }
	  }

需要注意的是：

1、setupIOStreams建立连接并接受线程，难点是在NetUtils.Conntect这个高效的连接socket方法，使用到了java NIO。下篇文章会分析

2、数据流的封装，发送使用到的Write有关的类建议学习下

3、Client发送消息的协议请阅读：hadoop RPC的发展及其应用层协议格式 http://fengshenwu.com/blog/2012/11/13/hadoop-rpc_develop/

参考：http://www.360doc.com/content/11/0711/16/3294720_132910457.shtml