nutch中调用CrawlDatum的set（）函数的地方

（1）Injector中的InjectReducer内部类的reduce函数中第一次调用

（2）CrawlDbReducer中的reduce函数多次调用set函数

  /**
   * 输入是，CrawlDb，Segment\crawl_fetch和Segment\carwl_parse中的<Text,CrawlDatum>键值对。
   */
  public void reduce(Text key, Iterator<CrawlDatum> values,
                     OutputCollector<Text, CrawlDatum> output, Reporter reporter)
    throws IOException {

//	System.out.println("CrawlDbReducer.java");
    CrawlDatum fetch = new CrawlDatum();
    CrawlDatum old = new CrawlDatum();

    boolean fetchSet = false;
    boolean oldSet = false;
    byte[] signature = null;
    boolean multiple = false; // avoid deep copy when only single value exists
    linked.clear();
    org.apache.hadoop.io.MapWritable metaFromParse = null;
    boolean finished=false;
    //为什么reduce的输入会成为<Text,Iterator<CrawlDatum>>,因为reduce之前有一个将map结果进行合并操作
    while (values.hasNext()) {
      CrawlDatum datum = (CrawlDatum)values.next();
      /**
       * 添加：表示已经处理 完成
       */
      if(datum.getFinished()){
    	  finished=true;
      }
      
      if (!multiple && values.hasNext()) multiple = true;
      if (CrawlDatum.hasDbStatus(datum)) {//说明加入数据链接库，但还没有被抓去
        if (!oldSet) {
          if (multiple) {
        	System.err.println("CrawlDbReducer: if(!oldSet) 调用set函数设置old");
            old.set(datum);
          } else {
            // no need for a deep copy - this is the only value
            old = datum;
          }
          oldSet = true;
        } else {
          // always take the latest version
          if (old.getFetchTime() < datum.getFetchTime()){
          	  System.err.println("CrawlDbReducer: else 调用set函数设置old");
        	  old.set(datum);
          }
        }
        continue;
      }

      if (CrawlDatum.hasFetchStatus(datum)) {
        if (!fetchSet) {
          if (multiple) {
          	System.err.println("CrawlDbReducer: if(!fetchSet) 调用set函数设置fetch");

            fetch.set(datum);
          } else {
            fetch = datum;
          }
          fetchSet = true;
        } else {
          // always take the latest version
          if (fetch.getFetchTime() < datum.getFetchTime()){
          	System.err.println("CrawlDbReducer: else 调用set函数设置fetch");

        	  fetch.set(datum);
          }
        }
        continue;
      }
      //上面两个过滤条件都是根据fetchTime找到最新的状态CrawlDatum,然后将将最新的状态保留

     //下面是对应已经返回过的状态，就是解析过的CrawlDatum状态
      switch (datum.getStatus()) {                // collect other info
      case CrawlDatum.STATUS_LINKED:
        CrawlDatum link;
        if (multiple) {
          link = new CrawlDatum();
      	System.err.println("CrawlDbReducer: while(values.hasNext()) case CarwlDatum.status_linked 调用set函数设置link");

          link.set(datum);
        } else {
          link = datum;
        }
        linked.insert(link);
        break;
      case CrawlDatum.STATUS_SIGNATURE:
        signature = datum.getSignature();
        break;
      case CrawlDatum.STATUS_PARSE_META:
        metaFromParse = datum.getMetaData();
        break;
      default:
        LOG.warn("Unknown status, key: " + key + ", datum: " + datum);
      }
    }
    
    // copy the content of the queue into a List
    // in reversed order
    int numLinks = linked.size();
    List<CrawlDatum> linkList = new ArrayList<CrawlDatum>(numLinks);
    for (int i = numLinks - 1; i >= 0; i--) {
      linkList.add(linked.pop());
    }
    
    // if it doesn't already exist, skip it
    /**
     * 不允许添加新连接，而且该url以前在链接库中不能存在，所以要忽略它
     */
    if (!oldSet && !additionsAllowed) return;
    
    // if there is no fetched datum, perhaps there is a link
    /**
     * 说明是一个链接
     */
    if (!fetchSet && linkList.size() > 0) {
      fetch = linkList.get(0);
      fetchSet = true;
    }
    
    // still no new data - record only unchanged old data, if exists, and return
    if (!fetchSet) {
      if (oldSet) {// at this point at least "old" should be present
    	  
//    	System.out.println("CrawlDbReducer->!fetchSet: url="+key);
    	old.setFinished(finished);//添加
    	System.err.println("old=>>>>>>>>>>>>>>>>>>>>>>>>>+start="+old.getIfStart()+" <<<<<<<<<+finished="+old.getFinished());
    	
        output.collect(key, old);//因为将所有的url和他的CrawlDatum另存到一个地址，新地址将作为新的连接数据库
      } else {
        LOG.warn("Missing fetch and old value, signature=" + signature);
      }
      return;
    }
    
    if (signature == null) signature = fetch.getSignature();
    long prevModifiedTime = oldSet ? old.getModifiedTime() : 0L;
    long prevFetchTime = oldSet ? old.getFetchTime() : 0L;

    // initialize with the latest version, be it fetch or link
	System.err.println("CrawlDbReducer: 调用set函数设置result");

    result.set(fetch);
    if (oldSet) {
      // copy metadata from old, if exists
      if (old.getMetaData().size() > 0) {
        result.putAllMetaData(old);
        // overlay with new, if any
        if (fetch.getMetaData().size() > 0)
          result.putAllMetaData(fetch);
      }
      // set the most recent valid value of modifiedTime
      if (old.getModifiedTime() > 0 && fetch.getModifiedTime() == 0) {
        result.setModifiedTime(old.getModifiedTime());
      }
    }
    
    switch (fetch.getStatus()) {                // determine new status

    case CrawlDatum.STATUS_LINKED:                // it was link
      if (oldSet) {                          // if old exists
      	System.err.println("CrawlDbReducer: case CrawlDatum.sattus_linked 调用set函数设置result");

        result.set(old);                          // use it
      } else {
        result = schedule.initializeSchedule((Text)key, result);
        result.setStatus(CrawlDatum.STATUS_DB_UNFETCHED);
        try {
          scfilters.initialScore((Text)key, result);
        } catch (ScoringFilterException e) {
          if (LOG.isWarnEnabled()) {
            LOG.warn("Cannot filter init score for url " + key +
                     ", using default: " + e.getMessage());
          }
          result.setScore(0.0f);
        }
      }
      break;
      
    case CrawlDatum.STATUS_FETCH_SUCCESS:         // succesful fetch
    case CrawlDatum.STATUS_FETCH_REDIR_TEMP:      // successful fetch, redirected
    case CrawlDatum.STATUS_FETCH_REDIR_PERM:
    case CrawlDatum.STATUS_FETCH_NOTMODIFIED:     // successful fetch, notmodified
      // determine the modification status
      int modified = FetchSchedule.STATUS_UNKNOWN;
      if (fetch.getStatus() == CrawlDatum.STATUS_FETCH_NOTMODIFIED) {
        modified = FetchSchedule.STATUS_NOTMODIFIED;
      } else {
        if (oldSet && old.getSignature() != null && signature != null) {
          if (SignatureComparator._compare(old.getSignature(), signature) != 0) {
            modified = FetchSchedule.STATUS_MODIFIED;
          } else {
            modified = FetchSchedule.STATUS_NOTMODIFIED;
          }
        }
      }
      // set the schedule
      result = schedule.setFetchSchedule((Text)key, result, prevFetchTime,
          prevModifiedTime, fetch.getFetchTime(), fetch.getModifiedTime(), modified);
      // set the result status and signature
      if (modified == FetchSchedule.STATUS_NOTMODIFIED) {
        result.setStatus(CrawlDatum.STATUS_DB_NOTMODIFIED);
        if (oldSet) result.setSignature(old.getSignature());
      } else {
        switch (fetch.getStatus()) {
        case CrawlDatum.STATUS_FETCH_SUCCESS:
          result.setStatus(CrawlDatum.STATUS_DB_FETCHED);
          break;
        case CrawlDatum.STATUS_FETCH_REDIR_PERM:
          result.setStatus(CrawlDatum.STATUS_DB_REDIR_PERM);
          break;
        case CrawlDatum.STATUS_FETCH_REDIR_TEMP:
          result.setStatus(CrawlDatum.STATUS_DB_REDIR_TEMP);
          break;
        default:
          LOG.warn("Unexpected status: " + fetch.getStatus() + " resetting to old status.");
          if (oldSet) result.setStatus(old.getStatus());
          else result.setStatus(CrawlDatum.STATUS_DB_UNFETCHED);
        }
        result.setSignature(signature);
        if (metaFromParse != null) {
            for (Entry<Writable, Writable> e : metaFromParse.entrySet()) {
              result.getMetaData().put(e.getKey(), e.getValue());
            }
          }
      }
      // if fetchInterval is larger than the system-wide maximum, trigger
      // an unconditional recrawl. This prevents the page to be stuck at
      // NOTMODIFIED state, when the old fetched copy was already removed with
      // old segments.
      if (maxInterval < result.getFetchInterval())
        result = schedule.forceRefetch((Text)key, result, false);
      break;
    case CrawlDatum.STATUS_SIGNATURE:
      if (LOG.isWarnEnabled()) {
        LOG.warn("Lone CrawlDatum.STATUS_SIGNATURE: " + key);
      }   
      return;
    case CrawlDatum.STATUS_FETCH_RETRY:           // temporary failure
      if (oldSet) {
        result.setSignature(old.getSignature());  // use old signature
      }
      result = schedule.setPageRetrySchedule((Text)key, result, prevFetchTime,
          prevModifiedTime, fetch.getFetchTime());
      if (result.getRetriesSinceFetch() < retryMax) {
        result.setStatus(CrawlDatum.STATUS_DB_UNFETCHED);
      } else {
        result.setStatus(CrawlDatum.STATUS_DB_GONE);
      }
      break;

    case CrawlDatum.STATUS_FETCH_GONE:            // permanent failure
      if (oldSet)
        result.setSignature(old.getSignature());  // use old signature
      result.setStatus(CrawlDatum.STATUS_DB_GONE);
      result = schedule.setPageGoneSchedule((Text)key, result, prevFetchTime,
          prevModifiedTime, fetch.getFetchTime());
      break;

    default:
      throw new RuntimeException("Unknown status: " + fetch.getStatus() + " " + key);
    }

    try {
      scfilters.updateDbScore((Text)key, oldSet ? old : null, result, linkList);
    } catch (Exception e) {
      if (LOG.isWarnEnabled()) {
        LOG.warn("Couldn't update score, key=" + key + ": " + e);
      }
    }
    // remove generation time, if any
    result.getMetaData().remove(Nutch.WRITABLE_GENERATE_TIME_KEY);
//    System.out.println("CrawlDbReducer: url="+key);
    result.setFinished(finished);//添加
	System.err.println("result>>>>>>>>>>>>>>>>>>>>>>>>>+start="+old.getIfStart()+" <<<<<<<<<+finished="+old.getFinished());

    output.collect(key, result);
  }
  
}