[转]一个用php写的中文分词类

一个用php写的中文分词类

1. <?php
2. classSegmentation{
3. var$options=array('lowercase'=>TRUE,
4. 'segment_english'=>FALSE);
5. var$dict_name='Unknown';
6. var$dict_words=array();
7. functionsetLowercase($value){
8. if($value){
9. $this->options['lowercase']=TRUE;
10. }else{
11. $this->options['lowercase']=FALSE;
12. }
13. returnTRUE;
14. }
15. functionsetSegmentEnglish($value){
16. if($value){
17. $this->options['segment_english']=TRUE;
18. }else{
19. $this->options['segment_english']=FALSE;
20. }
21. returnTRUE;
22. }
23. functionload($dict_file){
24. if(!file_exists($dict_file)){
25. returnFALSE;
26. }
27. $fp=fopen($dict_file,'r');
28. $temp=fgets($fp,1024);
29. if($temp===FALSE){
30. returnFALSE;
31. }else{
32. if(strpos($temp,"\t")!==FALSE){
33. list($dict_type,$dict_name)=explode("\t",trim($temp));
34. }else{
35. $dict_type=trim($temp);
36. $dict_name='Unknown';
37. }
38. $this->dict_name=$dict_name;
39. if($dict_type!=='DICT_WORD_W'){
40. returnFALSE;
41. }
42. }
43. while(!feof($fp)){
44. $this->dict_words[rtrim(fgets($fp,32))]=1;
45. }
46. fclose($fp);
47. returnTRUE;
48. }
49. functiongetDictName(){
50. return$this->dict_name;
51. }
52. functionsegmentString($str){
53. if(count($this->dict_words)===0){
54. returnFALSE;
55. }
56. $lines=explode("\n",$str);
57. return$this->_segmentLines($lines);
58. }
59. functionsegmentFile($filename){
60. if(count($this->dict_words)===0){
61. returnFALSE;
62. }
63. $lines=file($filename);
64. return$this->_segmentLines($lines);
65. }
66. function_segmentLines($lines){
67. $contents_segmented='';
68. foreach($linesas$line){
69. $contents_segmented.=$this->_segmentLine(rtrim($line))."\n";
70. }
71. do{
72. $contents_segmented=str_replace('','',$contents_segmented);
73. }while(strpos($contents_segmented,'')!==FALSE);
74. return$contents_segmented;
75. }
76. function_segmentLine($str){
77. $str_final='';
78. $str_array=array();
79. $str_length=strlen($str);
80. if($str_length>0){
81. if(ord($str{$str_length-1})>=129){
82. $str.='';
83. }
84. }
85. for($i=0;$i<$str_length;$i++){
86. if(ord($str{$i})>=129){
87. $str_array[]=$str{$i}.$str{$i+1};
88. $i++;
89. }else{
90. $str_tmp=$str{$i};
91. for($j=$i+1;$j<$str_length;$j++){
92. if(ord($str{$j})<129){
93. $str_tmp.=$str{$j};
94. }else{
95. break;
96. }
97. }
98. $str_array[]=array($str_tmp);
99. $i=$j-1;
100. }
101. }
103. $pos=count($str_array);
104. while($pos>0){
105. $char=$str_array[$pos-1];
106. if(is_array($char)){
107. $str_final_tmp=$char[0];
108. if($this->options['segment_english']){
109. $str_final_tmp=preg_replace("/([\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\\\\\]\^\_\`\{\|\}\~\t\f]+)/","$1",$str_final_tmp);
110. $str_final_tmp=preg_replace("/([\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\\\\\]\^\_\`\{\|\}\~\t\f])([\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\\\\\]\^\_\`\{\|\}\~\t\f])/","$1$2",$str_final_tmp);
111. }
112. if($this->options['lowercase']){
113. $str_final_tmp=strtolower($str_final_tmp);
114. }
115. $str_final="$str_final_tmp$str_final";
116. $pos--;
117. }else{
118. $word_found=0;
119. $word_array=array(0=>'');
120. if($pos<4){
121. $word_temp=$pos+1;
122. }else{
123. $word_temp=5;
124. }
125. for($i=1;$i<$word_temp;$i++){
126. $word_array[$i]=$str_array[$pos-$i].$word_array[$i-1];
127. }
129. for($i=($word_temp-1);$i>1;$i--){
131. if(array_key_exists($word_array[$i],$this->dict_words)){
132. $word_found=$i;
133. break;
134. }
135. }
136. if($word_found){
137. $str_final="$word_array[$word_found]$str_final";
138. $pos=$pos-$word_found;
139. }else{
140. $str_final="$char$str_final";
141. $pos--;
142. }
143. }
144. }
145. return$str_final;
146. }
147. }
148. ?>

来源参考：
http://www.phpchina.cn/code/2006/0607/381.html
http://www.xuchao.cn/?play=reply&id=851