字符串Hash函数

Hash查找因为其O(1)的查找性能而著称,被对查找性能要求高的应用所广泛采用。它的基本思想是:
(1) 创建一个定长的线性Hash表,一般可以初始化时指定length;

(2) 设计Hash函数,将关键字key散射到Hash表中。其中hash函数设计是最为关键的,均匀分布、冲突概率小全在它;

(3) 通常采用拉链方法来解决hash冲突问题,即散射到同一个hash表项的关键字,以链表形式来表示(也称为桶backet);

(4) 给定关键字key,就可以在O(1) + O(m)的时间复杂度内定位到目标。其中,m为拉链长度,即桶深。

Hash应用中,字符串是最为常见的关键字,应用非常普通,现在的程序设计语言中基本上都提供了字符串hash表的支持。字符串hash函数非常多,常见的主要有Simple_hash, RS_hash, JS_hash, PJW_hash, ELF_hash, BKDR_hash, SDBM_hash, DJB_hash, AP_hash, CRC_hash等。它们的C语言实现见后面附录代码: hash.h, hash.c。那么这么些字符串hash函数,谁好熟非呢?评估hash函数优劣的基准主要有以下两个指标:

(1) 散列分布性

即桶的使用率backet_usage = (已使用桶数) / (总的桶数),这个比例越高,说明分布性良好,是好的hash设计。

(2) 平均桶长

即avg_backet_len,所有已使用桶的平均长度。理想状态下这个值应该=1,越小说明冲突发生地越少,是好的hash设计。

hash函数计算一般都非常简洁,因此在耗费计算时间复杂性方面判别甚微,这里不作对比。

评估方案设计是这样的:

(1) 以200M的视频文件作为输入源,以4KB的块为大小计算MD5值,并以此作为hash关键字;

(2) 分别应用上面提到的各种字符串hash函数,进行hash散列模拟;

(3) 统计结果,用散列分布性和平均桶长两个指标进行评估分析。

测试程序见附录代码hashtest.c,测试结果如下表所示。从这个结果我们也可以看出,这些字符串hash函数真是不相仲伯,难以决出高低,所以实际应用中可以根据喜好选择。当然,最好实际测试一下,毕竟应用特点不大相同。其他几组测试结果也类似,这里不再给出。

Hash函数桶数Hash调用总数最大桶长平均桶长桶使用率%
simple_hash1024047198164.6399.00%
RS_hash1024047198164.6398.91%
JS_hash1024047198154.6498.87%
PJW_hash1024047198164.6399.00%
ELF_hash1024047198164.6399.00%
BKDR_hash1024047198164.6399.00%
SDBM_hash1024047198164.6398.90%
DJB_hash1024047198154.6498.85%
AP_hash1024047198164.6398.96%
CRC_hash1024047198164.6498.77%

附录源代码:

hash.h

 

  1. #ifndef _HASH_H 
  2. #define _HASH_H 
  3.  
  4. #ifdef __cplusplus 
  5. extern "C"
  6. #endif 
  7.  
  8. /* A Simple Hash Function */ 
  9. unsigned int simple_hash(char *str); 
  10.  
  11. /* RS Hash Function */ 
  12. unsigned int RS_hash(char *str); 
  13.  
  14. /* JS Hash Function */ 
  15. unsigned int JS_hash(char *str); 
  16.  
  17. /* P. J. Weinberger Hash Function */ 
  18. unsigned int PJW_hash(char *str); 
  19.  
  20. /* ELF Hash Function */ 
  21. unsigned int ELF_hash(char *str); 
  22.  
  23. /* BKDR Hash Function */ 
  24. unsigned int BKDR_hash(char *str); 
  25.  
  26. /* SDBM Hash Function */ 
  27. unsigned int SDBM_hash(char *str); 
  28.  
  29. /* DJB Hash Function */ 
  30. unsigned int DJB_hash(char *str); 
  31.  
  32. /* AP Hash Function */ 
  33. unsigned int AP_hash(char *str); 
  34.  
  35. /* CRC Hash Function */ 
  36. unsigned int CRC_hash(char *str); 
  37.  
  38. #ifdef __cplusplus 
  39. #endif 
  40.  
  41. #endif 

 

hash.c

  • #include <string.h> 
  • #include "hash.h" 
  •  
  • /* A Simple Hash Function */ 
  • unsigned int simple_hash(char *str) 
  •     register unsigned int hash; 
  •     register unsigned char *p; 
  •  
  •     for(hash = 0, p = (unsigned char *)str; *p ; p++) 
  •         hash = 31 * hash + *p; 
  •  
  •     return (hash & 0x7FFFFFFF); 
  •  
  • /* RS Hash Function */ 
  • unsigned int RS_hash(char *str) 
  •          unsigned int b = 378551; 
  •          unsigned int a = 63689; 
  •          unsigned int hash = 0; 
  •  
  •          while (*str) 
  •          { 
  •                  hash = hash * a + (*str++); 
  •                  a *= b; 
  •          } 
  •  
  •          return (hash & 0x7FFFFFFF); 
  •  
  • /* JS Hash Function */ 
  • unsigned int JS_hash(char *str) 
  •          unsigned int hash = 1315423911; 
  •  
  •          while (*str) 
  •          { 
  •                  hash ^= ((hash << 5) + (*str++) + (hash >> 2)); 
  •          } 
  •          
  •          return (hash & 0x7FFFFFFF); 
  •  
  • /* P. J. Weinberger Hash Function */ 
  • unsigned int PJW_hash(char *str) 
  •          unsigned int BitsInUnignedInt = (unsigned int)(sizeof(unsigned int) * 8); 
  •          unsigned int ThreeQuarters     = (unsigned int)((BitsInUnignedInt   * 3) / 4); 
  •          unsigned int OneEighth         = (unsigned int)(BitsInUnignedInt / 8); 
  •  
  •          unsigned int HighBits          = (unsigned int)(0xFFFFFFFF) << (BitsInUnignedInt - OneEighth); 
  •          unsigned int hash              = 0; 
  •          unsigned int test              = 0; 
  •  
  •          while (*str) 
  •          { 
  •                  hash = (hash << OneEighth) + (*str++); 
  •                  if ((test = hash & HighBits) != 0) 
  •                  { 
  •                          hash = ((hash ^ (test >> ThreeQuarters)) & (~HighBits)); 
  •                  } 
  •          } 
  •  
  •          return (hash & 0x7FFFFFFF); 
  •  
  • /* ELF Hash Function */ 
  • unsigned int ELF_hash(char *str) 
  •          unsigned int hash = 0; 
  •          unsigned int x     = 0; 
  •  
  •          while (*str) 
  •          { 
  •                  hash = (hash << 4) + (*str++); 
  •                  if ((x = hash & 0xF0000000L) != 0) 
  •                  { 
  •                          hash ^= (x >> 24); 
  •                          hash &= ~x; 
  •                  } 
  •          } 
  •  
  •          return (hash & 0x7FFFFFFF); 
  •  
  • /* BKDR Hash Function */ 
  • unsigned int BKDR_hash(char *str) 
  •          unsigned int seed = 131; // 31 131 1313 13131 131313 etc.. 
  •          unsigned int hash = 0; 
  •  
  •          while (*str) 
  •          { 
  •                  hash = hash * seed + (*str++); 
  •          } 
  •  
  •          return (hash & 0x7FFFFFFF); 
  •  
  • /* SDBM Hash Function */ 
  • unsigned int SDBM_hash(char *str) 
  •          unsigned int hash = 0; 
  •  
  •          while (*str) 
  •          { 
  •                  hash = (*str++) + (hash << 6) + (hash << 16) - hash; 
  •          } 
  •  
  •          return (hash & 0x7FFFFFFF); 
  •  
  • /* DJB Hash Function */ 
  • unsigned int DJB_hash(char *str) 
  •          unsigned int hash = 5381; 
  •  
  •          while (*str) 
  •          { 
  •                  hash += (hash << 5) + (*str++); 
  •          } 
  •  
  •          return (hash & 0x7FFFFFFF); 
  •  
  • /* AP Hash Function */ 
  • unsigned int AP_hash(char *str) 
  •          unsigned int hash = 0; 
  •          int i; 
  •          for (i=0; *str; i++) 
  •          { 
  •                  if ((i & 1) == 0) 
  •                  { 
  •                          hash ^= ((hash << 7) ^ (*str++) ^ (hash >> 3)); 
  •                  } 
  •                  else 
  •                  { 
  •                          hash ^= (~((hash << 11) ^ (*str++) ^ (hash >> 5))); 
  •                  } 
  •          } 
  •  
  •          return (hash & 0x7FFFFFFF); 
  •  
  • /* CRC Hash Function */ 
  • unsigned int CRC_hash(char *str) 
  •     unsigned int        nleft   = strlen(str); 
  •     unsigned long long  sum     = 0; 
  •     unsigned short int *w       = (unsigned short int *)str; 
  •     unsigned short int  answer  = 0; 
  •  
  •     /*
  •      * Our algorithm is simple, using a 32 bit accumulator (sum), we add
  •      * sequential 16 bit words to it, and at the end, fold back all the
  •      * carry bits from the top 16 bits into the lower 16 bits.
  •      */ 
  •     while ( nleft > 1 ) { 
  •         sum += *w++; 
  •         nleft -= 2; 
  •     } 
  •     /*
  •      * mop up an odd byte, if necessary
  •      */ 
  •     if ( 1 == nleft ) { 
  •         *( unsigned char * )( &answer ) = *( unsigned char * )w ; 
  •         sum += answer; 
  •     } 
  •     /*
  •      * add back carry outs from top 16 bits to low 16 bits
  •      * add hi 16 to low 16
  •      */ 
  •     sum = ( sum >> 16 ) + ( sum & 0xFFFF ); 
  •     /* add carry */ 
  •     sum += ( sum >> 16 ); 
  •     /* truncate to 16 bits */ 
  •     answer = ~sum; 
  •  
  •     return (answer & 0xFFFFFFFF); 
  •  

    hashtest.c

     

    1. #include <stdio.h> 
    2. #include <stdlib.h> 
    3. #include <sys/types.h> 
    4. #include <sys/stat.h> 
    5. #include <fcntl.h> 
    6. #include <errno.h> 
    7. #include <string.h> 
    8. #include "hash.h" 
    9. #include "md5.h" 
    10. struct hash_key { 
    11.     unsigned char *key; 
    12.     struct hash_key *next;  
    13. }; 
    14. struct hash_counter_entry { 
    15.     unsigned int hit_count; 
    16.     unsigned int entry_count; 
    17.     struct hash_key *keys; 
    18. }; 
    19. #define BLOCK_LEN   4096 
    20. static int backet_len = 10240; 
    21. static int hash_call_count = 0; 
    22. static struct hash_counter_entry *hlist = NULL; 
    23. unsigned int (*hash_func)(char *str); 
    24. void choose_hash_func(char *hash_func_name) 
    25.     if (0 == strcmp(hash_func_name, "simple_hash")) 
    26.         hash_func = simple_hash; 
    27.     else if (0 == strcmp(hash_func_name, "RS_hash")) 
    28.         hash_func = RS_hash; 
    29.     else if (0 == strcmp(hash_func_name, "JS_hash")) 
    30.         hash_func = JS_hash; 
    31.     else if (0 == strcmp(hash_func_name, "PJW_hash")) 
    32.         hash_func = PJW_hash; 
    33.     else if (0 == strcmp(hash_func_name, "ELF_hash")) 
    34.         hash_func = ELF_hash; 
    35.     else if (0 == strcmp(hash_func_name, "BKDR_hash")) 
    36.         hash_func = BKDR_hash; 
    37.     else if (0 == strcmp(hash_func_name, "SDBM_hash")) 
    38.         hash_func = SDBM_hash; 
    39.     else if (0 == strcmp(hash_func_name, "DJB_hash")) 
    40.         hash_func = DJB_hash; 
    41.     else if (0 == strcmp(hash_func_name, "AP_hash")) 
    42.         hash_func = AP_hash; 
    43.     else if (0 == strcmp(hash_func_name, "CRC_hash")) 
    44.         hash_func = CRC_hash; 
    45.     else 
    46.         hash_func = NULL; 
    47. void insert_hash_entry(unsigned char *key, struct hash_counter_entry *hlist) 
    48.     unsigned int hash_value = hash_func(key) % backet_len; 
    49.     struct hash_key *p; 
    50.     p = hlist[hash_value].keys; 
    51.     while(p) { 
    52.         if (0 == strcmp(key, p->key)) 
    53.             break; 
    54.         p = p->next; 
    55.     } 
    56.     if (p == NULL) 
    57.     { 
    58.         p = (struct hash_key *)malloc(sizeof(struct hash_key)); 
    59.         if (p == NULL)  
    60.         { 
    61.             perror("malloc in insert_hash_entry"); 
    62.             return; 
    63.         } 
    64.         p->key = strdup(key); 
    65.         p->next = hlist[hash_value].keys; 
    66.         hlist[hash_value].keys = p; 
    67.         hlist[hash_value].entry_count++; 
    68.     } 
    69.     hlist[hash_value].hit_count++; 
    70. void hashtest_init() 
    71.     int i; 
    72.     hash_call_count = 0; 
    73.     hlist = (struct hash_counter_entry *) malloc (sizeof(struct hash_counter_entry) *  backet_len); 
    74.     if (NULL == hlist) 
    75.     { 
    76.         perror("malloc in hashtest_init"); 
    77.         return; 
    78.     } 
    79.     for (i = 0; i < backet_len; i++) 
    80.     { 
    81.         hlist[i].hit_count = 0; 
    82.         hlist[i].entry_count = 0; 
    83.         hlist[i].keys = NULL; 
    84.     } 
    85. void hashtest_clean() 
    86.     int i; 
    87.     struct hash_key *pentry, *p; 
    88.     if (NULL == hlist) 
    89.         return; 
    90.     for (i = 0; i < backet_len; ++i) 
    91.     { 
    92.         pentry = hlist[i].keys; 
    93.         while(pentry) 
    94.         { 
    95.             p = pentry->next; 
    96.             if (pentry->key) free(pentry->key); 
    97.             free(pentry); 
    98.             pentry = p; 
    99.         } 
    100.     } 
    101.     free(hlist); 
    102. void show_hashtest_result() 
    103.     int i, backet = 0, max_link = 0, sum = 0; 
    104.     int conflict_count = 0, hit_count = 0; 
    105.     float avg_link, backet_usage; 
    106.     for(i = 0; i < backet_len; i++) 
    107.     { 
    108.         if (hlist[i].hit_count > 0)  
    109.         { 
    110.             backet++; 
    111.             sum += hlist[i].entry_count; 
    112.             if (hlist[i].entry_count > max_link) 
    113.             { 
    114.                 max_link = hlist[i].entry_count; 
    115.             } 
    116.             if (hlist[i].entry_count > 1) 
    117.             { 
    118.                 conflict_count++; 
    119.             } 
    120.             hit_count += hlist[i].hit_count; 
    121.         } 
    122.     } 
    123.     backet_usage = backet/1.0/backet_len * 100;; 
    124.     avg_link = sum/1.0/backet; 
    125.     printf("backet_len = %d/n", backet_len); 
    126.     printf("hash_call_count = %d/n", hash_call_count); 
    127.     printf("hit_count = %d/n", hit_count); 
    128.     printf("conflict count = %d/n", conflict_count); 
    129.     printf("longest hash entry = %d/n", max_link); 
    130.     printf("average hash entry length = %.2f/n", avg_link); 
    131.     printf("backet usage = %.2f%/n", backet_usage); 
    132. void usage() 
    133.     printf("Usage: hashtest filename hash_func_name [backet_len]/n"); 
    134.     printf("hash_func_name:/n"); 
    135.     printf("/tsimple_hash/n"); 
    136.     printf("/tRS_hash/n"); 
    137.     printf("/tJS_hash/n"); 
    138.     printf("/tPJW_hash/n"); 
    139.     printf("/tELF_hash/n"); 
    140.     printf("/tBKDR_hash/n"); 
    141.     printf("/tSDBM_hash/n"); 
    142.     printf("/tDJB_hash/n"); 
    143.     printf("/tAP_hash/n"); 
    144.     printf("/tCRC_hash/n"); 
    145. void md5_to_32(unsigned char *md5_16, unsigned char *md5_32) 
    146.     int i; 
    147.     for (i = 0; i < 16; ++i) 
    148.     { 
    149.         sprintf(md5_32 + i * 2, "%02x", md5_16[i]); 
    150.     } 
    151. int main(int argc, char *argv[]) 
    152.     int fd = -1, rwsize = 0; 
    153.     unsigned char md5_checksum[16 + 1] = {0}; 
    154.     unsigned char buf[BLOCK_LEN] = {0}; 
    155.     if (argc < 3)  
    156.     { 
    157.         usage(); 
    158.         return -1; 
    159.     } 
    160.     if (-1 == (fd = open(argv[1], O_RDONLY))) 
    161.     { 
    162.         perror("open source file"); 
    163.         return errno; 
    164.     } 
    165.     if (argc == 4) 
    166.     { 
    167.         backet_len = atoi(argv[3]); 
    168.     } 
    169.     hashtest_init(); 
    170.     choose_hash_func(argv[2]); 
    171.     while (rwsize = read(fd, buf, BLOCK_LEN)) 
    172.     { 
    173.         md5(buf, rwsize, md5_checksum); 
    174.         insert_hash_entry(md5_checksum, hlist); 
    175.         hash_call_count++; 
    176.         memset(buf, 0, BLOCK_LEN); 
    177.         memset(md5_checksum, 0, 16 + 1); 
    178.     } 
    179.     close(fd); 
    180.     show_hashtest_result(); 
    181.     hashtest_clean(); 
    182.     return 0; 

     

     

    原文地址:http://blog.youkuaiyun.com/liuben/article/details/5050697

评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值