提取Python stringlib中的"BMHBNFS"字符串查找算法

Python中的stringlib字符串查找算法是Boyer-Moore, Horspool, Sunday, Bloom Filter几种算法的合成体, 大概的原理如下:

def find(s, p):
    # find first occurrence of p in s
    n = len(s)
    m = len(p)
    skip = delta1(p)[p[m-1]]
    i = 0
    while i <= n-m:
        if s[i+m-1] == p[m-1]: # (boyer-moore)
            # potential match
            if s[i:i+m-1] == p[:m-1]:
                return i
            if s[i+m] not in p:
                i = i + m + 1 # (sunday)
            else:
                i = i + skip # (horspool)
        else:
            # skip
            if s[i+m] not in p:
                i = i + m + 1 # (sunday)
            else:
                i = i + 1
    return -1 # not found

以下是具体实现:

/* stringlib: fastsearch implementation */

#ifndef STRINGLIB_FASTSEARCH_H
#define STRINGLIB_FASTSEARCH_H

#include <stdint.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

/* fast search/count implementation, based on a mix between boyer-
   moore and horspool, with a few more bells and whistles on the top.
   for some more background, see: http://effbot.org/zone/stringlib.htm */

/* note: fastsearch may access s[n], which isn't a problem when using
   Python's ordinary string types, but may cause problems if you're
   using this code in other contexts.  also, the count mode returns -1
   if there cannot possible be a match in the target string, and 0 if
   it has actually checked for matches, but didn't find any.  callers
   beware! */

#define FAST_COUNT 0
#define FAST_SEARCH 1
#define FAST_RSEARCH 2

#ifndef LONG_BIT
#define LONG_BIT 32
#endif

#if LONG_BIT >= 128
#define STRINGLIB_BLOOM_WIDTH 128
#elif LONG_BIT >= 64
#define STRINGLIB_BLOOM_WIDTH 64
#elif LONG_BIT >= 32
#define STRINGLIB_BLOOM_WIDTH 32
#else
#error "LONG_BIT is smaller than 32"
#endif

#define STRINGLIB_BLOOM_ADD(mask, ch) \
	((mask |= (1UL << ((ch) & (STRINGLIB_BLOOM_WIDTH -1)))))
#define STRINGLIB_BLOOM(mask, ch)     \
	((mask &  (1UL << ((ch) & (STRINGLIB_BLOOM_WIDTH -1)))))

ssize_t fastsearch(const char *s, ssize_t n,
		   const char *p, ssize_t m,
		   ssize_t maxcount, int mode)
{
	unsigned long mask;
	ssize_t skip, count = 0;
	ssize_t i, j, mlast, w;

	w = n - m;

	if (w < 0 || (mode == FAST_COUNT && maxcount == 0)) {
		return -1;
	}

	/* look for special cases */
	if (m <= 1) {
		if (m <= 0) {
			return -1;
		}
		/* use special case for 1-character strings */
		if (mode == FAST_COUNT) {
			for (i = 0; i < n; i++)
				if (s[i] == p[0]) {
					count++;
					if (count == maxcount) {
						return maxcount;
					}
				}
			return count;
		}
		else if (mode == FAST_SEARCH) {
			for (i = 0; i < n; i++)
				if (s[i] == p[0]) {
					return i;
				}
		}
		else {      /* FAST_RSEARCH */
			for (i = n - 1; i > -1; i--)
				if (s[i] == p[0]) {
					return i;
				}
		}
		return -1;
	}

	mlast = m - 1;
	skip = mlast - 1;
	mask = 0;

	if (mode != FAST_RSEARCH) {

		/* create compressed boyer-moore delta 1 table */

		/* process pattern[:-1] */
		for (i = 0; i < mlast; i++) {
			STRINGLIB_BLOOM_ADD(mask, p[i]);
			if (p[i] == p[mlast]) {
				skip = mlast - i - 1;
			}
		}
		/* process pattern[-1] outside the loop */
		STRINGLIB_BLOOM_ADD(mask, p[mlast]);

		for (i = 0; i <= w; i++) {
			/* note: using mlast in the skip path slows things down on x86 */
			if (s[i + m - 1] == p[m - 1]) {
				/* candidate match */
				for (j = 0; j < mlast; j++)
					if (s[i + j] != p[j]) {
						break;
					}
				if (j == mlast) {
					/* got a match! */
					if (mode != FAST_COUNT) {
						return i;
					}
					count++;
					if (count == maxcount) {
						return maxcount;
					}
					i = i + mlast;
					continue;
				}
				/* miss: check if next character is part of pattern */
				if (!STRINGLIB_BLOOM(mask, s[i + m])) {
					i = i + m;
				}
				else {
					i = i + skip;
				}
			}
			else {
				/* skip: check if next character is part of pattern */
				if (!STRINGLIB_BLOOM(mask, s[i + m])) {
					i = i + m;
				}
			}
		}
	}
	else {      /* FAST_RSEARCH */

		/* create compressed boyer-moore delta 1 table */

		/* process pattern[0] outside the loop */
		STRINGLIB_BLOOM_ADD(mask, p[0]);
		/* process pattern[:0:-1] */
		for (i = mlast; i > 0; i--) {
			STRINGLIB_BLOOM_ADD(mask, p[i]);
			if (p[i] == p[0]) {
				skip = i - 1;
			}
		}

		for (i = w; i >= 0; i--) {
			if (s[i] == p[0]) {
				/* candidate match */
				for (j = mlast; j > 0; j--)
					if (s[i + j] != p[j]) {
						break;
					}
				if (j == 0)
					/* got a match! */
				{
					return i;
				}
				/* miss: check if previous character is part of pattern */
				if (!STRINGLIB_BLOOM(mask, s[i - 1])) {
					i = i - m;
				}
				else {
					i = i - skip;
				}
			}
			else {
				/* skip: check if previous character is part of pattern */
				if (!STRINGLIB_BLOOM(mask, s[i - 1])) {
					i = i - m;
				}
			}
		}
	}

	if (mode != FAST_COUNT) {
		return -1;
	}
	return count;
}

#endif

测试代码

#include <arch/cycle.h>

int main(int argc, char **argv)
{
	char *str = "GET / HTTP 1.0\r\nHost: www.xxx.com\r\nCache: \r\nCache:\r\n Length:\r\n";
	ssize_t rc = 0;
	uint64_t start, end;

	start = get_cycle_count();
	rc = fastsearch(str, strlen(str), "Cache:", 6, 2, FAST_SEARCH);
	end = get_cycle_count();
	printf("fastsearch return %u cost %llu \n", rc, end - start);
	printf("result = %s\n", str + rc);
	rc = fastsearch(str, strlen(str), "Cache:", 6, -1, FAST_COUNT);
	printf("result = %u\n", rc);
	return 0;
}

看stringlib测试数据, 还是蛮可以的.

我在tile平台上测试发现还没有snort中的BMH算法速度快.

不过这个只是单一测试, 没有考虑到cache的情况, 仅供参考.

原文参考:

The stringlib Library 有详细的描叙.