wanghuafeng

#coding:utf-8#利用dict特性去掉一个list中重复的数据def del_repeate_data(): mylist = [1,1,2,3,4,5,4,2,5,5,5] d = {} for x in mylist: d[x] = 1 mylist = list(d.keys()) print mylist#

#coding:utf-8import reimport osimport timeimport codecsPATH = os.path.dirname(__file__)s = u'what a Beautiful woRld'.lower()pattern = re.compile(u'[^a-z]+', re.U)#在非英文出进行切割for con in pattern

def get_all_checkout_file(): '''利用glob获取相关路径''' import glob glob_path = os.path.join(PATH, '0709modify', 'cuted_varify_sample_pinyin_role_checkout*') matched_path_list = glob.glob(glo

有关定义：__declspec(dllexport)

__author__ = 'wanghuafeng'#coding:utf-8import osimport sysimport codecsfrom collections import dequetry: PATH = os.path.dirname(os.path.abspath(__file__))except: PATH = os.getcwd()cl

__author__ = 'wanghuafeng'from datetime import timedelta, datetimeyesterday = datetime.today() + timedelta(-1)yesterday_format = yesterday.strftime('%Y_%m_%d')print yesterday_format #2015_01_05

def check(script): filename = r'F:\klm\wordservice\data\Cizu_and_singleword_komoxo95K.txt' script_compile = compile(script, '', 'eval')#预编译，耗时操作，列表解析中直接执行eval()，每次都会编译 with codecs.open(fil

def unique_list(xs): seen = set() # not seen.add(x) here acts to make the code shorter without using if statements, seen.add(x) always returns None. return [x for x in xs if x not in seen

测试数据，大小：1G  总行数：5193911行实现方式：codecs.open(); open(); enumerate()常识：codecs open()时默认mode为'rb'; 直接使用open()打开默认方式为'r'先给出结论：1、同一种打开方式下mode=‘rb'性能高于mode='r'2、不同打开方式(codecs.open(), open()) 相同的mode时，二

__author__ = 'wanghuafeng'# -*- coding: utf-8 -*-import timeimport itertoolsfor i in itertools.product([1, 2, 3], [4, 5], [6, 7]): print idef product(*args, **kwds): # product('ABCD',

__author__ = 'huafeng'#coding:utf-8import osimport subprocessimport codecsimport simplejsonremote_path = r'/home/mdev/tmp'def tar_(): tar_command = '''fab -H mdev -- "cd %s;

给定一个字符串s，检查s是否含有多个结束标记中的一个。import itertoolsdef anyTrue(predicate, sequence): return True in itertools.imap(predicate, sequence)def endsWith(s, *endings): return anyTrue(s.endswith, endings

d = {}for i in range(100000000): d['%s'%i] = Noneprint len(d)#100000000start_time = time.time()print 'a' in d#Falseprint "in time consume: ", time.time() - start_time#in time consume: 0.0s

字典（dict）：原始语料：546MCPU usage：36.7% ==> 76.9%占用内存3.2G     （占用空间为原始语料的6倍）查找时间复杂度：O(1)集合（set）：原始语料：546MCPU usage：34.6%==>70.6%占用内存：2.88G     （占用空间为原始语料的5.4倍）查找时间复杂度：O(1)数组（List

原始语料：546M1.readlines() with codecs.open(combine_bigram_remove_freq_1_filename, encoding='utf-8') as f: temp_list = [item for item in f.readlines()]CPU usage：34.6% ==> 61.7%占用内存：2.168G耗

import codecsimport ospath = os.path.dirname(__file__)odd_line_list = []even_line_list = []filename = os.path.join(path, 'forum_high_freq_sentence_sougou_checkout.txt')with codecs.open(filename

#coding:utf-8import osimport timeimport codecsdef get_lenght_single_word(): '''筛选出文件中不同长度的汉字''' filename = r'E:\SVN\linguistic_model\9_keys\0709modify\wordlist_61633_weight.txt' with

a = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]转化成:[[1,4,7],[2,5,8],[3,6,9]]原始版本：此处需将dic中的key值手动完成，数据量较大时，该方法实用性不强import itertoolscondict = {"numeric":["one","two","three"],"ordinal":["first","second

1、最原始方法“%s”import xlrdimport cx_Oracleimport timefile = xlrd.open_workbook('4,032 Cells.xls')sheet = file.sheets()[0]nrows = sheet.nrowsncols = sheet.ncols#~ print ncolsprint nrowsfor rown

1、range()(1)range(stop) -> 返回整型元素数组(2)range(start, stop[, step]) -> 返回整形元素类型的数组range(m,n) --------->[m,m+1,m+2 ... n-2,n-1]共n-m个元素，为前闭后开的区间[m,n)start元素在默认情况为0step则确定了增长（减小）的幅度例：range(1,10,

import reclass SQLParser: def __init__(self, sql): self.sql = sql self.paras = None self.num_para = sql.count('@') # @ only be used as the parameter identify

import rearg = "jing4 gong1 sang1 zi3"arg_count = arg.count(" ")patten ="([a-z]*)\d\s?"*(arg_count+1)print pattenarg_compile = re.compile(patten)match_obj = arg_compile.search(arg)print ma

import osos.chdir("../..")#表示上两级目录os.chadir("/")则表示切换到当前文件所在的根目录下os.chdir(os.pardir +"/" +os.pardir) #事实上os.path就是".."，只是它实现了跨平台操作print os.getcwd()

文件中有word pinyin freq参数，一个word可能对应多个pinyin和多个freq，例如：的 de 123的 di  100读取一个文件进行高freq的筛选def gen_single_word_dic(): import codecs,os THIS_PATH = os.path.dirname(os.path.abspath(__file__))

#encoding: utf8import os#当前文件的完整路径名：C:\Python27\Scripts\djangotest\justtest.pyprint os.path.dirname(os.path.dirname(__file__))#获取当前文件所在目录的上级目录：C:/Python27/Scriptsprint os.path.dirname(__file__)#

#encoding:utf-8import refrom tone2Pinyin import ConvertToneNumbersPinyinargs = "Zhào Qián Sūn Lǐ Zhōu Wú Zhèng Wáng"# lineOut = ConvertToneNumbersPinyin(args)# print lineOuts = u"""Zhào Qián

str.replace(old, new) 是有返回值的例：str = 'abc*de'执行：str.replace('*','') #此时返回abcde 但是如果此时执行：print str ==>'abc*de' #str 的值并没有改变所以在使用replace函数的时候可以设置变量的接收返回的截取后的字符串，以便使用可以是：replaced_st

from itertools import productpinyin_list = [[u'shei shi', u'shui shi'], [u'kai fa'], [u'jiao cheng']]keys = []for i in range(len(pinyin_list)): tmp_keys = [item for item in pinyin_list[i]]

数据格式：（词，拼音，词频）的 de 148709248的 di 1193135了 le 62873377了 liao 3199200是 shi 62432861一 yi 58994539不 bu 57479625不 fou 1136895将文件中多音字的高频词汇提取并返回def chose_high_freq_word(): '''ke

arr = ['11', '4', '3', '2', '1']arr = [int(val_int) for val_int in arr]print arrprint [float(item)/sum(arr) for item in arr]

因为多是以来一直使用f.readlines()来处理文件，今天看了一篇帖子说，f.read()的效率要高于f.readlines()而且f.read()还更智能。当然，“智能”这个词到底体现在哪我还不知道，但是效率觉得值得商榷，然后自己写个小程序测试下。有点坑了，所以嘛，有歧义的话，还是自己动手去验证吧。。。。测试文件大小为5M，近19万行。import timeimport codec

__author__ = 'huafeng'#coding:utf-8import requestsurl = 'http://www.zhihu.com/topics#%E6%91%84%E5%BD%B1'# s = requests.session()r = requests.get(url)#如果不请求过程中不涉及session信息的修改，则无需使用request.session