zz

feature_sample

import commands
import os
import re
import math
import time

pattern = re.compile(r'\S*\w*\.jpg$')#https://zhidao.baidu.com/question/386785665.html
pattern2 = re.compile(r'\S*\w*\.Jpg$')
pattern3 = re.compile(r'\S*\w*\.jpeg$')
pattern4 = re.compile(r'\S*\w*\.png$')


def get_img_list_str(dir_path, batch_num, index_file):
    img_list_str_list = []
    root = dir_path
    list_dir = os.walk(root)
    index = 0
    img_list_str = ""
    fo = open(index_file, mode="ab")
    for root, dirs, files in list_dir:
        total_img_file = 0
        for i in range(0, len(files)):
            f = files[i]
            if pattern.match(f) or pattern2.match(f) or pattern3.match(f) or pattern4.match(f):
                total_img_file += 1
        mem_num = math.ceil(float(total_img_file) / float(batch_num))
        print "==========" + str(total_img_file)+ " "+str(mem_num)
        for i in range(0, total_img_file):
            f = files[i]
            if pattern.match(f) or pattern2.match(f) or pattern3.match(f) or pattern4.match(f):
                img_path = os.path.join(root, f)#https://blog.youkuaiyun.com/u012193416/article/details/77247432
                img_list_str = img_list_str + img_path + ","
                # print img_list_str
                # print i % mem_num
                # time.sleep(1)
                # print ("%s : %s" % (i,(i ) % mem_num ))
                if (i > 0 and (i + 1) % mem_num == 0):
                    img_list_str = img_list_str.rstrip(",")#http://www.runoob.com/python/att-string-rstrip.html
                    fo.write(str(index) + "\n" + img_list_str + "\n")#http://www.runoob.com/python/python-file-write.html
                    img_list_str_list.append(img_list_str)
                    # print index
                    # time.sleep(0.5)
                    index += 1
                    img_list_str = ""
    if ((float(len(files)) % float(batch_num)) !=0):
        img_list_str = img_list_str.rstrip(",")
        fo.write(str(index) + "\n" + img_list_str + "\n")
        img_list_str_list.append(img_list_str)

    fo.close()
    print len(img_list_str_list)
    return img_list_str_list


def process(process_dir, output, batch_n):
    output_root = output
    batch_num = str(batch_n)
    if os.path.exists(output_root):
        commands.getstatusoutput("rm -rf " + output_root + "*")
    else:
        commands.getstatusoutput("mkdir -p "+ output_root)
    index_file = os.path.join(output_root , "index_file.txt")
    img_list_str_list = get_img_list_str(process_dir, batch_num,
                                         index_file)

    for b_index in range(0, len(img_list_str_list)):
        cmd = "python feature_client.py " + "--batch_index=" + str(
            b_index) + " " + "--save_bin=true " + "--outputDir=" + output_root + " " + img_list_str_list[b_index]
        print cmd
        mStatus, mResults = commands.getstatusoutput(cmd)
        if mStatus == 0:
            # print mResults
            pass
        else:
            print mStatus
            print "run command error"
def main():
    print "#################### process into extract face feature ####################"

    _process_root = "1M_all_7_0/"
    process_root = os.path.join("/home/liying/2.DataSet/1M_all/", _process_root)
    output_root =  "/home/liying/2.DataSet/pip_m1_0/Feature_files/"
    dir_list = os.walk(process_root)
    count = 0
    for root, dirs, files in dir_list:
        if root == process_root:
            for d in dirs:
                count += 1
                # if count <=589:
                #     continue
                print os.path.join(root,d)
                imgOutputDir = os.path.join("clsResult", _process_root, d)
                check_ifNone = "ls -l " + imgOutputDir + " | grep \"-\" | wc -l"
                s, r = commands.getstatusoutput(check_ifNone)
                if r.isdigit() and (int(r) > 5):
                    continue
                process(os.path.join(root,d),output_root,1)
                print "complete"
                cmd = "sh feature_cluster.sh"
                print cmd
                try:
                    mStatus, mResults = commands.getstatusoutput(cmd)
                except:
                    traceback.print_exc()
                    print mStatus, mResults
                # if count == 2:
                #     break
        break
def main_():
    process_root = "/home/workspace/DATA/rawface"
    process(process_root , "/home/xutianxi/Yuntian/FaceFeature/Feature_files_yu/", 60)
if __name__ == '__main__':
    main()

feature_client

#!/usr/bin/env python

import socket
import struct
import google.protobuf
import face_feature_pb2
import traceback
import os, sys
import commands
import argparse

DETECT_SERVER = "10.144.216.138"
DETECT_PORT = 38401


def client(ip, port, message):
    response = ""
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect((ip, port))
    try:

        header = struct.pack("!I", len(message))
        print("header length: " + str(len(message)))
        sock.sendall(header)
        sock.sendall(message)

        header = sock.recv(4)
        length, = struct.unpack("!I", header)

        receive_size = 0
        response = str()
        while receive_size < length:
            part_data = sock.recv(length - receive_size)
            response += part_data
            receive_size = len(response)
            print("request: data size:" + str(receive_size))

    finally:
        sock.close()

    return response


def extrator_feature(imagedatas):
    try:
        sendData = face_feature_pb2.face_feature_in()
        for image in imagedatas:
            face_feature = sendData.faces.add()
            face_feature.id = image["id"]
            face_feature.data = image["data"]
            # sendData.faces.append(face_feature)

        # Serialize
        sendDataStr = sendData.SerializeToString()

        # send to server
        receiveDataStr = client(DETECT_SERVER, DETECT_PORT, sendDataStr)

        print("receiveDataStr size: " + str(len(receiveDataStr)))
        receiveData = face_feature_pb2.face_feature_out()
        receiveData.ParseFromString(receiveDataStr)

        face_features = []
        for face in receiveData.faces:
            feature = dict()
            feature["id"] = face.id
            feature["feature"] = face.feature.decode('utf-8')
            face_features.append(feature)

        return face_features

    except Exception, e:
        print Exception, ':', e
        print traceback.print_exc()
        errInfo = sys.exc_info()
        print errInfo[0], ':', errInfo[1]
        return []


def usage():
    print("./python_client pic1.png pic2.png pic3.png")


def save_image(in_file, out_files):
    (filepath, tempfilename) = os.path.split(in_file)
    (filename, extension) = os.path.splitext(tempfilename)
    index = 0
    for file in out_files:
        ofilepath = os.path.join(filepath, filename + "_" + str(index) + ".png")
        fwo = open(ofilepath, "wb")
        fwo.write(file)
        fwo.close()

        index += 1


def save_feature_toBin(features, b_index, output_dir):
    # for b in range(0,img_batch_num):
    b = b_index
    os.mkdir(os.path.join(output_dir,"Feature_files_" + str(b)))
    feature_file_root = os.path.join(output_dir,"Feature_files_" + str(b))
    gFeature_file = os.path.join(feature_file_root, "face_feature.txt")
    mFeature_file = open(gFeature_file, mode="ab")
    gFeature_index = open(os.path.join(feature_file_root, "face_index.txt"), mode="ab")
    for feature in features:
        # print("################################")
        # print("id:" + str(feature["id"]))
        gFeature_index.write(str(feature["id"]) + "\n")
        # print("feature: " + str(feature["feature"]))
        mFeature_file.write(str(feature["feature"]) + "\n")
    mFeature_file.close()
    gFeature_index.close()

    # convert to bin file\
    gBin_file = os.path.join(feature_file_root, "face_feature.bin")
    mStatus, mResults = commands.getstatusoutput("./small_test " + gFeature_file + " " + gBin_file)
    if mStatus == 0:
        print " feature file has benn save to " + gBin_file
        return mResults
    else:
        return mStatus


def main():
    if len(sys.argv) < 2:
        print("./python_client pic1.png pic2.png pic3.png ")
        return
#  usage: python python_client.py 0609_2_0.png --save_bin=true
    parser = argparse.ArgumentParser()
    parser.add_argument('images', type=str, help='path of images: '
                                                 ' a string split by \',\' eg:[test1.jpg, test2.jpg]; ')
    # parser.add_argument('--mode', type=int, default='0', help='single:0, batch:1')
    parser.add_argument('--batch_index', type=int, default='0',
                        help='if mode is 1: batch, batch_index is from 0 to batch_num -1')
    parser.add_argument('--save_bin', type=bool, default=False,
                        help='if true, save feature as bin file. if false, output as str')
    parser.add_argument('--outputDir', type=str,
                        default='.', help='save the feature file to ')
    args = parser.parse_args()


    batch_index = args.batch_index
    id = 0
    images = []
    img_list = args.images.split(",")
    for file in img_list:
        print file
        fo = open(file, "rb")
        stri = fo.read()
        fo.close()
        image = dict()
        image["id"] = id
        image["data"] = stri
        images.append(image)
        id += 1
    features = extrator_feature(images)

    if args.save_bin:
        output = args.outputDir
        print("feature expression as bin file")
        if not os.path.exists(output):
            os.mkdir(output)
        save_feature_toBin(features, batch_index,output)
    else:
        print("feature expression as str")
        for feature in features:
            print("################################")
            print("id:" + str(feature["id"]))
            print("feature: " + str(feature["feature"]))


    #  ==============old version, xu============================
    # batch_index = int(sys.argv[1])
    # print "batch index is: " + sys.argv[1]
    #
    # id = 0
    # images = []
    # for file in sys.argv[2:]:
    #     print file
    #     fo = open(file, "rb")
    #     stri = fo.read()
    #     fo.close()
    #     image = dict()
    #     image["id"] = id
    #     image["data"] = stri
    #     images.append(image)
    #     id += 1
    #
    # features = extrator_feature(images)

    #     print("feature expression as str")
    #     for feature in features:
    #         print("################################")
    #         print("id:" + str(feature["id"]))
    #         print("feature: " + str(feature["feature"]))

    # print("feature expression as bin file")
    # save_feature_toBin(features, batch_index)

if __name__ == "__main__":
    main()