python 读文件并处理成DataFrame

1. 从hive表中批量读取数据，处理成DataFrame。

#!/usr/bin/python
# -*- coding: UTF-8 -*-
from pyhive import hive
import pandas as pd

def get_batch_data_from_hive(sql_str, batch_size=100):
    # sql_str = 'select * from zx_dm.lx_user_wt_gjc_contact_4_predict_data_d where pt="{}"'.format(last_dt)
    conn = hive.Connection(host='localhost', port=10000, username='zx_dm', auth='NOSASL', database='zx_dm')
    cursor = conn.cursor()
    cursor.execute(sql_str)
    while True:
        batch_data = cursor.fetchmany(size=batch_size)
        if len(batch_data) == 0:
             break
        df = pd.DataFrame(batch_data, columns=cols)
        # 处理一个batch
        fun(df)

    cursor.close()
    conn.close()

2.  使用pandas从hive表中读取DataFrame数据。

def get_data_from_hive(sql_str):
    conn = hive.Connection(host='sjpt-other-standby-9.wxxdc',\
port=10000,auth='KERBEROS', kerberos_se