mysql导数:
方法一:
脚本参考:
#!/bin/bash
if [ $# -eq 1 ] && [[ $1 =~ (.*)\.sql$ ]];then
[ ! -f ${1} ] && echo "sql文件不存在" && exit
nohup toexcel query -s 10.10.10.11 -uadmin -p'*****' -f ${1} -o $(date -d "today" +"%Y%m%d_%H").xlsx &
tail -f nohup.out
elif [ $# -eq 0 ];then
read -p $'请输入你要执行的提数sql: \n' sql
[ ! -f ${sql} ] && echo "sql文件不存在" && exit
nohup toexcel query -s 10.10.10.11 -uadmin -p'*****' -f ${sql} -o $(date -d "today" +"%Y%m%d_%H").xlsx > /dev/null 2>&1 &
tail -f nohup.out
else
echo "参数错误"
echo "USE $0 xx.sql "
fi
脚本运行:
sh tishu.sh tishu.sql
方法二:
mysql -uadmin -h 10.10.10.12 -p'*****' < tishu.sql | sed 's/\t/","/g' | sed 's/^/"/' | sed 's/$/"/' > /data/tishu-$(date +%Y%m%d%H%M%S).csv
注:需要将sql语句写到tishu.sql中
oracle导数:
执行脚本:sh tishu.sh tishu.sql
tishu.sh脚本:
#!/bin/bash
source /home/.profile
#set -x
sql_file=$1
file_name=$(basename "$sql_file" .sql)
file_full_name=$file_name".xlsx"
echo "$file_full_name"
#oracle -h '10.10.10.10' -P 1532 -u SELECT -p '*****' -S base -i ${sql_file} -o ${file_full_name}
#python /home/oracle_new.py -h '10.253.110.110' -P 1532 -u SELECT -p '*****' -S base -i ${sql_file} -o ${file_full_name} --toexcel 1
#oracle3 -h '10.10.10.10' -P 1532 -u SELECT -p '*****' -S base -i ${sql_file} -o ${file_full_name} --toexcel$ 1
/home/python_3.9.10/bin/python3 /home/oracle3.py -h '10.10.10.10' -P 1532 -u SELECT -p '*****' -S base -T 0 -i ${sql_file} -o ${file_full_name} -c "\t"
out_name=$file_name".xlsx"
echo "$out_name"
new_file_name=$file_name".xls"
file_txt=$file_name".txt"
threshold=200
while true; do
out_file_size=$(stat -c %s "$out_name")
if [ "$out_file_size" -gt "$threshold" ]; then
sed -i "s/|/\\t/g" ${out_name}
sed -i "s/richinfo/|/g" ${out_name}
#sed -i "s/'//g" ${out_name}
iconv -c -f utf8 -t gb18030 "$out_name" > "$new_file_name"
#cat ${out_name} > $file_txt
#sed -i 's/"//g' $file_txt
#mv $out_name $new_file_name
rm ${out_name}
#for i in {2..10};
#do
# add_file_name=${file_name}${i}".xlsx"
# out_file_name=${file_name}${i}".xls"
# if [ -f ${add_file_name} ]; then
# iconv -c -f utf8 -t gb18030 "$add_file_name" > "$out_file_name"
# rm ${add_file_name}
# else
# break;
#fi
#done
break;
else
sleep 4
fi
done;
oracle3.py 脚本:
from datetime import datetime
import re
from warnings import filterwarnings
filterwarnings('ignore', category=UserWarning)
import sys, os
from oracledb import connect, LOB, BLOB
import oracledb
os.environ['NLS_LANG'] = 'AMERICAN_AMERICA.AL32UTF8'
if datetime.now() > datetime.strptime('2025-03-01 01:00:00', '%Y-%m-%d %H:%M:%S'):
sys.exit()
else:
class oracle_select:
def __init__(self, confs):
self.LineSeparator = '--------------------'
self.sheet_cnt = 0
self.every_batch_num = 900000
self.write_mode = 'w'
self.confs = confs
注意下面的路径
oracledb.init_oracle_client(lib_dir="/home/oracle/12.2/client64/lib")
self.oracle_conn = connect(host=(self.confs['target_host']),
port=(self.confs['target_port']),
user=(self.confs['target_user']),
password=(self.confs['target_password']),
service_name=(self.confs['target_service_name']))
self.cur = self.oracle_conn.cursor()
self.cur.execute("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI:SS'")
def sql_get(self):
self.sql = input('oracle:')
if not self.sql.strip().endswith(';'):
if not self.sql.strip().endswith('\\G'):
while 1:
self.sql += ' ' + input('>>:')
if self.sql.strip().endswith(';') or self.sql.strip().endswith('\\G'):
break
def oracle_sql_change(self):
if re.findall('show +?create +?table ', self.sql):
self.table_name = self.sql.replace(re.findall('show +?create +?table ', self.sql)[0], '')
self.sql = f"SELECT dbms_metadata.get_ddl('TABLE', '{self.table_name}','{self.confs['target_user']}') as show_create_table FROM dual"
if re.findall('show +?create +?indexes ', self.sql):
self.table_name = self.sql.replace(re.findall('show +?create +?indexes ', self.sql)[0], '')
self.sql = f"SELECT index_name, DBMS_METADATA.GET_DDL('INDEX', index_name) AS index_creation_sql FROM all_indexes WHERE table_owner = '{self.confs['target_user']}' AND table_name = '{self.table_name}'"
def jh(self):
"""
交互函数,包括sql多行获取,数据打印格式
:return:
"""
while True:
try:
self.sql_get()
except EOFError:
print('\n\n程序被中断,退出。')
sys.exit()
except KeyboardInterrupt:
print('\n\n')
sql = ''
continue
self.sql = self.sql.strip().strip(';')
if self.sql.strip().endswith('\\G'):
self.sql = self.sql.strip('\\G')
self.confs['data_format'] = '1'
if self.sql.strip().startswith('show'):
self.oracle_sql_change()
if self.sql == 'exit' or self.sql == 'quit':
print('Bye')
sys.exit()
try:
self.cur.execute(self.sql)
except Exception as e:
print(str(e))
continue
self.rows = self.cur.fetchall()
self.columns = [desc[0] for desc in self.cur.description]
if self.confs['data_format'] == '1':
for i in self.rows:
print(self.LineSeparator)
for col, col_name in zip(i, self.columns):
print((col_name + ':').rjust(30) + str(col))
print(self.LineSeparator + '\n\n')
else:
print(' | '.join(self.columns))
print(self.LineSeparator)
for i in self.rows:
i = [str(s) for s in i]
print(' | '.join(i))
print('--------------------')
self.confs['data_format'] = self.confs['data_format_default']
def to_excel(self, head):
columns = [desc[0] for desc in self.cur.description]
while True:
rows = self.cur.fetchmany(self.every_batch_num)
if not rows:
break
self.sheet_cnt += 1
file_name = '.'.join(self.confs['out_data_path'].split('.')[0:-1]) + str(self.sheet_cnt) + '.xlsx'
with open(file_name, 'w', encoding=(self.confs['decode_type'])) as (f):
if head:
f.write('\t'.join(columns) + '\n')
for i in rows:
f.write('\t'.join(self.oracle_data_read_forexcel(i)) + '\n')
def to_sql(self, cur):
pass
def to_txt(self, cur, head):
columns = [desc[0] for desc in cur.description]
with open((self.confs['out_data_path']), (self.write_mode), encoding=(self.confs['decode_type'])) as (f):
if head:
f.write((f"{self.confs['cat_code']}").join(columns) + '\n')
while True:
rows = list(cur.fetchmany(self.every_batch_num))
if not rows:
break
for i in rows:
i = self.confs['cat_code'].join(self.oracle_data_read(i))
f.write(i + '\n')
def oracle_data_read(self, info):
"""
数据读取解析
:param info:
:param cat_code:
:return:
"""
r_li = []
for i in info:
if i is None or i == '':
i = ''
if type(i) is LOB or type(i) is BLOB:
r_li.append(str(i.read()))
continue
r_li.append(str(i))
return r_li
def oracle_data_read_forexcel(self, info):
"""
数据读取解析
:param info:
:param cat_code:
:return:
"""
r_li = []
for i in info:
if i is None or i == '':
i = ''
elif type(i) is BLOB or type(i) is LOB:
i = i.read()
else:
i = format(i, 'g') if isinstance(i, (int, float)) else str(i)
r_li.append('"' + str(i) + '"')
return r_li
def data_out(self):
"""
以脚本输出数据,txt或excel
:return:
"""
if self.confs['sql'] == "None":
with open((self.confs['sqls_file_path']), 'r', encoding='utf-8') as (f):
self.sql_list = f.read().split(';')
else:
self.sql_list = self.confs['sql'].split(';')
for sql in self.sql_list:
if sql.strip() == '':
pass
else:
self.cur.execute(sql.strip())
if self.confs['is_columns'] == "'1'":
head = True
else:
head = False
try:
if self.confs['to_excel'] == "'1'":
self.to_excel(head)
continue
else:
if self.confs['to_sql'] == "'1'":
self.to_sql(self.cur)
continue
else:
self.to_txt(self.cur, head)
except Exception as e:
print(e)
sys.exit()
self.write_mode = 'a'
def conf_get():
conf = {'target_host': 'None',
'target_port': 'None',
'target_user': 'None',
'target_password': 'None',
'target_service_name': 'None',
'sqls_file_path': 'None',
'out_data_path': 'None',
'sql': 'None',
'Interactive': "'0'",
'data_format': "'0'",
'data_format_default': "'0'",
'cat_code': "'\\t'",
'decode_type': "'utf-8'",
'is_columns': "'1'",
'to_excel': "'0'",
'to_sql': "'0'",
'columns': "''"}
for i in range(len(sys.argv)):
if sys.argv[i].strip().startswith('-h'):
conf['target_host'] = sys.argv[i + 1].strip()
else:
if sys.argv[i].strip().startswith('-P'):
conf['target_port'] = sys.argv[i + 1].strip()
else:
if sys.argv[i].strip().startswith('-u'):
conf['target_user'] = sys.argv[i + 1].strip()
else:
if sys.argv[i].strip().startswith('-p'):
conf['target_password'] = sys.argv[i + 1].strip()
else:
if sys.argv[i].strip().startswith('-S'):
conf['target_service_name'] = sys.argv[i + 1].strip()
else:
if sys.argv[i].strip().startswith('-i'):
conf['sqls_file_path'] = sys.argv[i + 1].strip()
else:
if sys.argv[i].strip().startswith('-o'):
conf['out_data_path'] = sys.argv[i + 1].strip()
else:
if sys.argv[i].strip().startswith('-e'):
conf['sql'] = sys.argv[i + 1].strip()
if sys.argv[i].strip().startswith('-T'):
conf['Interactive'] = sys.argv[i + 1].strip()
if sys.argv[i].strip().startswith('-G'):
conf['data_format'] = sys.argv[i + 1].strip()
conf['data_format_default'] = sys.argv[i + 1].strip()
if sys.argv[i].strip().startswith('-c'):
conf['cat_code'] = sys.argv[i + 1].strip().encode().decode('unicode_escape')
if sys.argv[i].strip().startswith('-d'):
conf['decode_type'] = sys.argv[i + 1].strip()
if sys.argv[i].strip().startswith('-t'):
conf['is_columns'] = sys.argv[i + 1].strip()
if sys.argv[i].strip().startswith('--toexcel'):
conf['to_excel'] = sys.argv[i + 1].strip()
if sys.argv[i].strip().startswith('--tosql'):
conf['to_sql'] = sys.argv[i + 1].strip()
if sys.argv[i].strip().startswith('--help'):
print("-h:host\n\n-P:port\n\n-u:username\n\n-p:password\n\n-S:servicename\n\n-i:sqlfile's path\n\n-o:outputfile's path\n\n-e:Enter SQL directly instead of using an SQL file.\n\n-T:Indicate whether to enter the interactive interface. Use 1 and 0 to represent yes or no, respectively. The default value is 0 (no).\n\n-G:Similar to MySQL's \\G mode, format the output data as key-value pairs. Use 0 and 1 to represent whether to format the output, with 0 indicating no formatting (default).\n\n-c:Set the delimiter. The default value is '\\t' (tab).\n\n-d:Set the encoding type. The default is UTF-8.\n\n-t:Indicate whether to display field names. Use 0 or 1 to represent no and yes, respectively. The default value is 1.\n\n--toexcel: Translate the query result into one sheet per 990,000 rows.--tosql: .show create table/indexes :The query must be made using the corresponding account to access the tables under its ownership.--help:find how to use it\n\n")
sys.exit()
return conf
if __name__ == '__main__':
try:
confs = conf_get()
oracle = oracle_select(confs)
except Exception as e:
print('参数错误')
print(e)
sys.exit()
if confs['Interactive'] == '1':
try:
oracle.jh()
except Exception as e:
print(e)
try:
oracle.data_out()
except Exception as e:
print(e)
登录oracle:
sh oracle.sh
oracle.sh 脚本:
source ~/.profile
/home/python3 /home/oracle3.py -h '10.10.10.10' -P 1522 -u SELECT -p '*****' -S base -T 1
oracle3.py 脚本:
from datetime import datetime
import re
from warnings import filterwarnings
filterwarnings('ignore', category=UserWarning)
import sys, os
from oracledb import connect, LOB, BLOB
import oracledb
os.environ['NLS_LANG'] = 'AMERICAN_AMERICA.AL32UTF8'
if datetime.now() > datetime.strptime('2025-03-01 01:00:00', '%Y-%m-%d %H:%M:%S'):
sys.exit()
else:
class oracle_select:
def __init__(self, confs):
self.LineSeparator = '--------------------'
self.sheet_cnt = 0
self.every_batch_num = 900000
self.write_mode = 'w'
self.confs = confs
oracledb.init_oracle_client(lib_dir="/home/wasadmin/app/oracle/12.2/client64/lib")
self.oracle_conn = connect(host=(self.confs['target_host']),
port=(self.confs['target_port']),
user=(self.confs['target_user']),
password=(self.confs['target_password']),
service_name=(self.confs['target_service_name']))
self.cur = self.oracle_conn.cursor()
self.cur.execute("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI:SS'")
def sql_get(self):
self.sql = input('oracle:')
if not self.sql.strip().endswith(';'):
if not self.sql.strip().endswith('\\G'):
while 1:
self.sql += ' ' + input('>>:')
if self.sql.strip().endswith(';') or self.sql.strip().endswith('\\G'):
break
def oracle_sql_change(self):
if re.findall('show +?create +?table ', self.sql):
self.table_name = self.sql.replace(re.findall('show +?create +?table ', self.sql)[0], '')
self.sql = f"SELECT dbms_metadata.get_ddl('TABLE', '{self.table_name}','{self.confs['target_user']}') as show_create_table FROM dual"
if re.findall('show +?create +?indexes ', self.sql):
self.table_name = self.sql.replace(re.findall('show +?create +?indexes ', self.sql)[0], '')
self.sql = f"SELECT index_name, DBMS_METADATA.GET_DDL('INDEX', index_name) AS index_creation_sql FROM all_indexes WHERE table_owner = '{self.confs['target_user']}' AND table_name = '{self.table_name}'"
def jh(self):
"""
交互函数,包括sql多行获取,数据打印格式
:return:
"""
while True:
try:
self.sql_get()
except EOFError:
print('\n\n程序被中断,退出。')
sys.exit()
except KeyboardInterrupt:
print('\n\n')
sql = ''
continue
self.sql = self.sql.strip().strip(';')
if self.sql.strip().endswith('\\G'):
self.sql = self.sql.strip('\\G')
self.confs['data_format'] = '1'
if self.sql.strip().startswith('show'):
self.oracle_sql_change()
if self.sql == 'exit' or self.sql == 'quit':
print('Bye')
sys.exit()
try:
self.cur.execute(self.sql)
except Exception as e:
print(str(e))
continue
self.rows = self.cur.fetchall()
self.columns = [desc[0] for desc in self.cur.description]
if self.confs['data_format'] == '1':
for i in self.rows:
print(self.LineSeparator)
for col, col_name in zip(i, self.columns):
print((col_name + ':').rjust(30) + str(col))
print(self.LineSeparator + '\n\n')
else:
print(' | '.join(self.columns))
print(self.LineSeparator)
for i in self.rows:
i = [str(s) for s in i]
print(' | '.join(i))
print('--------------------')
self.confs['data_format'] = self.confs['data_format_default']
def to_excel(self, head):
columns = [desc[0] for desc in self.cur.description]
while True:
rows = self.cur.fetchmany(self.every_batch_num)
if not rows:
break
self.sheet_cnt += 1
file_name = '.'.join(self.confs['out_data_path'].split('.')[0:-1]) + str(self.sheet_cnt) + '.xlsx'
with open(file_name, 'w', encoding=(self.confs['decode_type'])) as (f):
if head:
f.write('\t'.join(columns) + '\n')
for i in rows:
f.write('\t'.join(self.oracle_data_read_forexcel(i)) + '\n')
def to_sql(self, cur):
pass
def to_txt(self, cur, head):
columns = [desc[0] for desc in cur.description]
with open((self.confs['out_data_path']), (self.write_mode), encoding=(self.confs['decode_type'])) as (f):
if head:
f.write((f"{self.confs['cat_code']}").join(columns) + '\n')
while True:
rows = list(cur.fetchmany(self.every_batch_num))
if not rows:
break
for i in rows:
i = self.confs['cat_code'].join(self.oracle_data_read(i))
f.write(i + '\n')
def oracle_data_read(self, info):
"""
数据读取解析
:param info:
:param cat_code:
:return:
"""
r_li = []
for i in info:
if i is None or i == '':
i = ''
if type(i) is LOB or type(i) is BLOB:
r_li.append(str(i.read()))
continue
r_li.append(str(i))
return r_li
def oracle_data_read_forexcel(self, info):
"""
数据读取解析
:param info:
:param cat_code:
:return:
"""
r_li = []
for i in info:
if i is None or i == '':
i = ''
elif type(i) is BLOB or type(i) is LOB:
i = i.read()
else:
i = format(i, 'g') if isinstance(i, (int, float)) else str(i)
r_li.append('"' + str(i) + '"')
return r_li
def data_out(self):
"""
以脚本输出数据,txt或excel
:return:
"""
if self.confs['sql'] == "None":
with open((self.confs['sqls_file_path']), 'r', encoding='utf-8') as (f):
self.sql_list = f.read().split(';')
else:
self.sql_list = self.confs['sql'].split(';')
for sql in self.sql_list:
if sql.strip() == '':
pass
else:
self.cur.execute(sql.strip())
if self.confs['is_columns'] == "'1'":
head = True
else:
head = False
try:
if self.confs['to_excel'] == "'1'":
self.to_excel(head)
continue
else:
if self.confs['to_sql'] == "'1'":
self.to_sql(self.cur)
continue
else:
self.to_txt(self.cur, head)
except Exception as e:
print(e)
sys.exit()
self.write_mode = 'a'
def conf_get():
conf = {'target_host': 'None',
'target_port': 'None',
'target_user': 'None',
'target_password': 'None',
'target_service_name': 'None',
'sqls_file_path': 'None',
'out_data_path': 'None',
'sql': 'None',
'Interactive': "'0'",
'data_format': "'0'",
'data_format_default': "'0'",
'cat_code': "'\\t'",
'decode_type': "'utf-8'",
'is_columns': "'1'",
'to_excel': "'0'",
'to_sql': "'0'",
'columns': "''"}
for i in range(len(sys.argv)):
if sys.argv[i].strip().startswith('-h'):
conf['target_host'] = sys.argv[i + 1].strip()
else:
if sys.argv[i].strip().startswith('-P'):
conf['target_port'] = sys.argv[i + 1].strip()
else:
if sys.argv[i].strip().startswith('-u'):
conf['target_user'] = sys.argv[i + 1].strip()
else:
if sys.argv[i].strip().startswith('-p'):
conf['target_password'] = sys.argv[i + 1].strip()
else:
if sys.argv[i].strip().startswith('-S'):
conf['target_service_name'] = sys.argv[i + 1].strip()
else:
if sys.argv[i].strip().startswith('-i'):
conf['sqls_file_path'] = sys.argv[i + 1].strip()
else:
if sys.argv[i].strip().startswith('-o'):
conf['out_data_path'] = sys.argv[i + 1].strip()
else:
if sys.argv[i].strip().startswith('-e'):
conf['sql'] = sys.argv[i + 1].strip()
if sys.argv[i].strip().startswith('-T'):
conf['Interactive'] = sys.argv[i + 1].strip()
if sys.argv[i].strip().startswith('-G'):
conf['data_format'] = sys.argv[i + 1].strip()
conf['data_format_default'] = sys.argv[i + 1].strip()
if sys.argv[i].strip().startswith('-c'):
conf['cat_code'] = sys.argv[i + 1].strip().encode().decode('unicode_escape')
if sys.argv[i].strip().startswith('-d'):
conf['decode_type'] = sys.argv[i + 1].strip()
if sys.argv[i].strip().startswith('-t'):
conf['is_columns'] = sys.argv[i + 1].strip()
if sys.argv[i].strip().startswith('--toexcel'):
conf['to_excel'] = sys.argv[i + 1].strip()
if sys.argv[i].strip().startswith('--tosql'):
conf['to_sql'] = sys.argv[i + 1].strip()
if sys.argv[i].strip().startswith('--help'):
print("-h:host\n\n-P:port\n\n-u:username\n\n-p:password\n\n-S:servicename\n\n-i:sqlfile's path\n\n-o:outputfile's path\n\n-e:Enter SQL directly instead of using an SQL file.\n\n-T:Indicate whether to enter the interactive interface. Use 1 and 0 to represent yes or no, respectively. The default value is 0 (no).\n\n-G:Similar to MySQL's \\G mode, format the output data as key-value pairs. Use 0 and 1 to represent whether to format the output, with 0 indicating no formatting (default).\n\n-c:Set the delimiter. The default value is '\\t' (tab).\n\n-d:Set the encoding type. The default is UTF-8.\n\n-t:Indicate whether to display field names. Use 0 or 1 to represent no and yes, respectively. The default value is 1.\n\n--toexcel: Translate the query result into one sheet per 990,000 rows.--tosql: .show create table/indexes :The query must be made using the corresponding account to access the tables under its ownership.--help:find how to use it\n\n")
sys.exit()
return conf
if __name__ == '__main__':
try:
confs = conf_get()
oracle = oracle_select(confs)
except Exception as e:
print('参数错误')
print(e)
sys.exit()
if confs['Interactive'] == '1':
try:
oracle.jh()
except Exception as e:
print(e)
try:
oracle.data_out()
except Exception as e:
print(e)