项目在做mysql 到snowflake的迁移, 做一个通用连接模块适配不同数据库系统的建联
早期sqlalchemy目标是提供能兼容众多数据库(如 SQLite、MySQL、Postgres、Oracle、MS-SQL、SQLServer 和 Firebird)的企业级持久性模型
代码如下:
1. 因为mysql是搭建在云端,需要SSH桥接,加入ssh_tunnel 连接库
"""SSH Tunnel Connector"""
import logging
import os
import sshtunnel
from modules.utils.configloader import load_config
from modules.utils.constants import SSHServer
logger = logging.getLogger(__name__)
INSTANCE = load_config(f'config/profiles/{os.environ.get("ENV", "bdpqa")}')
class SSHTunnel:
"""SSH Tunnel Class
"""
def __init__(self, ssh_address=None, username=None, remote_bind_address=None):
"""Initialize
"""
rds_items = INSTANCE.data.get('customer')
self.tunnel = sshtunnel.SSHTunnelForwarder(
ssh_address_or_host=ssh_address or rds_items[SSHServer.SSH_ADDRESS.value],
ssh_username=username or rds_items[SSHServer.SSH_USERNAME.value],
ssh_pkey=rds_items[SSHServer.SSH_PKEY.value],
remote_bind_address=(remote_bind_address or rds_items[SSHServer.REMOTE_BIND_ADDRESS.value],
rds_items[SSHServer.REMOTE_PORT.value]),
local_bind_address=(rds_items[SSHServer.LOCAL_BIND_ADDRESS.value],
rds_items[SSHServer.LOCAL_PORT.value])
)
sshtunnel.SSH_TIMEOUT = 18000.0
sshtunnel.TUNNEL_TIMEOUT = 18000.0
def start(self):
"""Start Connection"""
try:
self.tunnel.start()
logger.info("Tunnel Connection Started")
except Exception as exception:
logger.error(exception)
def stop(self):
"""Stop Connection"""
try:
self.tunnel.stop()
logger.info("Tunnel Connection Finished")
except Exception as exception:
logger.error(exception)
2. database_connect 主代码, , SQL返回结果为Pandas dataframe格式
import pandas as pd
import sqlalchemy
from modules.utils.configloa