airflow从datahub获取数据到mysql数据库

最新推荐文章于 2025-06-15 09:00:00 发布

原创

最新推荐文章于 2025-06-15 09:00:00 发布 · 704 阅读

1 ·

CC 4.0 BY-SA版权

文章标签：

#数据库 #mysql #pandas #etl

使用airflow从datahub获取数据到DB，修改数据，打包csv档发送minio，task执行成功邮件通知复写等

import smtplib
import pendulum
from email.mime.text import MIMEText
from email.header import Header
from airflow import DAG
from airflow.utils.dates import days_ago
from datahub.operators.datahub_to_rdb import DatahubToRDBOperator
from datetime import datetime, timedelta
from airflow.operators.python import PythonOperator
from airflow.operators.dummy import DummyOperator
from airflow.providers.mysql.operators.mysql import MySqlOperator
import pandas as pd
import boto3
from airflow.hooks.mysql_hook import MySqlHook
import tempfile
from airflow.exceptions import AirflowException
import math


def  readdata():
    # 创建mysql引擎
    mysql_hook=MySqlHook(mysql_conn_id="数据库连接id")
    sqlalchemy_engine=mysql_hook.get_sqlalchemy_engine();
    try:
      df=pd.read_sql("SELECT *   FROM  employeeinfo  ",sqlalchemy_engine)
      print(df.head())
      print('--------------')
      print(df.info())
      #生成test.csv档，并设置索引为false
      df.to_csv(