nansen0324的博客

使用airflow从datahub获取数据到DB，修改数据，打包csv档发送minio，task执行成功邮件通知复写等

# 由于ES限制, 当偏移量>10000时需要特殊处理, 此处略from elasticsearch import Elasticsearchfrom elasticsearch.helpers import bulkimport csvimport osimport sysimport datetimeimport json# 解决写入文件时中文编码问题# reload(sys)# sys.setdefaultencoding('utf-8')# 存储列名称的listco

from airflow import DAGfrom airflow.operators.python import PythonOperatorfrom airflow.providers.amazon.aws.hooks.s3 import S3Hookfrom airflow.providers.postgres.hooks.postgres import PostgresHookfrom airflow.utils.dates import days_agofrom airflow.ex

import pandas as pdimport boto3from sqlalchemy import create_engine# create engine to connect mysql(請修user/password/db_name)pg_engine_destination = create_engine("mysql+mysqlconnector://用户名:密码@IP地址/schema名")# s3服务器(minio)连接资料s3_resource = boto3.re

import psycopg2import boto3stmt = """SELECT film.film_id, title, inventory_idFROM filmLEFT JOIN inventory ON inventory.film_id = film.film_idORDER BY title;"""#创建数据库连接db_conn = psycopg2.connect(host='数据库连接ip', dbname='数据库sche

pandas读取压缩档文件，再进行压缩上传到minio或其他s3服务器import boto3from zipfile import ZipFileimport pandas as pd# s3服务器的连接配置s3_resource = boto3.resource('s3', endpoint_url='minio网址或其他s3服务器', aws_access_key_id='账号',

pandas读取json文件并进行刷选导出xlsx文件原始json数据import pandas as pdimport pprint# 读取json文件df_tv_shows = pd.read_json("datas/tv_shows.json")# 读取json文件中shows数组下所有数据first_obj = df_tv_shows.loc[:, "shows"]# 将shows数组数据读出，其中episodes是shows下的数组df_tmp = pd.json_norma

pycharm智能提示忽略大小写settings=>Editor=>General=>code completion

pycharm安装依赖包file=>settings