pyspark头部代码

# coding=utf-8

##################### 注释 ###############################

from pyspark.sql import SparkSession

spark = SparkSession.builder.appName('aaa') \
    .config('hive.exec.dynamic.partition', 'true') \
    .config('hive.exec.dynamic.partition.mode', 'nonstrict') \
    .config('spark.sql.sources.partitionOverwriteMode', 'dynamic') \
    .config('spark.serializer', 'org.apache.spark.serializer.KryoSerializer') \
    .config('spark.kryoserializer.buffer.max', '512m') \
    .config('spark.rpc.message.maxSize', '2000') \
    .config('spark.debug.maxToStringFields', 1000) \
    .config("spark.sql.crossJoin.enabled", "true") \
    .enableHiveSupport().getOrCreate()
    # .config('spark.sql.hive.convertMetastoreOrc','false') \  # 如果spark检测不到hive数据可以使用
spark.sparkContext.setLogLevel("warn")

sc=spark.sparkContext
sc.addPyFile("hdfs:///aa/aaa.tar")
sc.addPyFile("hdfs:///bb/bbb.tar")
import aaa as bbb
import ccc as ddd
import os;
import pandas as pd;
import time;
from datetime import datetime,timedelta
import traceback
from pyspark.sql.types import *
from pyspark.sql.functions import acos, cos, radians, sin, lit, col
from pyspark.sql.window import Window
import sys