Pyspark UI界面

最新推荐文章于 2025-08-23 09:37:32 发布

原创

最新推荐文章于 2025-08-23 09:37:32 发布 · 446 阅读

1 ·

CC 4.0 BY-SA版权

文章标签：

#ui #spark #hadoop

from pyspark import SparkConf,SparkContext
from pyspark.sql import Row
from pyspark.sql.types import *
from pyspark.sql import SparkSession
from datetime import date, datetime
from pyspark.sql import functions as F

from pyecharts.charts import Bar,Line,Grid,WordCloud
from pyecharts import options as opts
from pyecharts.components import Table
from pyecharts.charts import WordCloud
from pyecharts.options import ComponentTitleOpts
from pyecharts.charts import PictorialBar
from pyecharts.globals import SymbolType
from pyecharts.charts import Pie
from pyecharts.charts import Funnel
from pyecharts.charts import Scatter

def toDate(inputStr):
    newStr = ""
    if len(inputStr) == 8:
        s1 = inputStr[0:4]
        s2 = inputStr[5:6]
        s3 = inputStr[7]
        newStr = s1+"-"+"0"+s2+"-"+"0"+s3
    else:
        s1 = inputStr[0:4]
        s2 = inputStr[5:6]
        s3 = inputStr[7:]
        newStr = s1+"-"+"0"+s2+"-"+s3
    date = datetime.strptime(newStr, "%Y-%m-%d")
    return date
 


spark = SparkSession.builder.config(conf = SparkConf()).getOrCreate()
fields = [StructField("date", DateType(),False),StructField("county", StringType(),False),StructField("state", StringType(),False),
                    StructField("cases", IntegerType(),False),StructField("deaths", IntegerType(),False),]
schema = StructType(fields)
 
rdd0 = spark.sparkContext.textFile("file:////home/hadoop/us-counties.txt")
rdd1 = rdd0.map(lambda x:x.split("\t")).map(lambda p: Row(toDate(p[0]),p[1],p[2],int