Spark MLlib线性回归代码实现及结果展示

最新推荐文章于 2024-08-29 15:24:37 发布

那记忆微凉

最新推荐文章于 2024-08-29 15:24:37 发布

阅读量1.8k

点赞数 1

分类专栏：机器学习 Spark

Spark 同时被 2 个专栏收录

33 篇文章

订阅专栏

机器学习

19 篇文章

订阅专栏

　　线性回归(Linear Regression)是利用称为线性回归方程的最小平方函数对一个或多个自变量和因变量之间关系进行建模的一种回归分析。

这种函数是一个或多个称为回归系数的模型参数的线性组合。只有一个自变量的情况称为一元线性回归,大于一个自变量情况的叫做多元线性回归。

代码实现：
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.DataFrame
import org.apache.spark.ml.feature.VectorAssembler
import org.apache.spark.ml.regression.LinearRegression

object LinearRegression {
  def main(args: Array[String]) {
    //设置环境
  val spark = SparkSession.builder ().appName ("LinearRegressionTest").master ("local[2]").getOrCreate()
    val sc = spark.sparkContext
    val sqlContext = spark.sqlContext
    //准备训练集合
    val raw_data = sc.textFile("src/sparkMLlib/man.txt")
    val map_data = raw_data.map{x=>
      val mid = x.replaceAll(","," ,")
      val split_list = mid.substring(0,mid.length-1).split(",")
      for(x <- 0 until split_list.length){
        if(split_list(x).trim.equals("")) split_list(x) = "0.0" else split_list(x) = split_list(x).trim
      }
      ( split_list(1).toDouble,split_list(2).toDouble,split_list(3).toDouble,split_list(4).toDouble,
        split_list(5).toDouble,split_list(6).toDouble,split_list(7).toDouble,split_list(8).toDouble,
        split_list(9).toDouble,split_list(10).toDouble,split_list(11).toDouble)
    }
    val mid = map_data.sample(false,0.6,0)//随机取样，训练模型
    val df = sqlContext.createDataFrame(mid)
    val colArray = Array("c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10", "c11")
    val data = df.toDF("c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10", "c11")
    val assembler = new VectorAssembler().setInputCols(colArray).setOutputCol("features")
    val vecDF = assembler.transform(data)
    //准备预测集合
    val map_data_for_predict = map_data
    val df_for_predict = sqlContext.createDataFrame(map_data_for_predict)
    val data_for_predict = df_for_predict.toDF("c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10", "c11")
    val colArray_for_predict = Array("c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10", "c11")
    val assembler_for_predict = new VectorAssembler().setInputCols(colArray_for_predict).setOutputCol("features")
    val vecDF_for_predict: DataFrame = assembler_for_predict.transform(data_for_predict)
    // 建立模型，进行预测
    // 设置线性回归参数
    val lr1 = new LinearRegression()
    val lr2 = lr1.setFeaturesCol("features").setLabelCol("c5").setFitIntercept(true)
    // RegParam：正则化
    val lr3 = lr2.setMaxIter(10).setRegParam(0.3).setElasticNetParam(0.8)
    // 将训练集合代入模型进行训练
    val lrModel = lr3.fit(vecDF)
    // 输出模型全部参数
    lrModel.extractParamMap()
    //coefficients 系数 intercept 截距
    println(s"Coefficients: ${lrModel.coefficients} Intercept: ${lrModel.intercept}")
    // 模型进行评价
    val trainingSummary = lrModel.summary
    trainingSummary.residuals.show()
    println(s"均方根差: ${trainingSummary.rootMeanSquaredError}")//RMSE:均方根差
    println(s"判定系数: ${trainingSummary.r2}")//r2:判定系数，也称为拟合优度，越接近1越好
    val predictions = lrModel.transform(vecDF_for_predict)
    val predict_result = predictions.selectExpr("features","c5", "round(prediction,1) as prediction")
    predict_result.rdd.saveAsTextFile("src/sparkMLlib/manResult")
    sc.stop()
  }
}

性能评估：

结果：