编写第一个Spark程序WordCount

这里使用IDEA创建项目，其次，在setting-plugins中查找并安装插件Scala

开始创建项目

配置sbt

name := "spark01"

version := "1.0"

scalaVersion := "2.11.7"

libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % "2.0.2"
)

编写代码ObWorCount

import org.apache.spark.{SparkContext, SparkConf}

/**
  * Created by Y.S.K on 2017/8/30 in spark01.
  */
object WordCount {

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("wordcount")
    val sc = new SparkContext(conf)

    val input = sc.textFile("/home/briup/helloSpark.txt")

    val lines = input.flatMap(line => line.split(" "))
    val count = lines.map(word => (word, 1)).reduceByKey { case (x, y) => x + y }

    val output = count.saveAsTextle("/home/briup/he

lloSparkRes")
  }
}

编译