package com.luogankun.spark.base

import org.apache.spark.SparkConf

import org.apache.spark.SparkContext

import org.apache.spark.SparkContext._

/**

 * 统计字符出现次数

 */

object WorkCount {

  def main(args: Array[String]) {

  if (args.length < 1) {

    System.err.println("Usage: <file>")

    System.exit(1)

  }

  val conf = new SparkConf()

  val sc = new SparkContext(conf)

  //SparkContext 是把代码提交到集群或者本地的通道,我们编写 Spark代码,无论是要运行本地还是集群都必须有 SparkContext 的实例。

  val line = sc.textFile(args(0))

  //把读取的内容保存给line变量,其实line是一个MappedRDD,Spark的代码,都是基于RDD操作的;

  line.flatMap(_.split("")).map((_, 1)).reduceByKey(_+_).collect.foreach(println)

  sc.stop

  }

}

#!/bin/bash

cd $SPARK_HOME/bin

spark-submit \

--master spark://master:7077 \

--class com.luogankun.spark.base.WorkCount \

--name wordcount \

--executor-memory 400M \

--driver-memory 512M \

/usr/local/myjar/spark.jar \

hdfs://master:9000/user/hadoop/hadoop.txt