去除null、NaN

去除 dataframe 中的 null 、 NaN 有方法 drop ，用 dataframe.na 找出带有 null、 NaN 的行，用 drop 删除行：

import org.apache.spark.{SparkConf, SparkContext}

import org.apache.spark.sql.{DataFrame, SQLContext, SparkSession}

/**

  * Created by TTyb on 2017/10/12.

  */

object test3 {

  def main(args: Array[String]): Unit = {

    val conf = new SparkConf().setAppName("TTyb").setMaster("local")

    val sc = new SparkContext(conf)

    val spark=new SQLContext(sc)

    val sentenceDataFrame = spark.createDataFrame(Seq(

      (1, "asf"),

      (2, "2143"),

      (3, "rfds"),

      (4, null),

      (5, "")

    )).toDF("label", "sentence")

    sentenceDataFrame.show()

    sentenceDataFrame.na.drop().show()

  }

}

去除空字符串

去除空字符串用 dataframe.where ：

import org.apache.spark.{SparkConf, SparkContext}

import org.apache.spark.sql.{DataFrame, SQLContext, SparkSession}

/**

  * Created by TTyb on 2017/10/12.

  */

object test3 {

  def main(args: Array[String]): Unit = {

    val conf = new SparkConf().setAppName("TTyb").setMaster("local")

    val sc = new SparkContext(conf)

    val spark=new SQLContext(sc)

    val sentenceDataFrame = spark.createDataFrame(Seq(

      (1, "asf"),

      (2, "2143"),

      (3, "rfds"),

      (4, null),

      (5, "")

    )).toDF("label", "sentence")

    sentenceDataFrame.show()

    // sentenceDataFrame.na.drop().show()

    sentenceDataFrame.where("sentence <> ''").show()

  }

}