spark sql加载

object FieldDemo extends App {
  private val session: SparkSession = SparkSession.builder().appName("test").master("local").getOrCreate()

  //private val frame: DataFrame = session.read.load("./data/emp.json")
  //private val frame: DataFrame = session.read.format("json").load("./data/emp.json")
  //读取csv文件,还可以读取表头,指定分隔符
  //private val frame: DataFrame = session.read.format("csv").option("seq", ";").option("header", value = true).load("x")
  //private val frame: DataFrame = session.read.format("orc").load("./data/sql/student.orc")
  val prop = new Properties()
  prop.setProperty("user", "root")
  prop.setProperty("password", "123456")
  //读取mysql中的数据
  private val frame: DataFrame = session.read.jdbc("jdbc:mysql://localhost:3306/db2", "emp", prop)
  frame.show()
}

spark sql落地

object SaveDemo extends App {
  private val session: SparkSession = SparkSession.builder().appName("test").master("local").getOrCreate()
  private val frame: DataFrame = session.read.json("./data/emp.json")
  //形参为目录.默认写出的格式为parquet
  //frame.write.save("data/dir1")
  //指定写出的格式
  //frame.write.format("json").save("data/dir2")
  //csv格式可以将表头写到第一行.
  frame.write.format("csv").save("data/dir3")
  frame.write.format("orc").save("data/dir4")
  frame.write.format("text").save("data/dir5")
  //可以保存到数据库中
  val prop = new Properties()
  prop.setProperty("user", "root")
  prop.setProperty("password", "123456")
  /*可以指定保存模式
  SaveMode.Append
  SaveMode.ErrorIfExists 默认的,已有的表会报错
  SaveMode.Ignore  如果数据存在,就不保存df的数据
  SaveMode.Overwrite 删除原有表重新建立
  */
  frame.write.mode(SaveMode.Append).jdbc("jdbc:mysql://localhost:3306/db2", "emp2", prop)

}