spark 写 parquet 文件到本地

package SQL
import org.apache.spark.SparkContext
import org.apache.spark.sql.{DataFrame, SparkSession}

object Demo7_2 extends App {
  val spark: SparkSession = SparkSession.builder().master("local[4]")
  .appName("demo1").getOrCreate()
  val sc: SparkContext = spark.sparkContext
  import spark.implicits._

  private val df: DataFrame = spark.read.json("D:\\INTELLIJ\\Spark\\data\\users.json")
  df.write.parquet("file:///D:\\INTELLIJ\\Spark\\parquetoutput")
  
  private val df2: DataFrame = spark.read.parquet("file:///D:\\INTELLIJ\\Spark\\parquetoutput")
  df2.show()
}

spark 写 parquet 文件到 hdfs 上

package SQL
import org.apache.spark.SparkContext
import org.apache.spark.sql.{DataFrame, SparkSession}

object Demo7_2 extends App {
  val spark: SparkSession = SparkSession.builder().master("local[4]")
  .appName("demo1").getOrCreate()
  val sc: SparkContext = spark.sparkContext
  import spark.implicits._

  private val df: DataFrame = spark.read.json("D:\\INTELLIJ\\Spark\\data\\users.json")
  df.write.parquet("hdfs://192.168.1.54:9000/kgc/parquetoutput")
 
  private val df2: DataFrame = spark.read.parquet("hdfs://192.168.1.54:9000/kgc/parquetoutput")
  df2.show()
}

需要注意的点：
（1）“file:///” 与 “hdfs://”
（2）“hdfs://ip:9000”
（3） 若报错hdfs上面的文件夹的权限问题，需要用到的命令是 hdfs dfs -chmod -R 777 /文件名