测试环境 : win
服务环境 : centos7.5
版本 : hadoop3.1.3
1.pom.xml
<dependencies> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> <version>3.1.3</version> </dependency> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>4.12</version> </dependency> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> <version>1.7.30</version> </dependency> </dependencies>
2.在resources 添加 log4j.properties
log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=target/spring.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n
3.客户端代码
package com.hadoop.hdfs; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.junit.After; import org.junit.Before; import org.junit.Test; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.Arrays; /** * @author cun.gao * @create 2021/8/22 - 9:08 */ public class HdfsClient { Configuration configuration ; FileSystem fs ; /* *注意事项: * 1. 参数优先级: (1)客户端代码中设置的值 > * (2)ClassPath 下的用户自定义配置文件 > * (3)然后是服务器的自定义配置(xxx-site.xml)> * (4)服务器的默认配置(xxx-default.xml) * */ @Before public void getFs() throws URISyntaxException, IOException, InterruptedException { // 参数设置 String URL = "hdfs://hadoop01:8020"; String UserName = "root"; // 创建 Configuration 对象,读取相关配置文件 configuration = new Configuration(); //设置参数 设置副本数 //configuration.set("dfs.replication", "20"); fs = FileSystem.get(new URI(URL), configuration,UserName); } //关闭资源 @After public void close() throws IOException { fs.close(); System.out.println("fs已关闭..."); } //创建目录 @Test public void testMkdirs() throws IOException { // 1 创建目录 fs.mkdirs(new Path("/java_api/demo1/")); } //上传文件 @Test public void testcopyFromLocalFile() throws IOException { // 2 上传文件 /* boolean delSrc 是否删除源文件, boolean overwrite 是否覆盖hdfs文件, Path src 源文件路径, Path dst hdfs路径 */ Path srcPath = new Path("d:/tmp/gaocun.txt"); Path dfsPath = new Path("/java_api/"); fs.copyFromLocalFile(false,false,srcPath,dfsPath); System.out.println(srcPath.getName()+"文件已上传"); } //下载文件 @Test public void testcopyToLocalFile() throws IOException { // 3 下载文件 /* boolean delSrc 是否删除源文件, Path src hdfs路径, Path dst 本地路径, boolean useRawLocalFileSystem 是否开启文件校验 * */ Path srcPath = new Path("d:/tmp/download.txt"); Path dfsPath = new Path("/java_api/gaocun.txt"); fs.copyToLocalFile(true,dfsPath,srcPath,true); System.out.println(dfsPath.getName()+"文件已下载"); } //文件更名和移动 @Test public void testRename() throws IOException { fs.rename(new Path("/java_api/demo"),new Path("/java_api/demoNew")); System.out.println("文件已改名"); } //删除文件和目录 @Test public void testdelete() throws IOException { fs.delete(new Path("/java_api/demoNew"),true); System.out.println("文件已删除"); } //文件详情查看 @Test public void testFindAll() throws IOException { RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/sanguo/wuguo.txt"), true); while (listFiles.hasNext()) { LocatedFileStatus fileStatus = listFiles.next(); System.out.println("========" + fileStatus.getPath() + "========="); System.out.println(fileStatus.getPermission()); System.out.println(fileStatus.getOwner()); System.out.println(fileStatus.getGroup()); System.out.println(fileStatus.getLen()); System.out.println(fileStatus.getModificationTime()); System.out.println(fileStatus.getReplication()); System.out.println(fileStatus.getBlockSize()); System.out.println(fileStatus.getPath().getName()); // 获取块信息 BlockLocation[] blockLocations = fileStatus.getBlockLocations(); System.out.println(Arrays.toString(blockLocations)); } } }