1.HDFS的put上传文件操作的java代码：
package Hdfs; import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils; public class FileSystemTest {
public static void main(String[] args) throws Exception {
//注：hadoop用户是具有集群文件系统权限的用户。
System.setProperty("HADOOP_USER_NAME","root");
System.setProperty("HADOOP_USER_PASSWORD","neusoft");
FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://neusoft-master:9000"), new Configuration());
//ls(fileSystem); //提取的方法
put(fileSystem); //上传的方法
} private static void put(FileSystem fileSystem) throws IOException,
FileNotFoundException {
//put创建上传图片C:\Users\SimonsZhao\Desktop\234967-13112015163685.jpg
FSDataOutputStream out = fileSystem.create(new Path("/234967-13112015163685.jpg"));
//两个流嵌套
IOUtils.copyBytes(new FileInputStream("C:\\Users\\SimonsZhao\\Desktop\\234967-13112015163685.jpg"),
out, 1024, true);
} private static void ls(FileSystem fileSystem) throws Exception {
//ls命令的操作
FileStatus[] listStatus = fileSystem.listStatus(new Path("/"));
for (FileStatus fileStatus : listStatus) {
System.out.println(fileStatus);
}
}
}
java客户端操作Hdfs
Linux服务器端验证：
问题总结：
( 1).权限问题：
log4j:WARN No appenders could be found for logger (org.apache.hadoop.metrics2.lib.MutableMetricsFactory).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
Exception in thread "main" org.apache.hadoop.security.AccessControlException: Permission denied: user=SimonsZhao, access=WRITE, inode="/234967-13112015163685.jpg":root:supergroup:-rw-r--r--
at org.apache.hadoop.hdfs.server.namenode.DefaultAuthorizationProvider.checkFsPermission(DefaultAuthorizationProvider.java:257)
at org.apache.hadoop.hdfs.server.namenode.DefaultAuthorizationProvider.check(DefaultAuthorizationProvider.java:238)
at org.apache.hadoop.hdfs.server.namenode.DefaultAuthorizationProvider.checkPermission(DefaultAuthorizationProvider.java:151)
at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:138)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkPermission(FSNamesystem.java:6609)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkPermission(FSNamesystem.java:6591)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkPathAccess(FSNamesystem.java:6516)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInternal(FSNamesystem.java:2751)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInt(FSNamesystem.java:2675)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFile(FSNamesystem.java:2560)
at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.create(NameNodeRpcServer.java:593)
at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.create(AuthorizationProviderProxyClientProtocol.java:111)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.create(ClientNamenodeProtocolServerSideTranslatorPB.java:393)
at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:617)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1060)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2086)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2082)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1707)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2080) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:73)
at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:1730)
at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1668)
at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1593)
at org.apache.hadoop.hdfs.DistributedFileSystem$6.doCall(DistributedFileSystem.java:397)
at org.apache.hadoop.hdfs.DistributedFileSystem$6.doCall(DistributedFileSystem.java:393)
at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:393)
at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:337)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:908)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:889)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:786)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:775)
at Hdfs.FileSystemTest.main(FileSystemTest.java:23)
Caused by: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.AccessControlException): Permission denied: user=SimonsZhao, access=WRITE, inode="/234967-13112015163685.jpg":root:supergroup:-rw-r--r--
at org.apache.hadoop.hdfs.server.namenode.DefaultAuthorizationProvider.checkFsPermission(DefaultAuthorizationProvider.java:257)
at org.apache.hadoop.hdfs.server.namenode.DefaultAuthorizationProvider.check(DefaultAuthorizationProvider.java:238)
at org.apache.hadoop.hdfs.server.namenode.DefaultAuthorizationProvider.checkPermission(DefaultAuthorizationProvider.java:151)
at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:138)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkPermission(FSNamesystem.java:6609)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkPermission(FSNamesystem.java:6591)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkPathAccess(FSNamesystem.java:6516)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInternal(FSNamesystem.java:2751)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInt(FSNamesystem.java:2675)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFile(FSNamesystem.java:2560)
at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.create(NameNodeRpcServer.java:593)
at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.create(AuthorizationProviderProxyClientProtocol.java:111)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.create(ClientNamenodeProtocolServerSideTranslatorPB.java:393)
at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:617)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1060)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2086)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2082)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1707)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2080) at org.apache.hadoop.ipc.Client.call(Client.java:1468)
at org.apache.hadoop.ipc.Client.call(Client.java:1399)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:232)
at com.sun.proxy.$Proxy9.create(Unknown Source)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:295)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
at com.sun.proxy.$Proxy10.create(Unknown Source)
at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:1725)
... 12 more
HDFS上传文件所报的异常信息
原因：在windows环境下提交任务是用的本机的admin用户，此处是SimonsZhao我个人账户，并不具有集群文件系统的操作权限。
解决办法：
//注：root用户是具有集群文件系统权限的用户。虚拟机操作系统的用户名为root，密码为xxx
System.setProperty("HADOOP_USER_NAME","root");
System.setProperty("HADOOP_USER_PASSWORD","xxx");
增加以上代码到main方法的首行即可。
(2).对上述问题可能需要修改两处配置文件(待验证)
修改hadoop安装集群master的hadoop-1.2.0/conf/mapred-site.xml，增加：
<property>
<name>dfs.permissions</name>
<value>false</value>
</property>
修改hdfs-site.xml文件里修改吧
<property>
<name>dfs.permissions</name>
<value>false</value>
</property>
hdfs fs -chmod 777 /tmp
<property>
<name>dfs.permissions</name>
<value>false</value>
</property>
hdfs fs -chmod 777 /tmp
2.如果想上传到HDFS的文件和linux本地的相同的话，请使用-p命令，如下
-p Preserves access and modification times, ownership and the mode.
保留访问和修改时间，所有权和模式。
[root@neusoft-master hadoop]# vi /usr/local/software/hello.txt
hello you
hello me
[root@neusoft-master hadoop]# hadoop dfs -put -p /usr/local/software/hello.txt /hello2
DEPRECATED: Use of this script to execute hdfs command is deprecated.
Instead use the hdfs command for it.
[root@neusoft-master hadoop]# hadoop dfs -ls /
DEPRECATED: Use of this script to execute hdfs command is deprecated.
Instead use the hdfs command for it.
Found 5 items
-rw-r--r-- 3 root supergroup 120064 2017-01-07 09:54 /234967-13112015163685.jpg
drwxr-xr-x - root supergroup 0 2017-01-06 22:18 /hbase
-rw-r--r-- 3 root root 20 2017-01-07 10:14 /hello2
drwxrwxrwx - root supergroup 0 2017-01-06 21:37 /tmp
drwxr-xr-x - root supergroup 0 2017-01-06 21:39 /user
上面的时间、用户都发生了改变，hello2的时间是当时上传的时间，这就是保留作用
[root@neusoft-master hadoop]# hadoop dfs -cat /hello2
DEPRECATED: Use of this script to execute hdfs command is deprecated.
Instead use the hdfs command for it.
17/01/07 10:19:06 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
hello you
hello me
3.java操作HDFS下载文件系统中的文件并输出在本地显示
package Hdfs; import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils; public class FileSystemTest {
public static void main(String[] args) throws Exception {
//注：hadoop用户是具有集群文件系统权限的用户。
System.setProperty("HADOOP_USER_NAME","root");
System.setProperty("HADOOP_USER_PASSWORD","neusoft");
FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://neusoft-master:9000"), new Configuration());
//ls(fileSystem); //提取的方法
//put(fileSystem); //上传的方法
show(fileSystem); //显示之前上传的图片
} private static void show(FileSystem fileSystem) throws IOException,
FileNotFoundException {
FSDataInputStream in = fileSystem.open(new Path("/234967-13112015163685.jpg"));
IOUtils.copyBytes(in,
new FileOutputStream("C:\\Users\\SimonsZhao\\Desktop\\___234967-13112015163685.jpg"),1024,true);
} private static void put(FileSystem fileSystem) throws IOException,
FileNotFoundException {
//put创建上传图片C:\Users\SimonsZhao\Desktop\234967-13112015163685.jpg
FSDataOutputStream out = fileSystem.create(new Path("/234967-13112015163685.jpg"));
//两个流嵌套
IOUtils.copyBytes(new FileInputStream("C:\\Users\\SimonsZhao\\Desktop\\234967-13112015163685.jpg"),
out, 1024, true);
} private static void ls(FileSystem fileSystem) throws Exception {
//ls命令的操作
FileStatus[] listStatus = fileSystem.listStatus(new Path("/"));
for (FileStatus fileStatus : listStatus) {
System.out.println(fileStatus);
}
}
}
本地下载显示HDFS上面的图片
在桌面显示如下：
Eclipse中抽取代码:alt+shift+M抽取为函数
4.删除HDFS根目录下面的图片
package Hdfs; import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils; public class FileSystemTest {
public static void main(String[] args) throws Exception {
//注：hadoop用户是具有集群文件系统权限的用户。
System.setProperty("HADOOP_USER_NAME","root");
System.setProperty("HADOOP_USER_PASSWORD","neusoft");
FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://neusoft-master:9000"), new Configuration());
//ls(fileSystem); //提取的方法
//put(fileSystem); //上传的方法
//show(fileSystem); //显示之前上传的图片
//删除根目录下面的图片
fileSystem.delete(new Path("/234967-13112015163685.jpg"),false);
} private static void show(FileSystem fileSystem) throws IOException,
FileNotFoundException {
FSDataInputStream in = fileSystem.open(new Path("/234967-13112015163685.jpg"));
IOUtils.copyBytes(in,
new FileOutputStream("C:\\Users\\SimonsZhao\\Desktop\\___234967-13112015163685.jpg"),1024,true);
} private static void put(FileSystem fileSystem) throws IOException,
FileNotFoundException {
//put创建上传图片C:\Users\SimonsZhao\Desktop\234967-13112015163685.jpg
FSDataOutputStream out = fileSystem.create(new Path("/234967-13112015163685.jpg"));
//两个流嵌套
IOUtils.copyBytes(new FileInputStream("C:\\Users\\SimonsZhao\\Desktop\\234967-13112015163685.jpg"),
out, 1024, true);
} private static void ls(FileSystem fileSystem) throws Exception {
//ls命令的操作
FileStatus[] listStatus = fileSystem.listStatus(new Path("/"));
for (FileStatus fileStatus : listStatus) {
System.out.println(fileStatus);
}
}
}
删除根目录下的图片