【甘道夫】Win7环境下Eclipse连接Hadoop2.2.0

2021-11-09 03:17:52

准备：

确保hadoop2.2.0集群正常运行

1.eclipse中建立java工程，导入hadoop2.2.0相关jar包

2.在src根目录下拷入log4j.properties，通过log4j查看详细日志

log4j.rootLogger=debug, stdout, R

log4j.appender.stdout=org.apache.log4j.ConsoleAppender

log4j.appender.stdout.layout=org.apache.log4j.PatternLayout

log4j.appender.stdout.layout.ConversionPattern=%5p - %m%n

log4j.appender.R=org.apache.log4j.RollingFileAppender

log4j.appender.R.File=firestorm.log

log4j.appender.R.MaxFileSize=100KB

log4j.appender.R.MaxBackupIndex=1

log4j.appender.R.layout=org.apache.log4j.PatternLayout

log4j.appender.R.layout.ConversionPattern=%p %t %c - %m%n

log4j.logger.com.codefutures=DEBUG

3.拷入一个可执行的hadoop程序，我用的是一个HdfsDAO，可以先保证HDFS操作能执行

import java.io.IOException;

import java.net.URI;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FSDataInputStream;

import org.apache.hadoop.fs.FSDataOutputStream;

import org.apache.hadoop.fs.FileStatus;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IOUtils;

import org.apache.hadoop.mapred.JobConf;

public class HdfsDAO {

private static final String HDFS = "hdfs://192.168.0.160:9000/";

public HdfsDAO(Configuration conf) {

this(HDFS, conf);

}

public HdfsDAO(String hdfs, Configuration conf) {

this.hdfsPath = hdfs;

this.conf = conf;

}

private String hdfsPath;

private Configuration conf;

public static void main(String[] args) throws IOException {

JobConf conf = config();

HdfsDAO hdfs = new HdfsDAO(conf);

// hdfs.copyFile("datafile/item.csv", "/tmp/new");

// hdfs.ls("/tmp/new");

hdfs.ls("/");

}

public static JobConf config(){

JobConf conf = new JobConf(HdfsDAO.class);

conf.setJobName("HdfsDAO");

conf.addResource("classpath:/hadoop/core-site.xml");

conf.addResource("classpath:/hadoop/hdfs-site.xml");

conf.addResource("classpath:/hadoop/mapred-site.xml");

return conf;

}

public void mkdirs(String folder) throws IOException {

Path path = new Path(folder);

FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

if (!fs.exists(path)) {

fs.mkdirs(path);

System.out.println("Create: " + folder);

}

fs.close();

}

public void rmr(String folder) throws IOException {

Path path = new Path(folder);

FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

fs.deleteOnExit(path);

System.out.println("Delete: " + folder);

fs.close();

}

public void ls(String folder) throws IOException {

Path path = new Path(folder);

FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

FileStatus[] list = fs.listStatus(path);

System.out.println("ls: " + folder);

System.out.println("==========================================================");

for (FileStatus f : list) {

System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath(), f.isDir(), f.getLen());

}

System.out.println("==========================================================");

fs.close();

}

public void createFile(String file, String content) throws IOException {

FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

byte[] buff = content.getBytes();

FSDataOutputStream os = null;

try {

os = fs.create(new Path(file));

os.write(buff, 0, buff.length);

System.out.println("Create: " + file);

} finally {

if (os != null)

os.close();

}

fs.close();

}

public void copyFile(String local, String remote) throws IOException {

FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

fs.copyFromLocalFile(new Path(local), new Path(remote));

System.out.println("copy from: " + local + " to " + remote);

fs.close();

}

public void download(String remote, String local) throws IOException {

Path path = new Path(remote);

FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

fs.copyToLocalFile(path, new Path(local));

System.out.println("download: from" + remote + " to " + local);

fs.close();

}

public void cat(String remoteFile) throws IOException {

Path path = new Path(remoteFile);

FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

FSDataInputStream fsdis = null;

System.out.println("cat: " + remoteFile);

try {

fsdis =fs.open(path);

IOUtils.copyBytes(fsdis, System.out, 4096, false);

} finally {

IOUtils.closeStream(fsdis);

fs.close();

}

public void location() throws IOException {

// String folder = hdfsPath + "create/";

// String file = "t2.txt";

// FileSystem fs = FileSystem.get(URI.create(hdfsPath), new

// Configuration());

// FileStatus f = fs.getFileStatus(new Path(folder + file));

// BlockLocation[] list = fs.getFileBlockLocations(f, 0, f.getLen());

// System.out.println("File Location: " + folder + file);

// for (BlockLocation bl : list) {

// String[] hosts = bl.getHosts();

// for (String host : hosts) {

// System.out.println("host:" + host);

// }

// fs.close();

}

4.运行HdfsDAO

报错：

java.io.IOException: HADOOP_HOME or hadoop.home.dir are not set.

at org.apache.hadoop.util.Shell.checkHadoopHome(Shell.java:225)

at org.apache.hadoop.util.Shell.<clinit>(Shell.java:250)

at org.apache.hadoop.util.StringUtils.<clinit>(StringUtils.java:76)

at org.apache.hadoop.conf.Configuration.getTrimmedStrings(Configuration.java:1546)

at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:519)

at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:453)

at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:136)

at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2433)

at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:88)

at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2467)

at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2449)

at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:367)

at HdfsDAO.copyFile(HdfsDAO.java:94)

at HdfsDAO.main(HdfsDAO.java:34)

ERROR - Failed to locate the winutils binary in the hadoop binary path

java.io.IOException: Could not locate executable null\bin\winutils.exe in the Hadoop binaries.

at org.apache.hadoop.util.Shell.getQualifiedBinPath(Shell.java:278)

at org.apache.hadoop.util.Shell.getWinUtilsPath(Shell.java:300)

at org.apache.hadoop.util.Shell.<clinit>(Shell.java:293)