Hadoop—HDFS的Java客户端编程

    可以使用HDFS Shell命令来完成对HDFS的操作，当然也可以使用Java编程语言来对HDFS实现操作。以下是一些Unit测试代码

package cn.nuc.hadoop.hdfs;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.commons.compress.utils.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;

public class HdfsUtil {

FileSystem fs = null;

@Before
public void init() throws IOException, InterruptedException,
URISyntaxException {
Configuration conf = new Configuration();// 读取配置文件

// 设置fs.defaultFS参数，如果没有设置，会出现java.lang.IllegalArgumentException:
// Wrong FS:hdfs://master:9000/user,expected: file:///
// 当然了，也可以将hadoop集群中的core-site.xml配置文件拷贝到该工程项目下,这样，在读取配置文件时就能够识别hdfs文件系统
conf.set("fs.defaultFS", "hdfs://master:9000/");

fs = FileSystem.get(new URI("hdfs://master:9000/"), conf, "hadoop");// 获取hdfs实例
}

// upload a loacl file to HDFS
@Test
public void upload() throws IOException {

Path dest = new Path("hdfs://master:9000/user/artifacts.xml");

FSDataOutputStream fsOut = fs.create(dest);

FileInputStream localIn = new FileInputStream(
"D:\\eclipse\\artifacts.xml");

IOUtils.copy(localIn, fsOut);
}

@Test
public void upload2() throws IllegalArgumentException, IOException {
fs.copyFromLocalFile(new Path("D:\\eclipse\\artifacts.xml"), new Path(
"hdfs://master:9000/user/artifacts2.xml"));
}

// download a file to local file
@Test
public void download() throws IllegalArgumentException, IOException {
fs.copyToLocalFile(new Path("hdfs://master:9000/user/artifacts2.xml"),
new Path("D:\\a.xml"));
}

// list HDFS file
@Test
public void listFiles() throws FileNotFoundException, IllegalArgumentException, IOException {
// listFiles列出的是文件信息，而且提供递归遍历
RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path("/"),
true);

while (files.hasNext()) {

LocatedFileStatus file = files.next();
Path filePath = file.getPath();
String fileName = filePath.getName();
System.out.println(fileName);

}

System.out.println("---------------------------------");

// listStatus 可以列出文件和文件夹的信息，但是不提供自带的递归遍历
FileStatus[] listStatus = fs.listStatus(new Path("/"));
for (FileStatus status : listStatus) {

String name = status.getPath().getName();
System.out.println(name
+ (status.isDirectory() ? " is dir" : " is file"));

}
}

// create HDFS folder
@Test
public void mkdir() throws IllegalArgumentException, IOException {
fs.mkdirs(new Path("hdfs://master:9000/aaa/bbb/ccc"));
}

// remove HDFS folder or file
@Test
public void remove() throws IllegalArgumentException, IOException {
fs.delete(new Path("hdfs://master:9000/aaa"), true);
}
}