package dada;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class HDFSToll {
//路径是否存在
public static boolean testExist(Configuration conf,String path) throws IOException
{
FileSystem fs=FileSystem.get(conf);
return fs.exists(new Path(path));
}
//创建目录
public static boolean mkdir (Configuration conf ,String remoteDir)throws IOException
{
FileSystem fs=FileSystem.get(conf);
Path dirPath=new Path(remoteDir);
boolean result=fs.mkdirs(dirPath);
fs.close();
return result;
}
/**
* 删除目录
*/
public static boolean rmDir(Configuration conf, String remoteDir) throws IOException {
FileSystem fs = FileSystem.get(conf);
Path dirPath = new Path(remoteDir);
/* 第二个参数表示是否递归删除所有文件 */
boolean result = fs.delete(dirPath, true);
fs.close();
return result;
}
//创建文件
public static void touch(Configuration conf,String remoteFilePath )throws IOException
{
FileSystem fs=FileSystem.get(conf);
Path remotePath=new Path(remoteFilePath);
FSDataOutputStream outputStream =fs.create(remotePath);
outputStream.close();
fs.close();
}
//删除文件
public static boolean rm(Configuration conf,String remoteFilePath)throws IOException
{
FileSystem fs=FileSystem.get(conf);
Path remotePath=new Path(remoteFilePath);
boolean result=fs.delete(remotePath,false);
fs.close();
return result;
}
//追加文件内容 到末尾
public static void appendContentToFile(Configuration conf,String content,String remoteFilePath)throws IOException
{
FileSystem fs=FileSystem.get(conf);
Path remotePath=new Path(remoteFilePath);
FSDataOutputStream out=fs.append(remotePath);
out.write(content.getBytes());
out.close();
fs.close();
}
//追加文件内容到开头
public static void appendContentToFile1(Configuration conf,String content,String remoteFilePath)throws IOException
{
String localTmpPath = "/usr/local/hadoop/enen.txt";
// 移动到本地
moveToLocalFile(conf, remoteFilePath, localTmpPath);
// 创建一个新文件
touch(conf, remoteFilePath);
// 先写入新内容
appendContentToFile(conf, content, remoteFilePath);
// 再写入原来内容
appendContentToFile(conf, localTmpPath, remoteFilePath);
System.out.println("已追加内容到文件开头: " + remoteFilePath);
}
/**
* 复制文件到指定路径
* 若路径已存在，则进行覆盖
*/
public static void copyFromLocalFile(Configuration conf, String localFilePath, String remoteFilePath) throws IOException {
FileSystem fs = FileSystem.get(conf);
Path localPath = new Path(localFilePath);
Path remotePath = new Path(remoteFilePath);
/* fs.copyFromLocalFile 第一个参数表示是否删除源文件，第二个参数表示是否覆盖 */
fs.copyFromLocalFile(false, true, localPath, remotePath);
fs.close();
}
//将文件1写入文件2
public static void appendFile1ToFile2(Configuration conf,String remoteFilePath,String remoteFilePath2)throws IOException
{
FileSystem fs=FileSystem.get(conf);
Path file=new Path(remoteFilePath);
FSDataInputStream getIt=fs.open(file);
BufferedReader d=new BufferedReader(new InputStreamReader(getIt));
String content1=d.readLine();
Path remotePath=new Path(remoteFilePath2);
FSDataOutputStream out=fs.append(remotePath);
out.write(content1.getBytes());
d.close();
out.close();
fs.close();
}
/**
* 追加文件内容
*/
public static void appendToFile(Configuration conf, String localFilePath, String remoteFilePath) throws IOException {
FileSystem fs = FileSystem.get(conf);
Path remotePath = new Path(remoteFilePath);
/* 创建一个文件读入流 */
FileInputStream in = new FileInputStream(localFilePath);
/* 创建一个文件输出流，输出的内容将追加到文件末尾 */
FSDataOutputStream out = fs.append(remotePath);
/* 读写文件内容 */
byte[] data = new byte[1024];
int read = -1;
while ( (read = in.read(data)) > 0 ) {
out.write(data, 0, read);
}
out.close();
in.close();
fs.close();
}
/**
* 下载文件到本地
* 判断本地路径是否已存在，若已存在，则自动进行重命名
*/
public static void copyToLocal(Configuration conf, String remoteFilePath, String localFilePath) throws IOException {
FileSystem fs = FileSystem.get(conf);
Path remotePath = new Path(remoteFilePath);
File f = new File(localFilePath);
/* 如果文件名存在，自动重命名(在文件名后面加上 _0, _1 ...) */
if (f.exists()) {
System.out.println(localFilePath + " 文件已存在.");
Integer i = 0;
while (true) {
f = new File(localFilePath + "_" + i.toString());
if (!f.exists()) {
localFilePath = localFilePath + "_" + i.toString();
System.out.println("将重新命名为: " + localFilePath);
break;//重命名文件
}
i++;
}
// System.out.println("将重新命名为: " + localFilePath);
}
else
System.out.println(localFilePath + " 文件不存在.");
// 下载文件到本地
Path localPath = new Path(localFilePath);
fs.copyToLocalFile(remotePath, localPath);
fs.close();
}
/**
* 移动文件到本地
* 移动后，删除源文件
*/
public static void moveToLocalFile(Configuration conf, String remoteFilePath, String localFilePath) throws IOException {
FileSystem fs = FileSystem.get(conf);
Path remotePath = new Path(remoteFilePath);
Path localPath = new Path(localFilePath);
fs.moveToLocalFile(remotePath, localPath);
}
}
