write & read a sequence file
write & read a sequence file
import java.io.IOException;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.Writer;
import org.apache.hadoop.io.SequenceFile.Reader;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.Configuration;
public class MySequenceFile {
static private final String[] DATA = {
"this is the first",
"this is the second",
"this is the third",
"this is the forth"
};
public static void main(String[] args) throws IOException {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Path path = new Path(args[0]);
IntWritable key = new IntWritable();
Text value = new Text();
SequenceFile.Writer writer = null;
writer = SequenceFile.createWriter(conf, Writer.file(path), Writer.keyClass(key.getClass()), Writer.valueClass(value.getClass()));
for( int i = 0; i < 1000; i++ ) {
key.set(i + 1);
value.set(DATA[i % DATA.length]);
writer.append(key,value);
}
writer.close();
SequenceFile.Reader reader = new SequenceFile.Reader(conf, Reader.file(path));
while( reader.next(key, value) ) {
String syncSeen = reader.syncSeen() ? "*" : "#";
System.err.println(key + "\t" + value + "\t" + reader.getPosition()+ "\t" + syncSeen);
}
reader.close();
}
}