测试准备
【首先同步时间：】
for node in CloudDeskTop master01 master02 slave01 slave02 slave03;do ssh $node "date -s '2017-12-30 21:32:30'";done
【slave各节点启动zookeeper集群：】
cd /software/zookeeper-3.4.10/bin/ && ./zkServer.sh start && cd - && jps
【master01启动HDFS集群：】
cd /software/ && start-dfs.sh && jps
【master01启动HBase：】
cd /software/hbase-1.2.6/bin && start-hbase.sh && jps
【master02上启动HBase:】
cd /software/hbase-1.2.6/bin && hbase-daemon.sh start master && jps
如有节点启动出现故障：
单独启动master：
cd /software/hbase-1.2.6/bin && hbase-daemon.sh start master && jps
单独启动regionserver：
cd /software/hbase-1.2.6/bin && hbase-daemon.sh start regionserver && jps
通过命令终端查看：
hbase(main):009:0> status
通过web终端查看：
http://master01的IP地址:16010/
【在主机CloudDeskTop导入java客户端开发所需jar包：】HBase1.2.6-All.zip
测试目标：
运用java代码编写程序操作HBase数据库，本次测试致力于对DML语法的java客户端操作；
package com.mmzs.bigdata.hbase.dml; import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.SubstringComparator; /**
* @author hadoop
*
* [HTable].put|delete|get|getScanner([Scan].addColumn|setStartRow|setStopRow|setFilter([FilterList].addFilter))
*
*/
public class DMLMain {
/**
* 操作HBase集群的客户端
*/
private static Admin admin; private static Connection conn; static{
//创建HBase配置
Configuration conf=HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum", "slave01:2181,slave02:2181,slave03:2181");
try {
//根据HBase配置获取HBase集群连接
conn=ConnectionFactory.createConnection(conf);
admin=conn.getAdmin();
} catch (IOException e) {
e.printStackTrace();
}
} public static void main(String[] args) {
//这种添加方式的读数据在后面查找时容易出IO异常，因为中间涉及了二进制转换
// put("mmzs:myuser","008","base","userName","ligang00");
// put("mmzs:myuser","008","base","userAge","18");
// put("mmzs:myuser","008","base","gender","woman"); /* Date d=new Date();
System.out.println(d);
byte[] b=getBytes(d);
System.out.println(b);
Object obj=getObject(b);
System.out.println(obj);*/ //添加数据
/* Map<String,Map<String,Map<String,Object>>> dataMap=new HashMap<String,Map<String,Map<String,Object>>>(); Map<String,Map<String,Object>> familyMap=new HashMap<String,Map<String,Object>>();
// dataMap.put("ligang+28+1.67", familyMap);
// dataMap.put("zhanghua+28+1.67", familyMap);
// dataMap.put("zhanghua+29+1.67", familyMap);
// dataMap.put("ligang+28+1.72", familyMap);
// dataMap.put("wangwu+29+1.82", familyMap);
dataMap.put("001", familyMap);
dataMap.put("002", familyMap);
dataMap.put("003", familyMap);
dataMap.put("004", familyMap);
dataMap.put("005", familyMap);
dataMap.put("006", familyMap); Map<String,Object> keyValMap=new HashMap<String,Object>();
keyValMap.put("height", 1.68);
keyValMap.put("weight", 75.5);
keyValMap.put("gender", "women");
keyValMap.put("username", "zhangsan"); familyMap.put("extra", keyValMap); puts("mmzs:myuser",dataMap);*/ // delete("mmzs:myuser","005");
// delete("mmzs:myuser","002","extra","username");
// deletes("mmzs:myuser","001","002","003","004","005","006","007","008"); // get("mmzs:myuser","005"); // scan("mmzs:myuser"); // scanByCondition("mmzs:myuser");
} /**
* 将二进制流读到内存通过反序列化重构生成对象
* @param object
* @return
*/
private static Object getObject(byte[] b){
ObjectInputStream ois=null;
try {
ByteArrayInputStream bais=new ByteArrayInputStream(b);
ois=new ObjectInputStream(bais);
return ois.readObject();
} catch (IOException e) {
e.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
}finally{
try{
if(null!=ois)ois.close();
}catch(IOException e){
e.printStackTrace();
}
}
return null;
} /**
* 将对象序列化成二进制数组
* @param object
* @return
*/
private static byte[] getBytes(Object object){
ObjectOutputStream oos=null;
try {
ByteArrayOutputStream baos=new ByteArrayOutputStream();
oos=new ObjectOutputStream(baos);
oos.writeObject(object);
oos.flush();
return baos.toByteArray();
} catch (IOException e) {
e.printStackTrace();
}finally{
try{
if(null!=oos)oos.close();
}catch(IOException e){
e.printStackTrace();
}
}
return null;
} /**
* 打印结果集
* @param ress
*/
private static void printResult(ResultScanner rss){
//遍历结果集,每一个Result对应一行记录(即对应一个RowKey)
for(Result res:rss){
String rowKey=new String(res.getRow());
List<Cell> cellList=res.listCells();
for(Cell cell:cellList){
//获取当前键值对所在的列族名称
String familyName=new String(CellUtil.cloneFamily(cell));
//获取当前键值对的键(Key)
String key=new String(CellUtil.cloneQualifier(cell));
//获取当前键值对的值(Value)
String value=getObject(CellUtil.cloneValue(cell)).toString(); System.out.println(rowKey+"\t"+familyName+"\t"+key+":"+value);
}
}
} /**
* 条件查询
* @param tabNameStr
*
* PrefixFilter和RowFilter都是基于行键(RowKey)的过滤器
*/
public static void scanByCondition(String tabNameStr){
TableName tabName=TableName.valueOf(tabNameStr);
Scan scan=new Scan(); //前缀过滤器,行键前缀是ligang的
PrefixFilter prefixFilter=new PrefixFilter("ligang".getBytes()); //子串过滤,行键中包含1.72的
RowFilter rowFilter=new RowFilter(CompareOp.EQUAL,new SubstringComparator("28")); //行键过滤器，列族 列名 比较操作 值,显示不包含满足条件的
SingleColumnValueFilter scvFilter=new SingleColumnValueFilter("base".getBytes(),"userName".getBytes(),CompareOp.EQUAL,new SubstringComparator("ligang")); //FilterList.Operator.MUST_PASS_ALL相当于and,FilterList.Operator.MUST_PASS_ONE相当于or
FilterList filterList=new FilterList(FilterList.Operator.MUST_PASS_ALL);
//添加使用的过滤器
// filterList.addFilter(prefixFilter);
// filterList.addFilter(rowFilter);
filterList.addFilter(scvFilter); // scan.setFilter(prefixFilter); scan.setFilter(filterList);//设置过滤
try {
Table table=conn.getTable(tabName);
ResultScanner ress=table.getScanner(scan);
printResult(ress);
} catch (IOException e) {
e.printStackTrace();
}
} /**
* 查询多条记录
* @param tabNameStr
* @param rowKey
*/
public static void scan(String tabNameStr){
TableName tabName=TableName.valueOf(tabNameStr); Scan scan=new Scan(); //过滤查询结果集中的字段
scan.addColumn("extra".getBytes(), "height".getBytes());
scan.addColumn("extra".getBytes(), "weight".getBytes()); //设置查询的起始和结束行索引(通过行键RowKey指定)
scan.setStartRow("002".getBytes());
scan.setStopRow("006".getBytes()); try {
Table table=conn.getTable(tabName);
//查询多行返回一个结果集
ResultScanner rss=table.getScanner(scan);
//遍历结果集,每一个Result对应一行记录(即对应一个RowKey)
printResult(rss);
} catch (IOException e) {
e.printStackTrace();
}
} /**
* 查询单条记录
* @param rowKey
* @param tabNameStr
*/
public static void get(String tabNameStr,String rowKey){
TableName tabName=TableName.valueOf(tabNameStr); Get get=new Get(rowKey.getBytes());
//相当于select..字段列表...from...,如果没有下面的addColumn方法调用则相当于select *...
get.addColumn("base".getBytes(), "height".getBytes());
get.addColumn("base".getBytes(), "gender".getBytes());
try {
Table table=conn.getTable(tabName);
Result result=table.get(get); //获取行键
String rowKeyStr=new String(result.getRow());
System.out.println("行键:"+rowKeyStr); //获取键所对应的值
byte[] byteName=result.getValue("base".getBytes(), "gender".getBytes());
String gender=getObject(byteName).toString();
System.out.println("gender:"+gender); //获取当前行中的所有键值对
List<Cell> cellList=result.listCells();
for(Cell cell:cellList){
//获取当前键值对所在的列族名称
String familyName=new String(CellUtil.cloneFamily(cell));
//获取当前键值对的键(Key)
String key=new String(CellUtil.cloneQualifier(cell));
//获取当前键值对的值(Value)
byte[] byteValue=CellUtil.cloneValue(cell);
String value=getObject(byteValue).toString(); System.out.println(rowKey+"\t"+familyName+"\t"+key+":"+value);
}
} catch (IOException e) {
e.printStackTrace();
}
} /**
* 批量删除多行
* @param tabNameStr
* @param rowKey
*/
public static void deletes(String tabNameStr,String... rowKeys){
if(rowKeys.length==0)return;
TableName tabName=TableName.valueOf(tabNameStr); List<Delete> deleteList=new ArrayList<Delete>();
for(String rowKey:rowKeys)deleteList.add(new Delete(rowKey.getBytes())); try {
Table table=conn.getTable(tabName);
table.delete(deleteList);
System.out.println("删除完成！");
} catch (IOException e) {
e.printStackTrace();
}
} /**
* 删除行中的键值对
* @param tabNameStr
* @param rowKey
* @param key
*/
public static void delete(String tabNameStr,String rowKey,String family,String key){
TableName tabName=TableName.valueOf(tabNameStr);
try {
Table table=conn.getTable(tabName);
Delete delete=new Delete(rowKey.getBytes());
delete.addColumn(family.getBytes(), key.getBytes());
table.delete(delete);
System.out.println("删除完成！");
} catch (IOException e) {
e.printStackTrace();
}
} /**
* 删除整行
* @param tabNameStr
* @param rowKey
*/
public static void delete(String tabNameStr,String rowKey){
TableName tabName=TableName.valueOf(tabNameStr);
try {
Table table=conn.getTable(tabName);
Delete delete=new Delete(rowKey.getBytes());
table.delete(delete);
System.out.println("删除完成！");
} catch (IOException e) {
e.printStackTrace();
}
} /**
* 增加或修改数据(表名、行键、列族、列、值)
*/
public static void put(String tabNameStr,String rowKey,String family,String key,String value){
TableName tabName=TableName.valueOf(tabNameStr);
try {
Table table=conn.getTable(tabName);
Put put=new Put(rowKey.getBytes());
put.addColumn(family.getBytes(), key.getBytes(), value.getBytes());
table.put(put);
System.out.println("操作完成！");
} catch (IOException e) {
e.printStackTrace();
}
} /**
* 批量插入或修改数据
* @param tabNameStr
* @param dataMap
*/
public static void puts(String tabNameStr,Map<String,Map<String,Map<String,Object>>> dataMap){
List<Put> putList=new ArrayList<Put>();
Set<Entry<String, Map<String, Map<String, Object>>>> entrys=dataMap.entrySet();
for(Entry<String, Map<String, Map<String, Object>>> entry:entrys){
//获取行的rowKey
byte[] rowKey=entry.getKey().getBytes();
Put put=new Put(rowKey);
putList.add(put);
//获取行的所有列族
Map<String, Map<String, Object>> familyMap=entry.getValue();
Set<Entry<String, Map<String, Object>>> familyEntrys=familyMap.entrySet();
for(Entry<String, Map<String, Object>> familyEntry:familyEntrys){
//获取列族名称
byte[] familyName=familyEntry.getKey().getBytes();
//获取列族下左右的键值对
Map<String, Object> keyVals=familyEntry.getValue();
Set<Entry<String, Object>> keyValEntrys=keyVals.entrySet();
for(Entry<String, Object> keyValEntry:keyValEntrys){
byte[] key=keyValEntry.getKey().getBytes();
byte[] value=getBytes(keyValEntry.getValue());
put.addColumn(familyName, key, value);
}
}
} TableName tabName=TableName.valueOf(tabNameStr);
try {
Table table=conn.getTable(tabName);
table.put(putList);
System.out.println("操作完成！");
} catch (IOException e) {
e.printStackTrace();
}
}
}
DMLMain
测试结果：
在命令端查看，查看方式，可参考：http://www.cnblogs.com/mmzs/p/8135327.html
测试小结：
HBase是分布式的、可扩展的、且基于Hadop的大数据NOSQL存储数据库(号称十亿行、百万列),该数据库的每一行也是保存一条记录,但是与关系数据库不同的是,该数据库无需事先定义字段和表结构,只需要指定表名和列族;每一行由指定的一到多个族组成,每个族上可以保存若干个键值对(每一行的每个族上的键值对数量可以是任意的),即每一行的单元格数量不一定相等,每一个单元格中保存同一个键名但不同版本值的若干个键值对(即同一个字段名中有不同版本的字段值),当需要查询某一个字段值时需要指定的坐标是:
表名(table name)—>行健(row key)—>列族(column family)—>字段名(column name)—>版本号(version code)
