Centos6.8-hadoop-2.7.2 64 bit源码编译（伪分布-5大守护进程在本机上）

时间：2021-10-06 23:12:21
Centos6.8-hadoop-2.7.2 64 bit源码编译（伪分布-5大守护进程在本机上） 
1.安装插件

安装jdk/ant/libtool/protobuf/maven/zlib-devel/openssl-devel/ncurses-devel、gcc、snappy、bzip

#set java

export JAVA_HOME=/usr/app/jdk1.8.0

export JRE_HOME=/usr/app/jdk1.8.0/jre

export CLASS_PATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib

export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin

export JAVA_HOME JRE_HOME CLASS_PATH PATH

#set maven

export M2_HOME=/usr/app/maven

export PATH=$PATH:$M2_HOME/bin

#set ant

export ANT_HOME=/usr/app/ant

export PATH=$PATH:$ANT_HOME/bin

#set hadoop

export HADOOP_HOME=/usr/app/hadoop

export PATH=$PATH:$HADOOP_HOME/bin

2.配置etc/hosts如 192.168.66.66 xinfang

3.配置ssh无密登陆

配置前，先安装 SSH 服务。

ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa

cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys

#把 id_rsa.pub 追加到授权的 key 里面去、

chmod 600 ~/.ssh/authorized_keys #授权

vim /etc/ssh/sshd_config

RSAAuthentication yes # 启用 RSA 认证

PubkeyAuthentication yes # 启用公钥私钥配对认证方式

AuthorizedKeysFile .ssh/authorized_keys # 公钥文件路径（和上面生成

的文件同）

service sshd restart#重启 SSHD 服务

验证 ssh，# ssh ip 或 ssh localhost

4.源码编译hadoop 64bit

#mkdir /usr/app -p

#cd /usr/app

#wget http://mirror.bit.edu.cn/apache/hadoop/core/hadoop-2.7.2/hadoop-2.7.2-src.tar.gz

#tar zxvf hadoop-2.7.2-src.tar.gz

#cd hadoop-2.7.2-src

#mvn package -Pdist,native -DskipTests -Dtar  -Drequire.snappy

#保持联网状态，大约50多分钟（中途失败，查找原因，重新执行上一句命令）

#编译完成后，cd hadoop-dist/target将hadoop-2.7.2.tar.gz拷贝出来解压

#新建tmp/name/data目录  mkdir /usr/app/hadoop -p  mkdir /usr/app/hadoop/hdfs/name -p mkdir /usr/app/hadoop/hdfs/data

#配置hadoop/etc/hadoop/六个配置文件

#4.1 core-site.xml

<configuration>

<property>

<name>hadoop.tmp.dir</name>

<value>/usr/app/hadoop/tmp</value>

</property>

<property>

<name>fs.default.name</name>

<value>hdfs://xinfang:9000</value>

</property>

<property>

<name>io.compression.codecs</name>

<value>org.apache.hadoop.io.compress.GzipCodec,

org.apache.hadoop.io.compress.DefaultCodec,

org.apache.hadoop.io.compress.BZip2Codec,

org.apache.hadoop.io.compress.SnappyCodec

</value>

</property>
<property>

<name>hadoop.proxyuser.root.hosts</name>

<value>*</value>

</property>

<property>

<name>hadoop.proxyuser.root.groups</name>

<value>*</value>

</property>

</configuration>

#4.2 hdfs-site.xml

<configuration>

<property>

<name>dfs.namenode.name.dir</name>

<value>file:/usr/app/hadoop/hdfs/name</value>

</property>

<property>

<name>dfs.datanode.data.dir</name>

<value>file:/usr/app/hadoop/hdfs/data</value>

</property>

<property>

<name>dfs.replication</name>

<value>1</value>

</property>

</configuration>

#4.3 mapred-site.xml

<configuration>

<property>

<name>mapreduce.framework.name</name>

<value>yarn</value>

</property>

<property>

<name>mapreduce.map.output.compress</name>

<value>true</value>

</property>

<property>

<name>mapreduce.map.output.compress.codec</name>

<value>org.apache.hadoop.io.compress.SnappyCodec</value>

</property>

</configuration>

#4.4 yarn-site.xml

<configuration>

<property>

<name>yarn.nodemanager.aux-services</name>

<value>mapreduce_shuffle</value>

</property>

</configuration>

#4.5 hadoop-env.sh

export JAVA_HOME=/usr/app/jdk1.8.0

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_HOME/lib/native/

export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib:$HADOOP_HOME/lib/native"

#4.6 yarn-env.sh

export JAVA_HOME=/usr/app/jdk1.8.0

5.启动hadoop

#ssh 192.168.66.66

#hdfs namenode -format(不可多次格式化，容易造成datanode与namenode的集群ID不一致)

#sh /usr/app/hadoop/sbin/start-all.sh

#jps查看守护进程

NodeManager

SecondaryNameNode

DataNode

ResourceManager

NameNode

6.web访问

http://192.168.66.66:50070/ - Hadoop DFS 状态

http://192.168.66.66:8088/ -YARN 的 ResourceManager --集群

相关文章