hadoop生态搭建（3节点）-04.hadoop配置

时间：2023-01-08 08:46:51

如果之前没有安装jdk和zookeeper，安装了的请直接跳过

# https://www.oracle.com/technetwork/java/javase/downloads/java-archive-javase8-2177648.html

# ==================================================================安装 jdk

mkdir -p /usr/java

tar -zxvf ~/jdk-8u111-linux-x64.tar.gz -C /usr/java

rm -r ~/jdk-8u111-linux-x64.tar.gz

# http://archive.apache.org/dist/zookeeper/

# ==================================================================安装 zookeeper
# zookeeper集群搭建要至少3台服务器，服务器都要部署zookeeper

tar -zxvf ~/zookeeper-3.4.12.tar.gz -C /usr/local

rm -r ~/zookeeper-3.4.12.tar.gz

# http://archive.apache.org/dist/hadoop/core/hadoop-2.7.6/

# ==================================================================安装 hadoop

tar -zxvf ~/hadoop-2.7.6.tar.gz -C /usr/local

rm -r ~/hadoop-2.7.6.tar.gz

# 配置环境变量

# ==================================================================node1 node2 node3

vi /etc/profile

# 在export PATH USER LOGNAME MAIL HOSTNAME HISTSIZE HISTCONTROL下添加

export JAVA_HOME=/usr/java/jdk1.8.0_111

export ZOOKEEPER_HOME=/usr/local/zookeeper-3.4.12

export HADOOP_HOME=/usr/local/hadoop-2.7.6

export PATH=$PATH:$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$ZOOKEEPER_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

export HADOOP_INSTALL=$HADOOP_HOME

export HADOOP_MAPRED_HOME=$HADOOP_HOME

export HADOOP_COMMON_HOME=$HADOOP_HOME

export HADOOP_HDFS_HOME=$HADOOP_HOME

export YARN_HOME=$HADOOP_HOME

export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native

# ==================================================================node1

# 使环境变量生效

source /etc/profile

# 查看配置结果

echo $HADOOP_HOME

# hadoop 配置 hadoop-env.sh

vi $HADOOP_HOME/etc/hadoop/hadoop-env.sh

export JAVA_HOME=/usr/java/jdk1.8.0_111

#export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib:$HADOOP_HOME/lib/native"

export HADOOP_OPTS="-Djava.library.path=/usr/local/hadoop-2.7.6/lib:/usr/local/hadoop-2.7.6/lib/native"

export HADOOP_PID_DIR=/usr/local/hadoop-2.7.6/pids

# export HADOOP_OPTS="$HADOOP_OPTS -Duser.timezone=GMT+08"

# hadoop 配置 mapred-env.sh

vi $HADOOP_HOME/etc/hadoop/mapred-env.sh

export JAVA_HOME=/usr/java/jdk1.8.0_111

export HADOOP_MAPRED_PID_DIR=/usr/local/hadoop-2.7.6/pids

# hadoop 配置 yarn-env.sh 修改 yarn-env.sh 中 JAVA_HEAP_MAX=-Xmx3072m 改为3G

vi $HADOOP_HOME/etc/hadoop/yarn-env.sh

export JAVA_HOME=/usr/java/jdk1.8.0_111

JAVA_HEAP_MAX=3G

# YARN_OPTS="$YARN_OPTS -Duser.timezone=GMT+08"

vi $HADOOP_HOME/etc/hadoop/slaves

node1

node2

node3

# 配置文件中对应需创建的文件夹

mkdir $HADOOP_HOME/tmp

mkdir $HADOOP_HOME/dfs

mkdir $HADOOP_HOME/dfs/name

mkdir $HADOOP_HOME/dfs/data

mkdir $HADOOP_HOME/logs

mkdir $HADOOP_HOME/journal

mkdir $HADOOP_HOME/journal/data

mkdir $HADOOP_HOME/yarn

mkdir $HADOOP_HOME/yarn/local

mkdir $HADOOP_HOME/yarn/logs

# core-site.xml

vi $HADOOP_HOME/etc/hadoop/core-site.xml

<configuration>

    <!-- 指定hdfs的nameservice为cluster -->

    <property>

        <name>fs.defaultFS</name>

        <value>hdfs://appcluster</value>

    </property>

    <!-- 指定zookeeper地址-->

    <property>

        <name>ha.zookeeper.quorum</name>

        <value>node1:2181,node2:2181,node3:2181</value>

    </property>

    <property>

        <name>ha.zookeeper.session-timeout.ms</name>

        <value>2000</value>

    </property>

    <!-- 故障检查时间 -->

    <property>

        <name>ha.failover-controller.cli-check.rpc-timeout.ms</name>

        <value>60000</value>

    </property>

    <!-- ipc通讯超时时间 -->

    <property>

        <name>ipc.client.connect.timeout</name>

        <value>20000</value>

    </property>

    <!-- 指定hadoop临时目录 -->

    <property>

        <name>hadoop.tmp.dir</name>

        <value>/usr/local/hadoop-2.7.6/tmp</value>

    </property>

    <property>

        <name>hadoop.security.authorization</name>

        <value>true</value>

    </property>

    <property>

        <name>hadoop.security.authentication</name>

        <value>simple</value>

    </property>

    <property>

        <name>hadoop.proxyuser.super.hosts</name>

        <value>*</value>

    </property>

    <property>

        <name>hadoop.proxyuser.super.groups</name>

        <value>*</value>

    </property>

    <property>

        <name>hadoop.proxyuser.root.groups</name>

        <value>*</value>

    </property>

    <property>

        <name>hadoop.proxyuser.root.hosts</name>

        <value>*</value>

    </property>

    <property>

        <name>hadoop.native.lib</name>

        <value>true</value>

        <description>Should native hadoop libraries, if present, be used.</description>

    </property>

</configuration>

# hdfs-site.xml

vi $HADOOP_HOME/etc/hadoop/hdfs-site.xml

<configuration>

    <!-- 指定namenode元数据存放路径；如果机器上有多块硬盘，推荐配置多个路径用逗号分隔 -->

    <property>

        <name>dfs.namenode.name.dir</name>

        <value>/usr/local/hadoop-2.7.6/dfs/name</value>

    </property>

    <!-- 指定datanode数据存储地址 -->

    <property>

        <name>dfs.datanode.data.dir</name>

        <value>/usr/local/hadoop-2.7.6/dfs/data</value>

    </property>

    <!-- 指定数据冗余份数，默认3份 -->

    <property>

        <name>dfs.replication</name>

        <value>2</value>

    </property>

    <!--指定hdfs的nameservice为cluster，需要和core-site.xml中的保持一致 -->

    <property>

        <name>dfs.nameservices</name>

        <value>appcluster</value>

    </property>

    <!-- cluster下面有两个NameNode，分别是nn1，nn2 -->

    <property>

        <name>dfs.ha.namenodes.appcluster</name>

        <value>nn1,nn2</value>

    </property>

    <!-- nn1、nn2的RPC通信地址 -->

    <property>

        <name>dfs.namenode.rpc-address.appcluster.nn1</name>

        <value>node1:8020</value>

    </property>

    <property>

        <name>dfs.namenode.rpc-address.appcluster.nn2</name>

        <value>node2:8020</value>

    </property>

    <!-- nn1、nn2的http通信地址 -->

    <property>

        <name>dfs.namenode.http-address.appcluster.nn1</name>

        <value>node1:50070</value>

    </property>

    <property>

        <name>dfs.namenode.http-address.appcluster.nn2</name>

        <value>node2:50070</value>

    </property>

    <property>

        <name>dfs.namenode.servicerpc-address.appcluster.nn1</name>

        <value>node1:53310</value>

    </property>

    <property>

        <name>dfs.namenode.servicerpc-address.appcluster.nn2</name>

        <value>node2:53310</value>

    </property>

    <!-- 指定NameNode的元数据在JournalNode上的存放位置 -->

    <property>

        <name>dfs.namenode.shared.edits.dir</name>

        <value>qjournal://node1:8485;node2:8485;node3:8485/appcluster</value>

    </property>

    <!-- 指定JournalNode在本地磁盘存放数据的位置 -->

    <property>

        <name>dfs.journalnode.edits.dir</name>

        <value>/usr/local/hadoop-2.7.6/journal/data</value>

    </property>

    <!-- 开启NameNode失败自动切换 -->

    <property>

        <name>dfs.ha.automatic-failover.enabled.appcluster</name>

        <value>true</value>

    </property>

    <!-- 配置失败自动切换实现方式 -->

    <property>

        <name>dfs.client.failover.proxy.provider.appcluster</name>

        <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>

    </property>

    <!-- 配置隔离机制方法，多个机制用换行分割，即每个机制暂用一行 -->

    <property>

        <name>dfs.ha.fencing.methods</name>

        <value>

        sshfence

        shell(/bin/true)

        </value>

    </property>

    <property>

        <name>dfs.ha.fencing.ssh.private-key-files</name>

        <value>/root/.ssh/id_rsa</value>

    </property>

    <property>

        <name>dfs.ha.fencing.ssh.connect-timeout</name>

        <value>30000</value>

    </property>

    <property>

        <name>dfs.webhdfs.enabled</name>

        <value>true</value>

    </property>

    <property>

        <name>dfs.permissions.enable</name>

        <value>false</value>

    </property>

    <property>

        <name>dfs.permissions</name>

        <value>false</value>

    </property>

    <!-- 提供web访问hdfs的权限 -->

    <property>

        <name>dfs.webhdfs.enabled</name>

        <value>true</value>

    </property>

</configuration>

# mapred-site.xml

cp $HADOOP_HOME/etc/hadoop/mapred-site.xml.template $HADOOP_HOME/etc/hadoop/mapred-site.xml
vi $HADOOP_HOME/etc/hadoop/mapred-site.xml

<configuration>

    <property>

        <name>mapreduce.framework.name</name>

        <value>yarn</value>

    </property>

</configuration>

# yarn-site.xml

vi $HADOOP_HOME/etc/hadoop/yarn-site.xml

<configuration>

    <property>

        <name>dfs.ha.automatic-failover.enabled.appcluster</name>

        <value>true</value>

    </property>

    <!--rm失联后重新链接的时间-->

    <property>

        <name>yarn.resourcemanager.connect.retry-interval.ms</name>

        <value>2000</value>

    </property>

    <!--开启resourcemanagerHA,默认为false-->

    <property>

        <name>yarn.resourcemanager.ha.enabled</name>

        <value>true</value>

    </property>

    <!--配置resourcemanager-->

    <property>

        <name>yarn.resourcemanager.ha.rm-ids</name>

        <value>rm1,rm2</value>

    </property>

    <property>

        <name>ha.zookeeper.quorum</name>

        <value>node1:2181,node2:2181,node3:2181</value>

    </property>

    <!--开启故障自动切换-->

    <property>

        <name>yarn.resourcemanager.ha.automatic-failover.enabled</name>

        <value>true</value>

    </property>

    <property>

        <name>yarn.resourcemanager.hostname.rm1</name>

        <value>node1</value>

    </property>

    <property>

        <name>yarn.resourcemanager.hostname.rm2</name>

        <value>node2</value>

    </property>

    <!--

    在node1上配置rm1,在node2上配置rm2,

    注意：一般都喜欢把配置好的文件远程复制到其它机器上，但这个在YARN的另一个机器上一定要修改

    -->

    <property>

        <name>yarn.resourcemanager.ha.id</name>

        <value>rm1</value>

        <description>If we want to launch more than one RM in single node,we need this configuration</description>

    </property>

    <!--开启自动恢复功能-->

    <property>

        <name>yarn.resourcemanager.recovery.enabled</name>

        <value>true</value>

    </property>

    <!--配置与zookeeper的连接地址-->

    <property>

        <name>yarn.resourcemanager.zk-state-store.address</name>

        <value>node1:2181,node2:2181,node3:2181</value>

    </property>

    <property>

        <name>yarn.resourcemanager.store.class</name>

        <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>

    </property>

    <property>

        <name>yarn.resourcemanager.zk-address</name>

        <value>node1:2181,node2:2181,node3:2181</value>

    </property>

    <property>

        <name>yarn.resourcemanager.cluster-id</name>

        <value>appcluster-yarn</value>

    </property>

    <!--schelduler失联等待连接时间-->

    <property>

        <name>yarn.app.mapreduce.am.scheduler.connection.wait.interval-ms</name>

        <value>5000</value>

    </property>

    <!--配置rm1,rm2-->

    <property>

        <name>yarn.resourcemanager.address.rm1</name>

        <value>node1:8032</value>

    </property>

    <property>

        <name>yarn.resourcemanager.address.rm2</name>

        <value>node2:8032</value>

    </property>

    <property>

        <name>yarn.resourcemanager.scheduler.address.rm1</name>

        <value>node1:8030</value>

    </property>

    <property>

        <name>yarn.resourcemanager.scheduler.address.rm2</name>

        <value>node2:8030</value>

    </property>

    <property>

        <name>yarn.resourcemanager.webapp.address.rm1</name>

        <value>node1:8088</value>

    </property>

    <property>

        <name>yarn.resourcemanager.webapp.address.rm2</name>

        <value>node2:8088</value>

    </property>

    <property>

        <name>yarn.resourcemanager.resource-tracker.address.rm1</name>

        <value>node1:8031</value>

    </property>

    <property>

        <name>yarn.resourcemanager.resource-tracker.address.rm2</name>

        <value>node2:8031</value>

    </property>

    <property>

        <name>yarn.resourcemanager.admin.address.rm1</name>

        <value>node1:8033</value>

    </property>

    <property>

        <name>yarn.resourcemanager.admin.address.rm2</name>

        <value>node2:8033</value>

    </property>

    <property>

        <name>yarn.resourcemanager.ha.admin.address.rm1</name>

        <value>node1:23142</value>

    </property>

    <property>

        <name>yarn.resourcemanager.ha.admin.address.rm2</name>

        <value>node2:23142</value>

    </property>

    <property>

        <name>yarn.nodemanager.aux-services</name>

        <value>mapreduce_shuffle</value>

    </property>

    <property>

        <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>

        <value>org.apache.hadoop.mapred.ShuffleHandler</value>

    </property>

    <property>

        <name>yarn.nodemanager.local-dirs</name>

        <value>/usr/local/hadoop-2.7.6/yarn/local</value>

    </property>

    <property>

        <name>yarn.nodemanager.log-dirs</name>

        <value>/usr/local/hadoop-2.7.6/yarn/logs</value>

    </property>

    <property>

        <name>mapreduce.shuffle.port</name>

        <value>23080</value>

    </property>

    <!--故障处理类-->

    <property>

        <name>yarn.client.failover-proxy-provider</name>

        <value>org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider</value>

    </property>

    <property>

        <name>yarn.resourcemanager.ha.automatic-failover.zk-base-path</name>

        <value>/yarn-leader-election</value>

        <description>Optionalsetting.Thedefaultvalueis/yarn-leader-election</description>

    </property>

    <property>

        <name>yarn.nodemanager.resource.memory-mb</name>

        <value>2048</value>

    </property>

    <property>

        <name>yarn.scheduler.maximum-allocation-mb</name>

        <value>2048</value>

    </property>

    <property>

        <name>yarn.nodemanager.resource.cpu-vcores</name>

        <value>1</value>

    </property>

</configuration>

# 修改日志的路径

vi $HADOOP_HOME/etc/hadoop/log4j.properties

hadoop.log.dir=/usr/local/hadoop-2.7.6/logs

# ==================================================================node1

scp -r $HADOOP_HOME node2:/usr/local/

scp -r $HADOOP_HOME node3:/usr/local/

# ==================================================================node2 node3

# 使环境变量生效

source /etc/profile

# 查看配置结果

echo $HADOOP_HOME

# ==================================================================node2

vi $HADOOP_HOME/etc/hadoop/yarn-site.xml

<property>

    <name>yarn.resourcemanager.ha.id</name>

    <value>rm2</value>

</property>

shutdown -h now
# 快照 hadoop集群前

# ==================================================================初次启动

# 1.启动zookeeper集群所有节点

# ==================================================================node1 node2 node3

zkServer.sh start

zkServer.sh status

# zkServer.sh stop

# $ZOOKEEPER_HOME/bin/zkServer.sh start

# $ZOOKEEPER_HOME/bin/zkServer.sh status

# $ZOOKEEPER_HOME/bin/zkServer.sh stop

# 2.格式化Zookeeper文件系统在active的namenode上运行

# ==================================================================node1

$HADOOP_HOME/bin/hdfs zkfc -formatZK

# 验证：$ZOOKEEPER_HOME/bin/zkCli.sh

# ls /

# ls /hadoop-ha

# 3.启动JournalNode集群

# ==================================================================node1 node2 node3

$HADOOP_HOME/sbin/hadoop-daemon.sh start journalnode

# jps

# QuorumPeerMain

# Jps

# JournalNode

# 4.格式化集群的 NameNode

# ==================================================================node1

$HADOOP_HOME/bin/hdfs namenode -format

# 如果不是首次format的话还是把NameNode和DataNode存放数据地址下的数据手动删除一下，否则会造成NameNode ID和DataNode ID不一致，

# rm -rf $HADOOP_HOME/dfs/name/* & rm -rf $HADOOP_HOME/dfs/data/*

# $HADOOP_HOME/bin/hdfs namenode -format

# （如果是HDFS联盟，即有多个HDFS集群同时工作，则用hdfs namenode -format -clusterId [clusterID]）

# 5.启动 NameNode

# ==================================================================node1

$HADOOP_HOME/sbin/hadoop-daemon.sh start namenode

$HADOOP_HOME/sbin/yarn-daemon.sh start resourcemanager

# 6.把NameNode的数据从node1同步到node2中,在standby的namenode格式化namenode,同步NameNode的数据

# ==================================================================node2

$HADOOP_HOME/bin/hdfs namenode -bootstrapStandby

$HADOOP_HOME/sbin/hadoop-daemon.sh start namenode

$HADOOP_HOME/sbin/yarn-daemon.sh start resourcemanager

# 7.启动所有的DataNode

# ==================================================================node1

$HADOOP_HOME/sbin/hadoop-daemons.sh start datanode

# 8.启动Yarn

# ==================================================================node1

$HADOOP_HOME/sbin/start-yarn.sh

# 9.在node1,node2启动ZooKeeperFailoverController(这里不用在node3中启动，因为node3这个节点是纯粹的DataNode)

# ==================================================================node1 node2

$HADOOP_HOME/sbin/hadoop-daemon.sh start zkfc

# 10.验证HA的故障自动转移是否好用

# kill 进程号

# $HADOOP_HOME/sbin/hadoop-daemon.sh start namenode

# 启动后在各个节点查看进程运行情况

# node1

jps

#2850 NodeManager

#2963 DFSZKFailoverController

#3027 Jps

#2374 NameNode

#2679 ResourceManager

#2523 DataNode

#2126 QuorumPeerMain

#2255 JournalNode

# node2

jps

#2209 JournalNode

#2674 DataNode

#2850 NodeManager

#2134 QuorumPeerMain

#2424 ResourceManager

#2973 DFSZKFailoverController

#3021 Jps

#2318 NameNode

# node3

jps

#2546 Jps

#2131 QuorumPeerMain

#2278 DataNode

#2200 JournalNode

#2446 NodeManager

# http://node1:50070/dfshealth.html#tab-overview

# http://node2:50070/dfshealth.html#tab-overview

# http://node1:8088/cluster/nodes

# ==================================================================停止集群

# ==================================================================node1

# stop已经启动的进程

$HADOOP_HOME/sbin/stop-all.sh

# ==================================================================node1 node2 node3

# 停止 zookeeper

zkServer.sh stop

# ==================================================================node1

$HADOOP_HOME/sbin/hadoop-daemon.sh stop zkfc

# ==================================================================node2

$HADOOP_HOME/sbin/yarn-daemon.sh stop resourcemanager

$HADOOP_HOME/sbin/hadoop-daemon.sh stop zkfc

# ==================================================================再次顺序启动方式

# ==================================================================node1 node2 node3

# 启动 zookeeper

zkServer.sh start

zkServer.sh status

# ==================================================================node1

# 启动hadoop所有进程

$HADOOP_HOME/sbin/start-all.sh

$HADOOP_HOME/sbin/hadoop-daemon.sh start zkfc

# ==================================================================node2

$HADOOP_HOME/sbin/yarn-daemon.sh start resourcemanager

$HADOOP_HOME/sbin/hadoop-daemon.sh start zkfc

hadoop dfsadmin -safemode get

# 命令强制离开

# hadoop dfsadmin -safemode leave

# ==================================================================停止集群

# ==================================================================node1

# stop已经启动的进程

$HADOOP_HOME/sbin/stop-all.sh

# ==================================================================node1 node2 node3

# 停止 zookeeper

zkServer.sh stop

# ==================================================================node1

$HADOOP_HOME/sbin/hadoop-daemon.sh stop zkfc

# ==================================================================node2

$HADOOP_HOME/sbin/yarn-daemon.sh stop resourcemanager

$HADOOP_HOME/sbin/hadoop-daemon.sh stop zkfc

shutdown -h now

# 快照 hadoop集群

# 先不急着进行simple 认证

# ==================================================================simple 认证
vi $HADOOP_HOME/etc/hadoop/core-site.xml

<property>

    <name>hadoop.http.filter.initializers</name>

    <value>org.apache.hadoop.security.AuthenticationFilterInitializer</value>

</property>

<property>

    <name>hadoop.http.authentication.type</name>

    <value>simple</value>

</property>

<property>

    <name>hadoop.http.authentication.token.validity</name>

    <value>3600</value>

</property>

<property>

    <name>hadoop.http.authentication.signature.secret.file</name>

    <value>/usr/local/hadoop-2.7.6/hadoop-http-auth-signature-secret</value>

</property>

<property>

    <name>hadoop.http.authentication.cookie.domain</name>

    <value></value>

</property>

<property>

    <name>hadoop.http.authentication.simple.anonymous.allowed</name>

    <value>false</value>

</property>
# 在上述配置的目录 $HADOOP_HOME 下生成文件hadoop-http-auth-signature-secret

echo "hadoop" > $HADOOP_HOME/hadoop-http-auth-signature-secret

scp -r $HADOOP_HOME/etc/hadoop/core-site.xml node2:$HADOOP_HOME/etc/hadoop/core-site.xml

scp -r $HADOOP_HOME/etc/hadoop/core-site.xml node3:$HADOOP_HOME/etc/hadoop/core-site.xml

scp -r $HADOOP_HOME/hadoop-http-auth-signature-secret node2:$HADOOP_HOME/hadoop-http-auth-signature-secret

scp -r $HADOOP_HOME/hadoop-http-auth-signature-secret node3:$HADOOP_HOME/hadoop-http-auth-signature-secret

# ==================================================================再次顺序启动方式

# ==================================================================node1 node2 node3

# 启动 zookeeper

zkServer.sh start

zkServer.sh status

# ==================================================================node1

# 启动hadoop所有进程

$HADOOP_HOME/sbin/start-all.sh

$HADOOP_HOME/sbin/hadoop-daemon.sh start zkfc

# ==================================================================node2

$HADOOP_HOME/sbin/yarn-daemon.sh start resourcemanager

$HADOOP_HOME/sbin/hadoop-daemon.sh start zkfc

hadoop dfsadmin -safemode get

# 命令强制离开
# hadoop dfsadmin -safemode leave

# 网页访问
# http://node1:50070?user.name=hadoop
# http://node2:50070?user.name=hadoop
# http://node1:8088?user.name=hadoop/cluster/nodes

shutdown -h now
# 快照 hadoop_simple认证

# ==================================================================Kerberos 认证

# 后续整理好后开放

相关文章