Hadoop记录-安装ambari hdp集群

时间：2022-09-19 02:25:06
#!/bin/sh

#配置用户sudo权限（参考/etc/sudoers文件，在/etc/sudoers.d/新建一个用户配置文件，注意要注销）

#需要在/etc/sudoers末尾追加：sfapp ALL=(ALL) NOPASSWD:ALL

#JDK配置,三台机器已配置，略

#修改hostname和加入主机映射（/etc/hosts）

#配置网络/etc/sysconfig/network

#配置无密登录

#ssh免秘钥登录需在sfapp用户下完成；

#在10.202.107.27 cnsz22VLK5444执行

#ssh-keygen #回车

#ssh-copy-id -i /home/sfapp/.ssh/id_rsa.pub sfapp@10.202.107.28 #拷贝公钥

#ssh-copy-id -i /home/sfapp/.ssh/id_rsa.pub sfapp@10.202.107.211 #拷贝公钥

#ssh sfapp@10.202.107.28 #测试登录

#ssh sfapp@10.202.107.211 #测试登录

#ssh-keygen -t rsa

#ssh-copy-id slave1

#ssh-copy-id slave2

#ssh-copy-id master

#在 master机器生成公钥文件

#ssh-keygen -t rsa

#cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

#chmod 600 ~/.ssh/authorized_keys

#拷贝到其他主机上

#scp ~/.ssh/authorized_keys root@slave1:~/.ssh/

#scp ~/.ssh/authorized_keys root@slave2:~/.ssh/

#ssh cnsz22VLK5444 date

#ssh cnsz22VLK5445 date

#ssh cnsz22VLK7714 date

#修改文件打开限制（所有机器）

#vim /etc/security/limits.conf

# End of file

#* soft nofile 65536

#* hard nofile 65536

#* soft nproc 131072

#* hard nproc 131072

#sysctl -p

#关闭防火墙和selinux

#ssh sfapp@10.202.107.27 "sudo systemctl stop firewalld;sudo systemctl disable firewalld"

#ssh sfapp@10.202.107.27 "sudo setenforce 0;sudo sed -ri '/^SELINUX/cSELINUX=disabled' /etc/selinux/config"

#ssh sfapp@10.202.107.28 "sudo systemctl stop firewalld;sudo systemctl disable firewalld"

#ssh sfapp@10.202.107.28 "sudo setenforce 0;sudo sed -ri '/^SELINUX/cSELINUX=disabled' /etc/selinux/config"

#ssh sfapp@10.202.107.211 "sudo systemctl stop firewalld;sudo systemctl disable firewalld"

#ssh sfapp@10.202.107.211 "sudo setenforce 0;sudo sed -ri '/^SELINUX/cSELINUX=disabled' /etc/selinux/config"

#配置ntp时钟(master做时钟服务器，其他slave同步时钟)

#sudo yum -y install ntp

#vim /etc/ntp.conf

#restrict 10.202.0.0 mask 255.255.255.0

#注释以下内容

#server 0.centos.pool.ntp.org iburst

#server 1.centos.pool.ntp.org iburst

#server 2.centos.pool.ntp.org iburst

#server 3.centos.pool.ntp.org iburst

#server 127.127.1.0

#fudge 127.127.1.0 stratum 10

#systemctl start ntpd.service #启动服务

#systemctl enable ntpd.service #开机自启动

#从节点同步主节点的十种

#crontab -e

#0-59/10 * * * * /usr/sbin/ntpdate 10.202.107.27

#crontab -l

#0-59/10 * * * * /usr/sbin/ntpdate 10.202.107.27

#其他系统设置

#sudo sh -c "echo umask 0022 >> /etc/profile"

#yum -y install httpd

#vim /etc/httpd/conf/httpd.conf 修改端口80为81，修改DocumentRoot 为"/app/ambari"

#sudo systemctl restart httpd.service

#chown -R sfapp. /app/ambari 设置sfapp所有

#浏览器测试登录

#下载文件

#HDP：http://public-repo-1.hortonworks.com/HDP/centos7/2.x/updates/2.6.3.0/HDP-2.6.3.0-centos7-rpm.tar.gz

#ambari：http://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/2.6.0.0/ambari-2.6.0.0-centos7.tar.gz

#HDP-UTILS：http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos7/HDP-UTILS-1.1.0.21-centos7.tar.gz

#上传文件ambari-2.6.0.0-centos7.tar.gz HDP-2.6.3.0-centos7-rpm.tar.gz HDP-UTILS-1.1.0.21-centos7.tar.gz

#tar -zxvf ambari-2.6.0.0-centos7.tar.gz

#tar -zxvf HDP-2.6.3.0-centos7-rpm.tar.gz

#mkdir HDP-UTILS

#tar -zxvf HDP-UTILS-1.1.0.21-centos7.tar.gz -C HDP-UTILS

#浏览器测试是否能访问到文件

#制作本地源

#yum install yum-utils createrepo yum-plugin-priorities -y

#createrepo ./

##ambari.repo

#[ambari-2.6.0.0]

#name=ambari Version - ambari-2.6.0.0

#baseurl=http://10.202.107.27:81/ambari/centos7/2.6.0.0-267

#gpgcheck=0

#gpgkey=http://10.202.107.27:81/ambari/centos7/2.6.0.0-267/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins

#enabled=1

#priority=1

##hdp.repo

#[HDP-2.6.3.0]

#name=HDP Version - HDP-2.6.3.0

#baseurl=http://10.202.107.27:81/HDP/centos7/2.6.3.0-235

#gpgcheck=1

#gpgkey=http://10.202.107.27:81/HDP/centos7/2.6.3.0-235/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins

#enabled=1

#priority=1

#[HDP-UTILS-1.1.0.21]

#name=HDP-UTILS Version - HDP-UTILS-1.1.0.21

#baseurl=http://10.202.107.27:81/HDP-UTILS

#gpgcheck=1

#gpgkey=http://10.202.107.27:81/HDP-UTILS/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins

#enabled=1

#priority=1

##cp ambari.repo hdp.repo /etc/yum.repos.d/

##scp ambari.repo hdp.repo slave1:/etc/yum.repos.d/

#yum clean all

#yum makecache

#yum repolist

#安装和配置ambari-server

#yum -y install ambari-server

#ambari-server setup

#n-3-n

#ambari-server start

#浏览器测试：ip:8080 admin/admin

##安装mysql

#yum -y install mariadb mariadb-server

#systemctl start mariadb

#systemctl enable mariadb

#mysql_secure_installation

#Enter current password for root (enter for none):<–初次运行直接回车

#Set root password? [Y/n] <– 是否设置root用户密码，输入y并回车或直接回车

#New password: <– 设置root用户的密码

#Re-enter new password: <– 再输入一次你设置的密码

#Remove anonymous users? [Y/n] <– 是否删除匿名用户，回车

#Disallow root login remotely? [Y/n] <–是否禁止root远程登录,回车,

#Remove test database and access to it? [Y/n] <– 是否删除test数据库，回车

#Reload privilege tables now? [Y/n] <– 是否重新加载权限表，回车

#mysql -uroot -ppassword

#vim /etc/my.cnf

#在[mysqld]标签下添加

init_connect='SET collation_connection = utf8_unicode_ci' init_connect='SET NAMES utf8' character-set-server=utf8

collation-server=utf8_unicode_ci

skip-character-set-client-handshake

vim /etc/my.cnf.d/client.cnf

#在[client]中添加

default-character-set=utf8

vim /etc/my.cnf.d/mysql-clients.cnf

#在[mysql]中添加

default-character-set=utf8

#全部配置完成，重启mariadb

systemctl restart mariadb

#之后进入MariaDB查看字符集

mysql> show variables like "%character%";show variables like "%collation%";

mysql>create user username@localhost identified by 'password';

mysql>grant all on *.* to username@localhost indentified by 'password';

mysql>grant all privileges on *.* to username@'%' identified by 'password';

mysql>grant all privileges on *.* to username@'hostname' identified by 'password' with grant option;

#其中只授予部分权限把 其中 all privileges或者all改为select,insert,update,delete,create,drop,index,alter,grant,references,reload,shutdown,process,file其中一部分。

####创建数据库参考

CREATE DATABASE ambari;

use ambari;

CREATE USER 'ambari'@'%' IDENTIFIED BY 'ambarizk123';

GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'%';

CREATE USER 'ambari'@'localhost' IDENTIFIED BY 'ambarizk123';

GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'localhost';

CREATE USER 'ambari'@'master' IDENTIFIED BY 'ambarizk123';

GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'master';

FLUSH PRIVILEGES;

source /var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql

show tables;

use mysql;

select Host User Password from user where user='ambari';

CREATE DATABASE hive;

use hive;

CREATE USER 'hive'@'%' IDENTIFIED BY 'hive';

GRANT ALL PRIVILEGES ON *.* TO 'hive'@'%';

CREATE USER 'hive'@'localhost' IDENTIFIED BY 'hive';

GRANT ALL PRIVILEGES ON *.* TO 'hive'@'localhost';

CREATE USER 'hive'@'master' IDENTIFIED BY 'hive';

GRANT ALL PRIVILEGES ON *.* TO 'hive'@'master';

FLUSH PRIVILEGES;

CREATE DATABASE oozie;

use oozie;

CREATE USER 'oozie'@'%' IDENTIFIED BY 'oozie';

GRANT ALL PRIVILEGES ON *.* TO 'oozie'@'%';

CREATE USER 'oozie'@'localhost' IDENTIFIED BY 'oozie';

GRANT ALL PRIVILEGES ON *.* TO 'oozie'@'localhost';

CREATE USER 'oozie'@'master' IDENTIFIED BY 'oozie';

GRANT ALL PRIVILEGES ON *.* TO 'oozie'@'master';

FLUSH PRIVILEGES;

###配置jdbc驱动

mkdir /usr/share/java

#下载并上传mysql-connector-java-5.1.40.jar重命令为mysql-connector-java.jar

cp /usr/share/java/mysql-connector-java.jar /var/lib/ambari-server/resources/mysql-jdbc-driver.jar

vim /etc/ambari-server/conf/ambari.properties

#添加server.jdbc.driver.path=/usr/share/java/mysql-connector-java.jar

ambari-server setup

#y-输入用户sfapp-3-配置jdk路径-y-3-配置mysql信息（主机、端口、数据库名、用户、密码）-y

ambari-server stop

ambari-server start

###网页引导安装

#Launch Install Wizard-输入集群名字-选择版本、选择操作系统、配置仓库路径

http://10.202.107.27:81/HDP/centos7/2.6.3.0-235

http://10.202.107.27:81/HDP-UTILS

#输入主机名或IP地址，导入主服务器的ssh私钥

#或者不导入私钥选第二个，在从服务器安装yum -y install ambari-agent sudo ambari-agent restart

#如果这里出现问题执行

ambari-server stop

ambari-server reset #重置命令

ambari-server setup #重新设置

ambari-server start

tail -300f /var/log/ambari-server/ambari-server.log

#选择安装的服务、配置参数

#安装完成

array=("sfapp@cnsz22VLK5444" "sfapp@cnsz22VLK5445" "sfapp@cnsz22VLK7714")

for i in "${array[@]}"

do

if [ ! -n $i ]

then

echo "HOSTNAME IS NULL"

else

ssh "$i" "ntpdate -u 6.6.1.52" #ssh登录

echo "$i"

fi

done

beeline -u " jdbc:hive2://10.202.77.201:10000" -n hive -p hive

set hive.execution.engine=tez;

hive -service hivemetastore

hive -service hiveserver2

hive -S -e "select * from xxx"   --S静音模式不打印MR2的进度信息   e加载hql查询语句

hive -f test.hql --加载一个hql文件

source  test.hql

for f in 'rpm -qa | grep xxx';do rpm -e --nodeps  ${f} done;

　　

相关文章