python脚本 用sqoop把mysql数据导入hive

时间：2024-05-28 18:04:26

转：https://blog.****.net/wulantian/article/details/53064123

用python把mysql数据库的数据导入到hive中，该过程主要是通过python语言操作sqoop.

#! /usr/bin/env python

# coding:utf-8

# --------------------------------

# Created by coco  on 16/2/23

# ---------------------------------

# Comment: 主要功能说明 :初始化业务数据库

import os

import pyhs2

conn=pyhs2.connect(host="192.168.8.94",port=10000,authMechanism="PLAIN",user="hdfs")

mysql_info={"host":"192.168.8.94","port":3306,"user":"root","passwd":"gc895316"}

print mysql_info

def run_hive_query(sql):

    with conn.cursor()  as cursor:

        cursor.execute(sql)

        return cursor.fetchall()

def mysql_to_hive(host,port,user,passwd,database,table):

    #os.system("hadoop fs -rm    -r /user/task/%s"%table)

    if [database] not in run_hive_query("show databases"):

        with conn.cursor() as cursor:

            cursor.execute("create database " +database)

    with conn.cursor() as cursor:

        cursor.execute("use  "+database)

    if [table] not in run_hive_query("show tables"):

        os.system("sqoop   import --connect   jdbc:mysql://%s:%s/%s --username  %s   --password  %s --table %s  --hive-database  %s  -m 10 --create-hive-table --hive-import   --hive-overwrite "%(

            host,port,database,user,passwd,table,database))

    else:

        os.system("sqoop   import --connect   jdbc:mysql://%s:%s/%s --username  %s   --password  %s --table %s  --hive-database  %s  -m 10 --hive-import   --hive-overwrite "%(

            host,port,database,user,passwd,table,database))

mysql_to_hive(mysql_info["host"],mysql_info["port"],mysql_info["user"],mysql_info["passwd"].replace("(","\("),"wwn","cm_vip")

相关文章