将zabbix数据库从innodb迁移到tokudb并分区

时间:2022-06-29 00:49:45

下载percona server,并启用tokudb引擎

wget -c https://www.percona.com/downloads/Percona-Server-5.6/Percona-Server-5.6.32-78.0/binary/tarball/Percona-Server-5.6.32-rel78.0-Linux.x86_64.ssl101.tar.gz

tar zxvfPercona-Server-5.6.32-rel78.0-Linux.x86_64.ssl101.tar.gz -C /usr/local/

ln -s Percona-Server-5.6.32-rel78.0-Linux.x86_64.ssl101/ mysql

对数据库进行配置,使之能够正常启动

### TOKUDB ###

plugin-load                     = ha_tokudb.so

tokudb_data_dir                 =/export/mysql/mysql3306/tokudb

tokudb-log-dir                  =/export/mysql/mysql3306/tokudb

tokudb_tmp_dir                  = /export/mysql/mysql3306/tmp

#tokudb_row_format              = tokudb_small

tokudb_cache_size               = 3G

tokudb_commit_sync              = 0

tokudb_directio                 = 1

tokudb_read_block_size          = 128K

tokudb_read_buf_size            = 128K

启用tokudb引擎

echo never >/sys/kernel/mm/redhat_transparent_hugepage/defrag

echo never >/sys/kernel/mm/redhat_transparent_hugepage/enabled

echo never >/sys/kernel/mm/transparent_hugepage/enabled

echo never >/sys/kernel/mm/transparent_hugepage/defrag

cd /usr/local/mysql

./bin/ps_tokudb_admin --enable

 

对数据进行操作,操作前准备

set sql_log_bin=0

set wait_timeout=10000000000000;

select * from zabbix.history intooutfile ‘/tmp/history.txt’;

truncate table zabbix.history;

alter table zabbix.history_unitengine = tokudb;

#alter table zabbix.historypartition by …

此处用的是别人写的存储过程对表进行分区,存储过程见文章最后

CALL partition_maintenance('zabbix','history', 28, 24, 5);

load data infile '/tmp/history.txt’into table history;

 

说明:

1. 在表非常大的情况下,不建议用mysqldump方式导出后再导入,也不建议直接altertable进行修改,因为用这2种方式耗时都比loaddata这种方式慢,而且是几倍的差距。

2. 本文记录的只是自己操作的一个方式,可能不是最优的迁移方式。400G的表,将近用了小一天的时间。

3. 不建议直接对zabbix生产库进行操作,可以先用xtrabackupex进行备份,备份时用流直接将数据传到测试的机器,可以节省很多时间

innobackupex --defaults-file=/export/mysql/mysql4306/my4306.cnf --user=root--stream=xbstream /tmp/ | ssh root@10.237.81.192 "xbstream -x -C /export/mysql"

4. 通过备份,还原,建议生产库的从库,记录主库的binlog位置,待从库操作完毕后,开启多线程同步去追主,等主从数据一致后可以将zabbix连接到从库上,再对原生产库进行操作。

5. 如果zabbix库中要分区的表有id等主键,需要将主键删除后才可建议分区表,因为分区表为了最大限度的利用索引,分区列必须包含主键。而分区又是以时间来分,故需将id等主键删除或改成普通索引。

存储过程如下:

DELIMITER $$

CREATE PROCEDURE`partition_create`(SCHEMANAME varchar(64), TABLENAME varchar(64), PARTITIONNAMEvarchar(64), CLOCK int)

BEGIN

        /*

           SCHEMANAME = The DB schema in whichto make changes

           TABLENAME = The table withpartitions to potentially delete

           PARTITIONNAME = The name of thepartition to create

        */

        /*

           Verify that the partition does notalready exist

        */

 

        DECLARE RETROWS INT;

        SELECT COUNT(1) INTO RETROWS

        FROM information_schema.partitions

        WHERE table_schema = SCHEMANAME ANDtable_name = TABLENAME AND partition_description >= CLOCK;

 

        IF RETROWS = 0 THEN

                /*

                   1. Print a messageindicating that a partition was created.

                   2. Create the SQL to createthe partition.

                   3. Execute the SQL from #2.

                */

                SELECT CONCAT("partition_create(", SCHEMANAME, ",", TABLENAME,",", PARTITIONNAME, ",", CLOCK, ")" ) AS msg;

               SET @sql = CONCAT('ALTER TABLE ', SCHEMANAME, '.', TABLENAME, ' ADD PARTITION (PARTITION ',PARTITIONNAME, ' VALUES LESS THAN (', CLOCK, '));' );

                PREPARE STMT FROM @sql;

                EXECUTE STMT;

                DEALLOCATE PREPARE STMT;

        END IF;

END$$

DELIMITER ;

 

 

 

DELIMITER $$

CREATE PROCEDURE`partition_drop`(SCHEMANAME VARCHAR(64), TABLENAME VARCHAR(64),DELETE_BELOW_PARTITION_DATE BIGINT)

BEGIN

        /*

           SCHEMANAME = The DB schema in whichto make changes

           TABLENAME = The table withpartitions to potentially delete

           DELETE_BELOW_PARTITION_DATE = Deleteany partitions with names that are dates older than this one (yyyy-mm-dd)

        */

        DECLARE done INT DEFAULT FALSE;

        DECLARE drop_part_name VARCHAR(16);

 

        /*

           Get a list of all the partitionsthat are older than the date

           in DELETE_BELOW_PARTITION_DATE.  All partitions are prefixed with

           a "p", so use SUBSTRING TOget rid of that character.

        */

        DECLARE myCursor CURSOR FOR

                SELECT partition_name

                FROMinformation_schema.partitions

                WHERE table_schema = SCHEMANAMEAND table_name = TABLENAME AND CAST(SUBSTRING(partition_name FROM 2) ASUNSIGNED) < DELETE_BELOW_PARTITION_DATE;

        DECLARE CONTINUE HANDLER FOR NOT FOUNDSET done = TRUE;

 

        /*

           Create the basics for when we needto drop the partition.  Also, create

           @drop_partitions to hold acomma-delimited list of all partitions that

           should be deleted.

        */

        SET @alter_header = CONCAT("ALTERTABLE ", SCHEMANAME, ".", TABLENAME, " DROP PARTITION");

        SET @drop_partitions = "";

 

        /*

           Start looping through all thepartitions that are too old.

        */

        OPEN myCursor;

        read_loop: LOOP

                FETCH myCursor INTOdrop_part_name;

                IF done THEN

                        LEAVE read_loop;

                END IF;

                SET @drop_partitions =IF(@drop_partitions = "", drop_part_name, CONCAT(@drop_partitions,",", drop_part_name));

        END LOOP;

        IF @drop_partitions != ""THEN

                /*

                   1. Build the SQL to drop allthe necessary partitions.

                   2. Run the SQL to drop thepartitions.

                   3. Print out the tablepartitions that were deleted.

                */

                SET @full_sql = CONCAT(@alter_header,@drop_partitions, ";");

                PREPARE STMT FROM @full_sql;

                EXECUTE STMT;

                DEALLOCATE PREPARE STMT;

 

                SELECT CONCAT(SCHEMANAME,".", TABLENAME) AS `table`, @drop_partitions AS `partitions_deleted`;

        ELSE

                /*

                   No partitions are beingdeleted, so print out "N/A" (Not applicable) to indicate

                   that no changes were made.

                */

                SELECT CONCAT(SCHEMANAME,".", TABLENAME) AS `table`, "N/A" AS `partitions_deleted`;

        END IF;

END$$

DELIMITER ;

DELIMITER $$

CREATE PROCEDURE`partition_maintenance`(SCHEMA_NAME VARCHAR(32), TABLE_NAME VARCHAR(32),KEEP_DATA_DAYS INT, HOURLY_INTERVAL INT, CREATE_NEXT_INTERVALS INT)

BEGIN

        DECLARE OLDER_THAN_PARTITION_DATEVARCHAR(16);

        DECLARE PARTITION_NAME VARCHAR(16);

        DECLARE OLD_PARTITION_NAME VARCHAR(16);

        DECLARE LESS_THAN_TIMESTAMP INT;

        DECLARE CUR_TIME INT;

 

        CALL partition_verify(SCHEMA_NAME,TABLE_NAME, HOURLY_INTERVAL);

        SET CUR_TIME =UNIX_TIMESTAMP(DATE_FORMAT(NOW(), '%Y-%m-%d 00:00:00'));

 

        SET @__interval = 1;

        create_loop: LOOP

                IF @__interval >CREATE_NEXT_INTERVALS THEN

                        LEAVE create_loop;

                END IF;

 

                SET LESS_THAN_TIMESTAMP =CUR_TIME + (HOURLY_INTERVAL * @__interval * 3600);

                SET PARTITION_NAME =FROM_UNIXTIME(CUR_TIME + HOURLY_INTERVAL * (@__interval - 1) * 3600,'p%Y%m%d%H00');

                IF(PARTITION_NAME !=OLD_PARTITION_NAME) THEN

                     CALLpartition_create(SCHEMA_NAME, TABLE_NAME, PARTITION_NAME, LESS_THAN_TIMESTAMP);

              ENDIF;

                SET @__interval=@__interval+1;

                SET OLD_PARTITION_NAME = PARTITION_NAME;

        END LOOP;

 

        SETOLDER_THAN_PARTITION_DATE=DATE_FORMAT(DATE_SUB(NOW(), INTERVAL KEEP_DATA_DAYSDAY), '%Y%m%d0000');

        CALL partition_drop(SCHEMA_NAME,TABLE_NAME, OLDER_THAN_PARTITION_DATE);

 

END$$

DELIMITER ;

 

 

 

DELIMITER $$

CREATE PROCEDURE`partition_verify`(SCHEMANAME VARCHAR(64), TABLENAME VARCHAR(64),HOURLYINTERVAL INT(11))

BEGIN

        DECLARE PARTITION_NAME VARCHAR(16);

        DECLARE RETROWS INT(11);

        DECLARE FUTURE_TIMESTAMP TIMESTAMP;

 

        /*

         * Check if any partitions exist forthe given SCHEMANAME.TABLENAME.

         */

        SELECT COUNT(1) INTO RETROWS

        FROM information_schema.partitions

        WHERE table_schema = SCHEMANAME ANDtable_name = TABLENAME AND partition_name IS NULL;

 

        /*

         * If partitions do not exist, go aheadand partition the table

         */

        IF RETROWS = 1 THEN

                /*

                 * Take the current date at00:00:00 and add HOURLYINTERVAL to it. This is the timestamp below which we will store values.

                 * We begin partitioning basedon the beginning of a day.  This isbecause we don't want to generate a random partition

                 * that won't necessarily fallin line with the desired partition naming (ie: if the hour interval is 24hours, we could

                 * end up creating a partitionnow named "p201403270600" when all other partitions will be like"p201403280000").

                 */

                SET FUTURE_TIMESTAMP =TIMESTAMPADD(HOUR, HOURLYINTERVAL, CONCAT(CURDATE(), " ",'00:00:00'));

                SET PARTITION_NAME =DATE_FORMAT(CURDATE(), 'p%Y%m%d%H00');

 

                -- Create the partitioningquery

                SET @__PARTITION_SQL =CONCAT("ALTER TABLE ", SCHEMANAME, ".", TABLENAME, "PARTITION BY RANGE(`clock`)");

                SET @__PARTITION_SQL =CONCAT(@__PARTITION_SQL, "(PARTITION ", PARTITION_NAME, " VALUESLESS THAN (", UNIX_TIMESTAMP(FUTURE_TIMESTAMP), "));");

 

                -- Run the partitioning query

                PREPARE STMT FROM@__PARTITION_SQL;

                EXECUTE STMT;

                DEALLOCATE PREPARE STMT;

        END IF;

END$$

DELIMITER ;