Merge pull request zhishutech#15 from xuclachina/master

修复部分已知Bug

mysql-snapshot/db_pool.py

@@ -5,6 +5,7 @@
 import traceback
 import pymysql
 import warnings
+from retry import retry
 warnings.filterwarnings("ignore")
 
 from DBUtils.PooledDB import PooledDB
@@ -18,10 +19,9 @@ def __init__(self, host=None, port=None, user=None, password=None):
         self.host = port
         self.host = user
         self.host = password
-        self.pool = PooledDB(creator=pymysql, mincached=1, maxcached=10, maxconnections=100, 
+        self.pool = PooledDB(creator=pymysql, mincached=30, maxcached=50, maxconnections=200, maxusage=0,
                              blocking=True, host=host, port=int(port), user=user, passwd=password,
-                             db="information_schema", charset='utf8')
-
+                             db="information_schema", charset='utf8')      
     def get_connection(self):
         return self.pool.connection()
 
@@ -33,7 +33,7 @@ def __init__(self, conn_setting=None):
         user = conn_setting['user']
         password = conn_setting['password']
         global db_pool_ins
-        if db_pool_ins == None:
+        if db_pool_ins is None:
             db_pool_ins = DBPool(host, port, user, password)
         self.conn = db_pool_ins.get_connection()
         self.cursor = self.conn.cursor()
@@ -64,6 +64,7 @@ def data_operate_count(self, sql, params=()):
         self.conn.commit()
         return count
 
+    @retry(delay=1)
     def data_inquiry(self, sql, params=()):
         """
         SELECT 操作
@@ -78,5 +79,3 @@ def commit(self):
 
     def rollback(self):
         self.conn.rollback()
-
-

mysql-snapshot/readme.md

@@ -14,10 +14,21 @@ pip install -r requirements.txt
 示例
 
 ```python
-python /tmp/snapshot/snapshot.py --host=127.0.0.1 --port=3306 --user=xucl --password=xuclxucl --conditions="{'Threads_connected': 500, 'Threads_running': 20, 'Innodb_row_lock_current_waits': 5, 'Slow_queries': 5, 'cpu_user':10, 'cpu_sys':10, 'cpu_iowait':5, 'sys_iops':500, 'sql_delay':60}" --storedir=/tmp
+python snapshot.py --host=127.0.0.1 --port=3306 --user=xucl --password=xuclxucl --conditions="{'Threads_connected': 500, 'Threads_running': 20, 'Innodb_row_lock_current_waits': 5, 'Slow_queries': 5, 'Innodb_buffer_pool_wait_free': 5, 'cpu_user':10, 'cpu_sys':10, 'cpu_iowait':5, 'sys_iops':500, 'sql_delay':60}" --interval=30 --storedir=/tmp
 ```
 
-
+触发条件说明：
+
+- Threads_connected：MySQL实例连接数
+- Threads_running：MySQL实例当前运行连接数
+- Innodb_row_lock_current_waits：当前行锁等待
+- Slow_queries：每秒记录慢查询数
+- Innodb_buffer_pool_wait_free：每秒产生ibp等待事件
+- cpu_user：user cpu占用
+- cpu_sys：sys cpu占用
+- cpu_iowait：cpu iowait占用
+- sys_iops：系统iops占用
+- sql_delay：主从延迟时间
 
 **生成拍照文件**
 

mysql-snapshot/requirements.txt

@@ -1,3 +1,7 @@
 configparser==3.7.4
 DBUtils==1.3
+decorator==4.4.0
+psutil==5.6.3
+py==1.8.0
 PyMySQL==0.9.3
+retry==0.9.2

mysql-snapshot/snapshot.py

@@ -2,13 +2,15 @@
 
 # -------------------------------------------------------------------------------
 # Name:         snapshot
-# Description:  
+# Description:
 # Author:       xucl
 # Date:         2019-04-17
 # -------------------------------------------------------------------------------
 
 from utils import *
 from db_pool import DBAction
+from threading import Lock
+
 
 class Snapshot(object):
     def __init__(self, connection_settings, interval=None, conditions=None, storedir=None):
@@ -24,14 +26,16 @@ def __init__(self, connection_settings, interval=None, conditions=None, storedir
         self.conditions = conditions
         self.stordir = storedir
 
-
     def run(self):
         condition_dict = eval(self.conditions)
-        dbaction = DBAction(conn_setting)
+
         while True:
+            lock = Lock()
+            global dbaction
+            dbaction = DBAction(self.conn_setting)
             status_dict1 = get_mysql_status(dbaction)
             sys_dict1 = get_sys_status()
-            slow_log,error_log = get_log_dir(dbaction)
+            slow_log, error_log = get_log_dir(dbaction)
             slave_status_dict = get_slave_status(dbaction)
 
             time.sleep(1)
@@ -40,43 +44,45 @@ def run(self):
             sys_dict2 = get_sys_status()
 
             origin_status_list = ['Threads_connected', 'Threads_running', 'Innodb_row_lock_current_waits']
-            origin_sys_status_list= ['cpu_user', 'cpu_sys', 'cpu_iowait']
-            diff_status_list = ['Slow_queries']
+            origin_sys_status_list = ['cpu_user', 'cpu_sys', 'cpu_iowait']
+            diff_status_list = ['Slow_queries', 'Innodb_buffer_pool_wait_free']
             diff_sys_status = ['sys_iops']
 
             origin_status_dict = get_origin_status(status_dict1, origin_status_list)
             origin_status_sys_dict = get_origin_sys_status(sys_dict1, origin_sys_status_list)
             diff_status_dict = get_diff_status(status_dict1, status_dict2, diff_status_list)
             diff_sys_status = get_sys_diff_status(sys_dict1, sys_dict2, diff_sys_status)
 
-            check_dict = dict(origin_status_dict, **diff_status_dict, **origin_status_sys_dict, **diff_sys_status,
-                              **slave_status_dict)
+            check_dict = dict(origin_status_dict, **diff_status_dict, **origin_status_sys_dict, **diff_sys_status, **slave_status_dict)
 
             collect_flag = check_conditions(check_dict, condition_dict)
 
             time_now = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S')
             filedir = create_unique_dir(self.stordir, time_now)
             if collect_flag:
-                do_in_thread(mysql_variables, dbaction, filedir)
-                do_in_thread(mysql_status, dbaction, filedir)
-                do_in_thread(mysql_innodb_status, dbaction, filedir)
-                do_in_thread(mysql_slave_status, dbaction, filedir)
-                do_in_thread(mysql_processlist, dbaction, filedir)
-                do_in_thread(mysql_transactions, dbaction, filedir)
-                do_in_thread(mysql_lock_info, dbaction, filedir)
-                do_in_thread(mysql_error_log, slow_log, filedir)
-                do_in_thread(mysql_slow_log, error_log, filedir)
-                do_in_thread(system_message, '/var/log/messages', filedir)
-                do_in_thread(system_dmesg, '/var/log/dmesg', filedir)
-                do_in_thread(system_top, filedir)
-                do_in_thread(system_iostat, filedir)
-                do_in_thread(system_mpstat, filedir)
-                do_in_thread(system_tcpdump, filedir)
-                do_in_thread(system_mem_info, filedir)
-                do_in_thread(system_interrupts, filedir)
-                do_in_thread(system_ps, filedir)
-                do_in_thread(system_netstat, filedir)
-                do_in_thread(system_vmstat, filedir)
+                lock.acquire()
+                thread_objs = []
+                mysql_func_list = [mysql_variables, mysql_status, mysql_innodb_status, mysql_slave_status, mysql_processlist, mysql_transactions, mysql_lock_info]
+                sys_func_list = [mysql_error_log, mysql_slow_log, system_message, system_dmesg, system_top, system_iostat, system_mpstat, system_tcpdump, system_mem_info,
+                                 system_interrupts, system_ps, system_netstat, system_vmstat]
+                sys_arg_list = [slow_log, error_log, '/var/log/messages', '/var/log/dmesg', '', '', '', '', '', '', '', '', '']
+                for func in mysql_func_list:
+                    dbaction = DBAction(self.conn_setting)
+                    t = do_in_thread(func, dbaction, filedir)
+                    thread_objs.append(t)
+
+                for index, func in enumerate(sys_func_list):
+                    if sys_arg_list[index]:
+                        t = do_in_thread(func, sys_arg_list[index], filedir)
+                    else:
+                        t = do_in_thread(func, filedir)
+                    thread_objs.append(t)
+
+                for thread_obj in thread_objs:
+                    thread_obj.join()
+
+            lock.release()
+
             time.sleep(self.interval)