wechat_utils.py

# -*- coding: utf-8 -*-
"""
Created on Tue Mar  7 12:44:30 2017

@author: Quantum Liu
"""
from keras import __version__ as kv
kv=int(kv[0])
import platform
pv=int(platform.python_version()[0])
import numpy as np
import scipy.io as sio
import itchat
from keras.callbacks import Callback
import time
import matplotlib  
matplotlib.use('Agg') # 
import matplotlib.pyplot as plt
from math import ceil
from itchat.content import TEXT
if pv>2:
    import _thread as th
else:
    import thread as th
import os
from os import system
import re
import traceback
import platform
from requests.exceptions import ConnectionError
#==============================================================================
#==============================================================================
# A log in function call it at first
#函数，需要首先调用
#==============================================================================
def login():
    if 'Windows' in platform.system():
        itchat.auto_login(enableCmdQR=1,hotReload=True)#
    else:
        itchat.auto_login(enableCmdQR=2,hotReload=True)#
    itchat.dump_login_status()#dump
#==============================================================================
# 
#==============================================================================
def send_text(text):
    #send text msgs to 'filehelper'
    #给文件助手发送文本信息
    try:
        itchat.send_msg(msg=text,toUserName='filehelper')
        return
    except (ConnectionError,NotImplementedError,KeyError):
        traceback.print_exc()
        print('\nConection error,failed to send the message!\n')
        return
    else:
        return
def send_img(filename):
    #send text imgs to 'filehelper'
    #给文件助手发送
    try:
        itchat.send_image(filename,toUserName='filehelper')
        return
    except (ConnectionError,NotImplementedError,KeyError):
        traceback.print_exc()
        print('\nConection error,failed to send the figure!\n')
        return
    else:
        return
#==============================================================================
#     
#==============================================================================
class sendmessage(Callback):
    #A subclss of keras.callbacks.Callback class
    #keras.callbacks.Callback class的子类
    def __init__(self,savelog=True,fexten=''):
        self.fexten=(fexten if fexten else '')#the name of log and figure files 
        self.savelog=bool(savelog)#save log or not
        
    def t_send(self,msg,toUserName='filehelper'):
        try:
            itchat.send_msg(msg=msg,toUserName=toUserName)
            return
        except (ConnectionError,NotImplementedError,KeyError):
            traceback.print_exc()
            print('\nConection error,failed to send the message!\n')
            return
        else:
            return
    def t_send_img(self,filename,toUserName='filehelper'):
        try:
            itchat.send_image(filename,toUserName=toUserName)
            return
        except (ConnectionError,NotImplementedError,KeyError):
            traceback.print_exc()
            print('\nConection error,failed to send the figure!\n')
            return
        else:
            return
       
            
    def shutdown(self,sec,save=True,filepath='temp.h5'):
        #Function used to shut down the computer
        #sec:waitting time to shut down the computer,sencond
        #save:wether saving the model
        #filepath:the filepath for saving the model
        #关机函数
        #sec:关机等待秒数
        #save:是否保存模型
        #filepath:保存模型的文件名
        if save:
            self.model.save(filepath, overwrite=True)
            self.t_send('Command accepted,the model has already been saved,shutting down the computer....', toUserName='filehelper')
        else:
            self.t_send('Command accepted,shutting down the computer....', toUserName='filehelper')
        if 'Windows' in platform.system():
            th.start_new_thread(system, ('shutdown -s -t %d' %sec,))
        else:
            m=(int(sec/60) if int(sec/60) else 1)
            th.start_new_thread(system, ('shutdown -h -t %d' %m,))
            
#==============================================================================
#         
#==============================================================================
    def cancel(self):
        #Cancel function to cancel shutting down the computer
        #取消关机函数
        self.t_send('Command accepted,cancel shutting down the computer....', toUserName='filehelper')
        if 'Windows' in platform.system():
            th.start_new_thread(system, ('shutdown -a',))
        else:
            th.start_new_thread(system, ('shutdown -c',))
#==============================================================================
#         
#==============================================================================
    def GetMiddleStr(self,content,startStr,endStr):
        #get the string between two specified strings
        #从指定的字符串之间截取字符串
        try:
          startIndex = content.index(startStr)
          if startIndex>=0:
            startIndex += len(startStr)
          endIndex = content.index(endStr)
          return content[startIndex:endIndex]
        except:
            return ''
#==============================================================================
# 
#==============================================================================
    def validateTitle(self,title):
        #transform a string to a validate filename
        #将字符串转化为合法文件名
        rstr = r"[\/\\\:\*\?\"\<\>\|]"  # '/\:*?"<>|'
        new_title = re.sub(rstr, "", title).replace(' ','')
        return new_title
#==============================================================================
#         
#==============================================================================
    def prog(self):#Show progress
        nb_batches_total=(self.params['nb_epoch'] if not kv-1 else self.params['epochs'])*self.params['nb_sample']/self.params['batch_size']
        nb_batches_epoch=self.params['nb_sample']/self.params['batch_size']
        prog_total=(self.t_batches/nb_batches_total if nb_batches_total else 0)+0.01
        prog_epoch=(self.c_batches/nb_batches_epoch if nb_batches_epoch else 0)+0.01
        if self.t_epochs:
            now=time.time()
            t_mean=float(sum(self.t_epochs)) / len(self.t_epochs)
            eta_t=(now-self.train_start)*((1/prog_total)-1)
            eta_e=t_mean*(1-prog_epoch)
            t_end=time.asctime(time.localtime(now+eta_t))
            e_end=time.asctime(time.localtime(now+eta_e))
            m='\nTotal:\nProg:'+str(prog_total*100.)[:5]+'%\nEpoch:'+str(self.epoch[-1])+'/'+str(self.stopped_epoch)+'\nETA:'+str(eta_t)[:8]+'sec\nTrain will be finished at '+t_end+'\nCurrent epoch:\nPROG:'+str(prog_epoch*100.)[:5]+'%\nETA:'+str(eta_e)[:8]+'sec\nCurrent epoch will be finished at '+e_end
            self.t_send(msg=m)
            print(m)
        else:
            now=time.time()
            eta_t=(now-self.train_start)*((1/prog_total)-1)
            eta_e=(now-self.train_start)*((1/prog_epoch)-1)
            t_end=time.asctime(time.localtime(now+eta_t))
            e_end=time.asctime(time.localtime(now+eta_e))
            m='\nTotal:\nProg:'+str(prog_total*100.)[:5]+'%\nEpoch:'+str(len(self.epoch))+'/'+str(self.stopped_epoch)+'\nETA:'+str(eta_t)[:8]+'sec\nTrain will be finished at '+t_end+'\nCurrent epoch:\nPROG:'+str(prog_epoch*100.)[:5]+'%\nETA:'+str(eta_e)[:8]+'sec\nCurrent epoch will be finished at '+e_end
            self.t_send(msg=m)
            print(m)
            
#==============================================================================
# 
#==============================================================================
    def get_fig(self,level='all',metrics=['all']):
        #Get figure of train infomation
        #level:show the information of which level
        #metrics:metrics want to show,only show available ones
        #获取训练状态图表
        #level:显示batch级别函数epoch级别
        #metrics:希望获得的指标，只显示存在的指标，若指定了不存在的指标将不会被显示
        color_list='rgbyck'*10
        def batches(color_list='rgbyck'*10,metrics=['all']):
            if 'all' in metrics:
                m_available=list(self.logs_batches.keys())
            else:
                m_available=([val for val in list(self.logs_batches.keys()) if val in metrics]if[val for val in list(self.logs_batches.keys()) if val in metrics]else list(self.logs_batches.keys()))
            nb_rows_batches=int(ceil(len(m_available)*1.0/2))
            fig_batches=plt.figure('all_subs_batches')
            for i,k in enumerate(m_available):
                p=plt.subplot(nb_rows_batches,2,i+1)
                data=self.logs_batches[k]
                p.plot(range(len(data)),data,color_list[i]+'-',label=k)
                p.set_title(k+' in batches',fontsize=14)
                p.set_xlabel('batch',fontsize=10)
                p.set_ylabel(k,fontsize=10)
                #p.legend()
            filename=(self.fexten if self.fexten else self.validateTitle(self.localtime))+'_batches.jpg'
            plt.tight_layout()
            plt.savefig(filename)
            plt.close('all')
#==============================================================================
#             try:
#                 itchat.send_image(filename,toUserName='filehelper')
#             except (socket.gaierror,ConnectionError,NotImplementedError,TypeError,KeyError):
#                 traceback.print_exc()
#                 print('\nConection error!\n')
#                 return
#==============================================================================
            self.t_send_img(filename,toUserName='filehelper')
            time.sleep(.5)
            self.t_send('Sending batches figure',toUserName='filehelper')
            return
#==============================================================================
#             
#==============================================================================
        def epochs(color_list='rgbyck'*10,metrics=['all']):
            if 'all' in metrics:
                m_available=list(self.logs_epochs.keys())
            else:
                m_available=([val for val in list(self.logs_epochs.keys()) if val in metrics]if[val for val in list(self.logs_epochs.keys()) if val in metrics]else list(self.logs_epochs.keys()))
            nb_rows_epochs=int(ceil(len(m_available)*1.0/2))
            fig_epochs=plt.figure('all_subs_epochs')
            for i,k in enumerate(m_available):
                p=plt.subplot(nb_rows_epochs,2,i+1)
                data=self.logs_epochs[k]
                p.plot(range(len(data)),data,color_list[i]+'-',label=k)
                p.set_title(k+' in epochs',fontsize=14)
                p.set_xlabel('epoch',fontsize=10)
                p.set_ylabel(k,fontsize=10)
            filename=(self.fexten if self.fexten else self.validateTitle(self.localtime))+'_epochs.jpg'
            plt.tight_layout()
            plt.savefig(filename)
            plt.close('all')
#==============================================================================
#             try:
#                 itchat.send_image(filename,toUserName='filehelper')
#             except (socket.gaierror,ConnectionError,NotImplementedError,TypeError,KeyError):
#                 traceback.print_exc()
#                 print('\nConection error!\n')
#                 return
#==============================================================================
            self.t_send_img(filename,toUserName='filehelper')
            time.sleep(.5)
            self.t_send('Sending epochs figure',toUserName='filehelper')
            return
#==============================================================================
#             
#==============================================================================
        try:
            if not self.epoch and (level in ['all','epochs']):
                level='batches'
            if level=='all':
                batches(metrics=metrics)
                epochs(metrics=metrics)
                th.exit()
                return
            elif level=='epochs':
                epochs(metrics=metrics)
                th.exit()
                return
            elif level=='batches':
                batches(metrics=metrics)
                th.exit()
                return
            else:
                batches(metrics=metrics)
                epochs(metrics=metrics)
                th.exit()
                return
        except Exception:
            traceback.print_exc()
            self.t_send('Failed to send figure',toUserName='filehelper')
            th.exit()
            return
#==============================================================================
#             
#==============================================================================
    def gpu_status(self,av_type_list):
        for t in av_type_list:
            cmd='nvidia-smi -q --display='+t
            #print('\nCMD:',cmd,'\n')
            r=os.popen(cmd)
            info=r.readlines()
            r.close()
            content = " ".join(info)
            #print('\ncontent:',content,'\n')
            index=content.find('Attached GPUs')
            s=content[index:].replace(' ','').rstrip('\n')
            self.t_send(s, toUserName='filehelper')
            time.sleep(.5)
        #th.exit()
#==============================================================================
# 
#==============================================================================
    def on_train_begin(self, logs={}):
        self.epoch=[]
        self.t_epochs=[]
        self.t_batches=0
        self.logs_batches={}
        self.logs_epochs={}
        self.train_start=time.time()
        self.localtime = time.asctime( time.localtime(self.train_start) )
        self.mesg = 'Train started at: '+self.localtime
        self.t_send(self.mesg, toUserName='filehelper')
        self.stopped_epoch = (self.params['epochs'] if kv-1 else self.params['nb_epoch'])
        @itchat.msg_register(TEXT)
#==============================================================================
#         registe methods to reply msgs,similar to main()
#         注册消息响应方法，相当于主函数
#==============================================================================
        def manualstop(msg):
            text=msg['Text']
            stop_training_cmdlist=['Stop now',"That's enough",u'停止训练',u'放弃治疗']
            #The keywords of stop training,if any of them is in the msg you sent,the command would be accepted
            #停止训练的关键词列表，发送的消息中包含任意一项都可触发命令
            shut_down_cmdlist=[u'关机','Shut down','Shut down the computer',u'别浪费电了',u'洗洗睡吧']
            #The keywords of shutting down,similair to stop_training_cmdlist
            #关机关键词列表，和stop_training_cmdlist类似
            cancel_cmdlist=[u'取消','cancel','aaaa']
            #The keywords of cancel shutting down,similair to stop_training_cmdlist
            #取消关机关键词列表，和stop_training_cmdlist类似
            get_fig_cmdlist=[u'获取图表','Show me the figure']
            #The keywords of getting figure,similair to stop_training_cmdlist
            #获取图表关键词列表，和stop_training_cmdlist类似
            gpu_cmdlist=['GPU','gpu',u'显卡']
            type_list=['MEMORY', 'UTILIZATION', 'ECC', 'TEMPERATURE', 'POWER', 'CLOCK', 'COMPUTE', 'PIDS', 'PERFORMANCE', 'SUPPORTED_CLOCKS,PAGE_RETIREMENT', 'ACCOUNTING']
            prog_cmdlist=[u'进度','Progress']
            if msg['ToUserName']=='filehelper':
                print('\n',text,'\n')
                if 'Stop at' in text:
                    # Specify stop epoch，training will be stop after that epoch
                    #指定停止轮数，训练在指定epoch完成后会停止
                    #Example:send:'Stop at:8' from your phone,and then training will be stopped after epoch8
                    #例如：手机发送“Stop at：8”，训练将在epoch8完成后停止
                    self.stopped_epoch = int(re.findall(r"\d+\.?\d*",text)[0])
                    if kv-1:
                        self.params['epochs']=self.stopped_epoch
                    else:
                        self.params['nb_epoch']=self.stopped_epoch
                    self.t_send('Command accepted,training will be stopped at epoch'+str(self.stopped_epoch), toUserName='filehelper')
#==============================================================================
#                 
#==============================================================================
                if any((k in text) for k in stop_training_cmdlist) :
                    #Stop training after current epoch finished
                    #当前epoch完成后停止训练
                    #example：send:'Stop now' or send:'停止训练' from your phone,and then training will be stopped after current epoch
                    #例如：手机发送“停止训练”或者“Stop now”，训练将会在当前epoch完成后被停止
                    self.model.stop_training = True
                    self.t_send('Command accepted,stop training now at epoch'+str(self.epoch[-1]+1), toUserName='filehelper')
#==============================================================================
#                 
#==============================================================================
                if any((k in text) for k in shut_down_cmdlist):
                    #Shutting down the computer after specified sec，specify waiting seconds and saved model filename by {sec} and [name](without .h5)
                    #在指定秒数后关机，用{sec}和[name]指定等待时间和保存文件名,文件名不包括.h5
                    #example:send:'Shut down now [test]{120}' from phone,the computer will be shut down after 120s,and save the model as test.h5
                    #or send:'Shut down now{120},don't save',then the model won't be saved.
                    if any((k in text) for k in [u'不保存模型',"don't save"]):
                        save=False
                    else:
                        save=True
                        filepath=(self.GetMiddleStr(text,'[',']')+'.h5' if self.GetMiddleStr(text,'[',']') else (self.fexten if self.fexten else self.validateTitle(self.localtime))+'.h5')
                    print('\n',filepath,'\n')
                    sec=int((self.GetMiddleStr(text,'{','}') if self.GetMiddleStr(text,'{','}')>'30' else 120))
                    self.shutdown(sec,save=save,filepath=filepath)
#==============================================================================
#                     
#==============================================================================
                if any((k in text) for k in cancel_cmdlist):
                    #Cancel shutting down the computer
                    self.cancel()
#==============================================================================
#                     
#==============================================================================
                if any((k in text) for k in get_fig_cmdlist):   
                    #Get figure of train infomation,specify metrics and level you want to show by[metrics]and{level},defualt are both 'all'
                    #example:send:'Show me the figure [loss]{batches}' from phone,you will recive a jpg image of losses in batches
                    #send:'Show me the figure'，you will recive two jpg images of all metrics in batches and epochs
                    #获取图表，通过[metrics]和{level}指定参数，如果没有指定则皆默认为’all'
                    #例如，手机发送"获取图表[loss]{batches}",会收到一个jpg格式的loss随batches变化的图片
                    #手机发送"获取图表",则会得到两张图片，分别是所有指标随batch和epoch的变化
                    metrics=(self.GetMiddleStr(text,'[',']').split() if self.GetMiddleStr(text,'[',']').split() else ['all'])
                    level=(self.GetMiddleStr(text,'{','}') if self.GetMiddleStr(text,'{','}') else 'all' )
                    if level in ['all','epochs','batches']:
                        th.start_new_thread(self.get_fig,(level,metrics))
                    else:
                        print("\nGot no level,using default 'all'\n")
                        self.t_send("Got no level,using default 'all'", toUserName='filehelper')
                        th.start_new_thread(self.get_fig,())
                if any((k in text) for k in gpu_cmdlist):
                    sp_type_lsit=(self.GetMiddleStr(text,'[',']').split() if self.GetMiddleStr(text,'[',']').split() else ['MEMORY'])
                    av_type_list=[val for val in sp_type_lsit if val in type_list]
                    self.gpu_status(av_type_list,)
                if any((k in text) for k in prog_cmdlist):
                    try:
                        self.prog()
                    except:
                        traceback.print_exc()
        th.start_new_thread(itchat.run, ())
#==============================================================================
#     
#==============================================================================
    def on_batch_end(self, batch, logs=None):
        logs = logs or {}
        for k in self.params['metrics']:
            if k in logs:
                self.logs_batches.setdefault(k, []).append(logs[k])
        self.c_batches+=1
        self.t_batches+=1
#==============================================================================
#                 
#==============================================================================
    def on_epoch_begin(self, epoch, logs=None):
        self.t_s=time.time()
        self.epoch.append(epoch)
        self.c_batches=0
        self.t_send('Epoch'+str(epoch+1)+'/'+str(self.stopped_epoch)+' started', toUserName='filehelper')
        self.mesg = ('Epoch:'+str(epoch+1)+' ')
#==============================================================================
#         
#==============================================================================
    def on_epoch_end(self, epoch, logs=None):
        for k in self.params['metrics']:
            if k in logs:
                self.mesg+=(k+': '+str(logs[k])[:5]+' ')
                self.logs_epochs.setdefault(k, []).append(logs[k])
#==============================================================================
#         except:
#             itchat.auto_login(hotReload=True,enableCmdQR=True)
#             itchat.dump_login_status()
#             self.t_send(self.mesg, toUserName='filehelper')
#==============================================================================
        if epoch+1>=self.stopped_epoch:
            self.model.stop_training = True
        logs = logs or {}
        self.epoch.append(epoch)
        self.t_epochs.append(time.time()-self.t_s)
        if self.savelog:
            sio.savemat((self.fexten if self.fexten else self.validateTitle(self.localtime))+'_logs_batches'+'.mat',{'log':np.array(self.logs_batches)})
            sio.savemat((self.fexten if self.fexten else self.validateTitle(self.localtime))+'_logs_batches'+'.mat',{'log':np.array(self.logs_epochs)})
        th.start_new_thread(self.get_fig,())
#==============================================================================
#         try:
#             itchat.send(self.mesg, toUserName='filehelper')
#         except:
#             traceback.print_exc()
#             return
#==============================================================================
        self.t_send(self.mesg, toUserName='filehelper')
        return
#==============================================================================
#         
#==============================================================================
    def on_train_end(self, logs=None):
        self.t_send('Train stopped at epoch'+str(self.epoch[-1]+1), toUserName='filehelper')