From 760f652ea87068d5506389ebb1ede5c54c27cae9 Mon Sep 17 00:00:00 2001 From: Kirill Dubovikov Date: Sat, 17 Jun 2017 17:12:49 +0300 Subject: [PATCH 1/5] fixed a bug in xgb defs, migrated everything to python 3 --- common_defs.py | 44 +++++++++++++++--------------- defs/xgb.py | 12 ++++----- hyperband.py | 72 +++++++++++++++++++++++++------------------------- main.py | 24 ++++++++--------- 4 files changed, 76 insertions(+), 76 deletions(-) diff --git a/common_defs.py b/common_defs.py index 0f5692d..db1cbe4 100644 --- a/common_defs.py +++ b/common_defs.py @@ -13,9 +13,9 @@ from hyperopt import hp from hyperopt.pyll.stochastic import sample except ImportError: - print "In order to achieve operational capability, this programme requires hyperopt to be installed (pip install hyperopt), unless you make get_params() use something else." - -# + print("In order to achieve operational capability, this programme requires hyperopt to be installed (pip install hyperopt), unless you make get_params() use something else.") + +# # handle floats which should be integers # works with flat params @@ -27,21 +27,21 @@ def handle_integers( params ): new_params[k] = int( v ) else: new_params[k] = v - + return new_params - + ### def train_and_eval_sklearn_classifier( clf, data ): - + x_train = data['x_train'] y_train = data['y_train'] - + x_test = data['x_test'] - y_test = data['y_test'] - - clf.fit( x_train, y_train ) - + y_test = data['y_test'] + + clf.fit( x_train, y_train ) + try: p = clf.predict_proba( x_train )[:,1] # sklearn convention except IndexError: @@ -51,7 +51,7 @@ def train_and_eval_sklearn_classifier( clf, data ): auc = AUC( y_train, p ) acc = accuracy( y_train, np.round( p )) - print "\n# training | log loss: {:.2%}, AUC: {:.2%}, accuracy: {:.2%}".format( ll, auc, acc ) + print("\n# training | log loss: {:.2%}, AUC: {:.2%}, accuracy: {:.2%}".format( ll, auc, acc )) # @@ -64,8 +64,8 @@ def train_and_eval_sklearn_classifier( clf, data ): auc = AUC( y_test, p ) acc = accuracy( y_test, np.round( p )) - print "# testing | log loss: {:.2%}, AUC: {:.2%}, accuracy: {:.2%}".format( ll, auc, acc ) - + print("# testing | log loss: {:.2%}, AUC: {:.2%}, accuracy: {:.2%}".format( ll, auc, acc )) + #return { 'loss': 1 - auc, 'log_loss': ll, 'auc': auc } return { 'loss': ll, 'log_loss': ll, 'auc': auc } @@ -73,14 +73,14 @@ def train_and_eval_sklearn_classifier( clf, data ): # "clf", even though it's a regressor def train_and_eval_sklearn_regressor( clf, data ): - + x_train = data['x_train'] y_train = data['y_train'] - + x_test = data['x_test'] - y_test = data['y_test'] - - clf.fit( x_train, y_train ) + y_test = data['y_test'] + + clf.fit( x_train, y_train ) p = clf.predict( x_train ) mse = MSE( y_train, p ) @@ -88,7 +88,7 @@ def train_and_eval_sklearn_regressor( clf, data ): mae = MAE( y_train, p ) - print "\n# training | RMSE: {:.4f}, MAE: {:.4f}".format( rmse, mae ) + print("\n# training | RMSE: {:.4f}, MAE: {:.4f}".format( rmse, mae )) # @@ -98,7 +98,7 @@ def train_and_eval_sklearn_regressor( clf, data ): rmse = sqrt( mse ) mae = MAE( y_test, p ) - print "# testing | RMSE: {:.4f}, MAE: {:.4f}".format( rmse, mae ) - + print("# testing | RMSE: {:.4f}, MAE: {:.4f}".format( rmse, mae )) + return { 'loss': rmse, 'rmse': rmse, 'mae': mae } diff --git a/defs/xgb.py b/defs/xgb.py index 0bd1b12..d0e8af2 100644 --- a/defs/xgb.py +++ b/defs/xgb.py @@ -25,7 +25,7 @@ 'default', hp.quniform( 'mcw_', 1, 10, 1 ) ]), - + 'subsample': hp.choice( 'ss', [ 'default', hp.uniform( 'ss_', 0.5, 1.0 ) @@ -37,7 +37,7 @@ 'colsample_bylevel': hp.choice( 'cbl', [ 'default', hp.uniform( 'cbl_', 0.5, 1.0 ) - ]), + ]), 'gamma': hp.choice( 'g', [ 'default', hp.uniform( 'g_', 0, 1 ) @@ -71,11 +71,11 @@ def get_params(): def try_params( n_iterations, params, get_predictions = False ): n_estimators = int( round( n_iterations * trees_per_iteration )) - print "n_estimators:", n_estimators + print("n_estimators:", n_estimators) pprint( params ) clf = XGB( n_estimators = n_estimators, nthread = -1, **params ) - - return train_and_eval_sklearn_classifier( clf, data, get_predictions = get_predictions ) - \ No newline at end of file + return train_and_eval_sklearn_classifier( clf, data) + + diff --git a/hyperband.py b/hyperband.py index bcb6b94..87b9a02 100644 --- a/hyperband.py +++ b/hyperband.py @@ -6,12 +6,12 @@ class Hyperband: - + def __init__( self, get_params_function, try_params_function ): self.get_params = get_params_function self.try_params = try_params_function - - self.max_iter = 81 # maximum iterations per configuration + + self.max_iter = 200 # maximum iterations per configuration self.eta = 3 # defines configuration downsampling rate (default = 3) self.logeta = lambda x: log( x ) / log( self.eta ) @@ -22,79 +22,79 @@ def __init__( self, get_params_function, try_params_function ): self.counter = 0 self.best_loss = np.inf self.best_counter = -1 - + # can be called multiple times def run( self, skip_last = 0, dry_run = False ): - + for s in reversed( range( self.s_max + 1 )): - + # initial number of configurations - n = int( ceil( self.B / self.max_iter / ( s + 1 ) * self.eta ** s )) - + n = int( ceil( self.B / self.max_iter / ( s + 1 ) * self.eta ** s )) + # initial number of iterations per config - r = self.max_iter * self.eta ** ( -s ) + r = self.max_iter * self.eta ** ( -s ) # n random configurations - T = [ self.get_params() for i in range( n )] - + T = [ self.get_params() for i in range( n )] + for i in range(( s + 1 ) - int( skip_last )): # changed from s + 1 - - # Run each of the n configs for + + # Run each of the n configs for # and keep best (n_configs / eta) configurations - + n_configs = n * self.eta ** ( -i ) n_iterations = r * self.eta ** ( i ) - - print "\n*** {} configurations x {:.1f} iterations each".format( - n_configs, n_iterations ) - + + print("\n*** {} configurations x {:.1f} iterations each".format( + n_configs, n_iterations )) + val_losses = [] early_stops = [] - + for t in T: - + self.counter += 1 - print "\n{} | {} | best so far: {:.4f} (run {})\n".format( - self.counter, ctime(), self.best_loss, self.best_counter ) - + print("\n{} | {} | best so far: {:.4f} (run {})\n".format( + self.counter, ctime(), self.best_loss, self.best_counter )) + start_time = time() - + if dry_run: result = { 'loss': random(), 'log_loss': random(), 'auc': random()} else: result = self.try_params( n_iterations, t ) # <--- - + assert( type( result ) == dict ) assert( 'loss' in result ) - + seconds = int( round( time() - start_time )) - print "\n{} seconds.".format( seconds ) - - loss = result['loss'] + print("\n{} seconds.".format( seconds )) + + loss = result['loss'] val_losses.append( loss ) - + early_stop = result.get( 'early_stop', False ) early_stops.append( early_stop ) - + # keeping track of the best result so far (for display only) # could do it be checking results each time, but hey if loss < self.best_loss: self.best_loss = loss self.best_counter = self.counter - + result['counter'] = self.counter result['seconds'] = seconds result['params'] = t result['iterations'] = n_iterations - + self.results.append( result ) - + # select a number of best configurations for the next loop # filter out early stops, if any indices = np.argsort( val_losses ) T = [ T[i] for i in indices if not early_stops[i]] T = T[ 0:int( n_configs / self.eta )] - + return self.results - + diff --git a/main.py b/main.py index d4b8d6b..954a259 100644 --- a/main.py +++ b/main.py @@ -4,7 +4,7 @@ "includes displaying best results and saving to a file" import sys -import cPickle as pickle +import pickle from pprint import pprint from hyperband import Hyperband @@ -17,32 +17,32 @@ #from defs.keras_mlp import get_params, try_params #from defs.polylearn_fm import get_params, try_params #from defs.polylearn_pn import get_params, try_params -#from defs.xgb import get_params, try_params -from defs.meta import get_params, try_params +from defs.xgb import get_params, try_params +#from defs.meta import get_params, try_params try: output_file = sys.argv[1] if not output_file.endswith( '.pkl' ): - output_file += '.pkl' + output_file += '.pkl' except IndexError: output_file = 'results.pkl' - -print "Will save results to", output_file + +print("Will save results to", output_file) # hb = Hyperband( get_params, try_params ) results = hb.run( skip_last = 1 ) -print "{} total, best:\n".format( len( results )) +print("{} total, best:\n".format( len( results ))) for r in sorted( results, key = lambda x: x['loss'] )[:5]: - print "loss: {:.2%} | {} seconds | {:.1f} iterations | run {} ".format( - r['loss'], r['seconds'], r['iterations'], r['counter'] ) + print("loss: {:.2%} | {} seconds | {:.1f} iterations | run {} ".format( + r['loss'], r['seconds'], r['iterations'], r['counter'] )) pprint( r['params'] ) - print + print() -print "saving..." +print("saving...") with open( output_file, 'wb' ) as f: - pickle.dump( results, f ) \ No newline at end of file + pickle.dump( results, f ) From 7c4749af9cca252b67ba384df5b0c375e0a0e534 Mon Sep 17 00:00:00 2001 From: Kirill Dubovikov Date: Sun, 25 Jun 2017 12:29:29 +0300 Subject: [PATCH 2/5] added future imports for Python 2.x support and refactored module docstrings --- common_defs.py | 3 ++- defs/xgb.py | 5 +++-- hyperband.py | 1 + load_data.py | 35 ++++++++++++++++++++++++++++++----- main.py | 5 +++-- 5 files changed, 39 insertions(+), 10 deletions(-) diff --git a/common_defs.py b/common_defs.py index db1cbe4..1d531a1 100644 --- a/common_defs.py +++ b/common_defs.py @@ -1,4 +1,5 @@ -"imports and definitions shared by various defs files" +from __future__ import print_function +"""imports and definitions shared by various defs files""" import numpy as np diff --git a/defs/xgb.py b/defs/xgb.py index d0e8af2..8a32663 100644 --- a/defs/xgb.py +++ b/defs/xgb.py @@ -1,5 +1,6 @@ -"function (and parameter space) definitions for hyperband" -"binary classification with XGBoost" +from __future__ import print_function +"""function (and parameter space) definitions for hyperband +binary classification with XGBoost""" from common_defs import * diff --git a/hyperband.py b/hyperband.py index 87b9a02..12feb1d 100644 --- a/hyperband.py +++ b/hyperband.py @@ -1,3 +1,4 @@ +from __future__ import print_function import numpy as np from random import random diff --git a/load_data.py b/load_data.py index d714348..3c57983 100644 --- a/load_data.py +++ b/load_data.py @@ -2,14 +2,39 @@ "x_train, y_train, x_test, y_test - numpy arrays" "defs files import data from here" -import cPickle as pickle +import pickle +import pandas as pd +import numpy as np +from sklearn.model_selection import train_test_split -data_file = 'data/classification.pkl' +type_map = {'smoke': np.float, 'alco': np.float, 'active': np.float} +train = pd.read_csv('../data/raw/train.csv', sep=';') -print "loading..." +def cat_col(data, col, bins): + hist = np.histogram(data[col], bins=bins) + data[col] = np.digitize(data[col], hist[1]) + return data[col] -with open( data_file, 'rb' ) as f: - data = pickle.load( f ) +def preprocess_data(data): + result = data.copy() + result.drop(result[(result['ap_hi'] < 0) | (result['ap_lo'] < 0)].index) # ap < 0 is clearly an error + result.drop(result[(result['ap_hi'] >= 300) | (result['ap_lo'] >= 200)].index) # ap < 0 is clearly an error + result = pd.concat([result, pd.get_dummies(data['cholesterol'], prefix='chol')], axis=1) + result['age'] /= 365 + result['gender'] = result['gender'] == 1 + result.drop(['id', 'cardio', 'cholesterol'], inplace=True, axis=1) + result['age'] = cat_col(result, 'age', 40) + result['height'] = cat_col(result, 'height', 40) + result['weight'] = cat_col(result, 'weight', 40) + + return result + +x = preprocess_data(train) +y = train['cardio'] + +x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, stratify=y) + +data = {'x_train': x_train, 'y_train': y_train, 'x_test': x_test, 'y_test': y_test} diff --git a/main.py b/main.py index 954a259..458a5eb 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,8 @@ #!/usr/bin/env python +from __future__ import print_function -"a more polished example of using hyperband" -"includes displaying best results and saving to a file" +"""a more polished example of using Hyperband +includes displaying best results and saving to a file""" import sys import pickle From ceb02afb79737af87368c97dab68007aa84e3ee8 Mon Sep 17 00:00:00 2001 From: James Winegar Date: Sat, 9 Dec 2017 00:39:22 -0600 Subject: [PATCH 3/5] 2to3 started, need to identify python 2 adjustments --- common_defs.py | 3 ++- defs/gb.py | 3 ++- defs/keras_mlp.py | 19 ++++++++++--------- defs/meta.py | 7 ++++--- defs/polylearn_fm.py | 3 ++- defs/polylearn_fm_pn.py | 5 +++-- defs/polylearn_pn.py | 3 ++- defs/rf.py | 3 ++- defs/rf_xt.py | 3 ++- defs/sgd.py | 3 ++- defs/xgb.py | 5 +++-- defs/xt.py | 3 ++- defs_regression/gb.py | 3 ++- defs_regression/keras_mlp.py | 19 ++++++++++--------- defs_regression/meta.py | 3 ++- defs_regression/polylearn_fm.py | 3 ++- defs_regression/polylearn_fm_pn.py | 5 +++-- defs_regression/polylearn_pn.py | 3 ++- defs_regression/rf.py | 3 ++- defs_regression/rf_xt.py | 3 ++- defs_regression/sgd.py | 3 ++- defs_regression/xt.py | 3 ++- hyperband.py | 3 ++- load_data.py | 6 +++--- load_data_for_regression.py | 3 ++- main.py | 5 +++-- main_regression.py | 15 ++++++++------- main_simple.py | 1 + 28 files changed, 84 insertions(+), 57 deletions(-) diff --git a/common_defs.py b/common_defs.py index 1d531a1..add7714 100644 --- a/common_defs.py +++ b/common_defs.py @@ -1,3 +1,4 @@ + from __future__ import print_function """imports and definitions shared by various defs files""" @@ -23,7 +24,7 @@ def handle_integers( params ): new_params = {} - for k, v in params.items(): + for k, v in list(params.items()): if type( v ) == float and int( v ) == v: new_params[k] = int( v ) else: diff --git a/defs/gb.py b/defs/gb.py index a7c47ca..b6ea24b 100644 --- a/defs/gb.py +++ b/defs/gb.py @@ -1,3 +1,4 @@ +from __future__ import print_function "function (and parameter space) definitions for hyperband" "binary classification with gradient boosting" @@ -33,7 +34,7 @@ def get_params(): def try_params( n_iterations, params ): n_estimators = int( round( n_iterations * trees_per_iteration )) - print "n_estimators:", n_estimators + print("n_estimators:", n_estimators) pprint( params ) clf = GB( n_estimators = n_estimators, verbose = 0, **params ) diff --git a/defs/keras_mlp.py b/defs/keras_mlp.py index b838746..8642cd3 100644 --- a/defs/keras_mlp.py +++ b/defs/keras_mlp.py @@ -1,3 +1,4 @@ +from __future__ import print_function "function (and parameter space) definitions for hyperband" "binary classification with Keras (multilayer perceptron)" @@ -55,22 +56,22 @@ def get_params(): # print hidden layers config in readable way def print_layers( params ): for i in range( 1, params['n_layers'] + 1 ): - print "layer {} | size: {:>3} | activation: {:<7} | extras: {}".format( i, + print("layer {} | size: {:>3} | activation: {:<7} | extras: {}".format( i, params['layer_{}_size'.format( i )], params['layer_{}_activation'.format( i )], - params['layer_{}_extras'.format( i )]['name'] ), + params['layer_{}_extras'.format( i )]['name'] ), end=' ') if params['layer_{}_extras'.format( i )]['name'] == 'dropout': - print "- rate: {:.1%}".format( params['layer_{}_extras'.format( i )]['rate'] ), - print + print("- rate: {:.1%}".format( params['layer_{}_extras'.format( i )]['rate'] ), end=' ') + print() def print_params( params ): - pprint({ k: v for k, v in params.items() if not k.startswith( 'layer_' )}) + pprint({ k: v for k, v in list(params.items()) if not k.startswith( 'layer_' )}) print_layers( params ) - print + print() def try_params( n_iterations, params ): - print "iterations:", n_iterations + print("iterations:", n_iterations) print_params( params ) y_train = data['y_train'] @@ -129,7 +130,7 @@ def try_params( n_iterations, params ): auc = AUC( y_train, p ) acc = accuracy( y_train, np.round( p )) - print "\n# training | log loss: {:.2%}, AUC: {:.2%}, accuracy: {:.2%}".format( ll, auc, acc ) + print("\n# training | log loss: {:.2%}, AUC: {:.2%}, accuracy: {:.2%}".format( ll, auc, acc )) # @@ -139,7 +140,7 @@ def try_params( n_iterations, params ): auc = AUC( y_test, p ) acc = accuracy( y_test, np.round( p )) - print "# testing | log loss: {:.2%}, AUC: {:.2%}, accuracy: {:.2%}".format( ll, auc, acc ) + print("# testing | log loss: {:.2%}, AUC: {:.2%}, accuracy: {:.2%}".format( ll, auc, acc )) return { 'loss': ll, 'log_loss': ll, 'auc': auc, 'early_stop': model.stop_training } diff --git a/defs/meta.py b/defs/meta.py index 8d43601..ddd49f5 100644 --- a/defs/meta.py +++ b/defs/meta.py @@ -1,3 +1,4 @@ +from __future__ import print_function # meta classifier from common_defs import * @@ -5,8 +6,8 @@ # import all the functions for m in models: - exec( "from defs.{} import get_params as get_params_{}" ).format( m, m ) - exec( "from defs.{} import try_params as try_params_{}" ).format( m, m ) + exec(( "from defs.{} import get_params as get_params_{}" ).format( m, m )) + exec(( "from defs.{} import try_params as try_params_{}" ).format( m, m )) space = { 'model': hp.choice( 'model', models ) } @@ -20,7 +21,7 @@ def get_params(): def try_params( n_iterations, params ): params_ = dict( params ) m = params_.pop( 'model' ) - print m + print(m) return eval( "try_params_{}( n_iterations, params_ )".format( m )) diff --git a/defs/polylearn_fm.py b/defs/polylearn_fm.py index ca413a7..7fa7c5b 100644 --- a/defs/polylearn_fm.py +++ b/defs/polylearn_fm.py @@ -1,3 +1,4 @@ +from __future__ import print_function "function (and parameter space) definitions for hyperband" "binary classification with polylearn FM" @@ -36,7 +37,7 @@ def get_params(): def try_params( n_iterations, params ): max_iter = int( round( n_iterations * iters_per_iteration )) - print "max_iter:", max_iter + print("max_iter:", max_iter) pprint( params ) if params['scaler']: diff --git a/defs/polylearn_fm_pn.py b/defs/polylearn_fm_pn.py index f028b78..87f17a3 100644 --- a/defs/polylearn_fm_pn.py +++ b/defs/polylearn_fm_pn.py @@ -1,3 +1,4 @@ +from __future__ import print_function "function (and parameter space) definitions for hyperband" "binary classification with polylearn FM/PN" @@ -47,7 +48,7 @@ def get_params(): def try_params( n_iterations, params ): max_iter = int( round( n_iterations * iters_per_iteration )) - print "max_iter:", max_iter + print("max_iter:", max_iter) if params['scaler']: scaler = eval( "{}()".format( params['scaler'] )) @@ -72,7 +73,7 @@ def try_params( n_iterations, params ): params_.pop( 'classifier' ) params_.update( local_params ) - print classifier + print(classifier) pprint( params_ ) params_.pop( 'scaler' ) diff --git a/defs/polylearn_pn.py b/defs/polylearn_pn.py index 0d57104..6fa15b7 100644 --- a/defs/polylearn_pn.py +++ b/defs/polylearn_pn.py @@ -1,3 +1,4 @@ +from __future__ import print_function "function (and parameter space) definitions for hyperband" "binary classification with polylearn polynomial networks" @@ -36,7 +37,7 @@ def get_params(): def try_params( n_iterations, params ): max_iter = int( round( n_iterations * iters_per_iteration )) - print "max_iter:", max_iter + print("max_iter:", max_iter) pprint( params ) if params['scaler']: diff --git a/defs/rf.py b/defs/rf.py index 017b80f..427d09a 100644 --- a/defs/rf.py +++ b/defs/rf.py @@ -1,3 +1,4 @@ +from __future__ import print_function "function (and parameter space) definitions for hyperband" "binary classification with random forest" @@ -32,7 +33,7 @@ def get_params(): def try_params( n_iterations, params ): n_estimators = int( round( n_iterations * trees_per_iteration )) - print "n_estimators:", n_estimators + print("n_estimators:", n_estimators) pprint( params ) clf = RF( n_estimators = n_estimators, verbose = 0, n_jobs = -1, **params ) diff --git a/defs/rf_xt.py b/defs/rf_xt.py index e5593ff..030e199 100644 --- a/defs/rf_xt.py +++ b/defs/rf_xt.py @@ -1,3 +1,4 @@ +from __future__ import print_function "function (and parameter space) definitions for hyperband" "binary classification with random forest / extra trees" "both have the same parameters" @@ -35,7 +36,7 @@ def get_params(): def try_params( n_iterations, params ): n_estimators = int( round( n_iterations * trees_per_iteration )) - print "n_estimators:", n_estimators + print("n_estimators:", n_estimators) pprint( params ) classifier = params['classifier'] diff --git a/defs/sgd.py b/defs/sgd.py index fe06e35..05e5728 100644 --- a/defs/sgd.py +++ b/defs/sgd.py @@ -1,3 +1,4 @@ +from __future__ import print_function "function (and parameter space) definitions for hyperband" "binary classification with extremely linear SGD classifier" @@ -36,7 +37,7 @@ def get_params(): def try_params( n_iterations, params ): n_iterations = int( round( n_iterations )) - print "n_iterations:", n_iterations + print("n_iterations:", n_iterations) pprint( params ) if params['scaler']: diff --git a/defs/xgb.py b/defs/xgb.py index f753af0..35003a5 100644 --- a/defs/xgb.py +++ b/defs/xgb.py @@ -1,4 +1,5 @@ from __future__ import print_function + """function (and parameter space) definitions for hyperband binary classification with XGBoost""" @@ -64,7 +65,7 @@ def get_params(): params = sample( space ) - params = { k: v for k, v in params.items() if v is not 'default' } + params = { k: v for k, v in list(params.items()) if v is not 'default' } return handle_integers( params ) # @@ -77,6 +78,6 @@ def try_params( n_iterations, params, get_predictions = False ): clf = XGB( n_estimators = n_estimators, nthread = -1, **params ) - return train_and_eval_sklearn_classifier( clf, data ) + return train_and_eval_sklearn_classifier( clf, data ) diff --git a/defs/xt.py b/defs/xt.py index 212f2d3..a534032 100644 --- a/defs/xt.py +++ b/defs/xt.py @@ -1,3 +1,4 @@ +from __future__ import print_function "function (and parameter space) definitions for hyperband" "binary classification with extremely randomized trees" @@ -32,7 +33,7 @@ def get_params(): def try_params( n_iterations, params ): n_estimators = int( round( n_iterations * trees_per_iteration )) - print "n_estimators:", n_estimators + print("n_estimators:", n_estimators) pprint( params ) clf = XT( n_estimators = n_estimators, verbose = 0, n_jobs = -1, **params ) diff --git a/defs_regression/gb.py b/defs_regression/gb.py index e2b975e..9512943 100644 --- a/defs_regression/gb.py +++ b/defs_regression/gb.py @@ -1,3 +1,4 @@ +from __future__ import print_function "function (and parameter space) definitions for hyperband" "regression with gradient boosting" @@ -37,7 +38,7 @@ def get_params(): def try_params( n_iterations, params ): n_estimators = int( round( n_iterations * trees_per_iteration )) - print "n_estimators:", n_estimators + print("n_estimators:", n_estimators) pprint( params ) clf = GB( n_estimators = n_estimators, verbose = 0, **params ) diff --git a/defs_regression/keras_mlp.py b/defs_regression/keras_mlp.py index 40fd106..2ac370a 100644 --- a/defs_regression/keras_mlp.py +++ b/defs_regression/keras_mlp.py @@ -1,3 +1,4 @@ +from __future__ import print_function "function (and parameter space) definitions for hyperband" "regression with Keras (multilayer perceptron)" @@ -59,22 +60,22 @@ def get_params(): # print hidden layers config in readable way def print_layers( params ): for i in range( 1, params['n_layers'] + 1 ): - print "layer {} | size: {:>3} | activation: {:<7} | extras: {}".format( i, + print("layer {} | size: {:>3} | activation: {:<7} | extras: {}".format( i, params['layer_{}_size'.format( i )], params['layer_{}_activation'.format( i )], - params['layer_{}_extras'.format( i )]['name'] ), + params['layer_{}_extras'.format( i )]['name'] ), end=' ') if params['layer_{}_extras'.format( i )]['name'] == 'dropout': - print "- rate: {:.1%}".format( params['layer_{}_extras'.format( i )]['rate'] ), - print + print("- rate: {:.1%}".format( params['layer_{}_extras'.format( i )]['rate'] ), end=' ') + print() def print_params( params ): - pprint({ k: v for k, v in params.items() if not k.startswith( 'layer_' )}) + pprint({ k: v for k, v in list(params.items()) if not k.startswith( 'layer_' )}) print_layers( params ) - print + print() def try_params( n_iterations, params ): - print "iterations:", n_iterations + print("iterations:", n_iterations) print_params( params ) y_train = data['y_train'] @@ -134,7 +135,7 @@ def try_params( n_iterations, params ): mae = MAE( y_train, p ) - print "\n# training | RMSE: {:.4f}, MAE: {:.4f}".format( rmse, mae ) + print("\n# training | RMSE: {:.4f}, MAE: {:.4f}".format( rmse, mae )) # @@ -144,7 +145,7 @@ def try_params( n_iterations, params ): rmse = sqrt( mse ) mae = MAE( y_test, p ) - print "# testing | RMSE: {:.4f}, MAE: {:.4f}".format( rmse, mae ) + print("# testing | RMSE: {:.4f}, MAE: {:.4f}".format( rmse, mae )) return { 'loss': rmse, 'rmse': rmse, 'mae': mae, 'early_stop': model.stop_training } diff --git a/defs_regression/meta.py b/defs_regression/meta.py index b4727b1..dee24fa 100644 --- a/defs_regression/meta.py +++ b/defs_regression/meta.py @@ -1,3 +1,4 @@ +from __future__ import print_function # meta regressor from common_defs import * @@ -21,7 +22,7 @@ def try_params( n_iterations, params ): params_ = dict( params ) r = params_.pop( 'regressor' ) - print r + print(r) return eval( "try_params_{}( n_iterations, params_ )".format( r )) diff --git a/defs_regression/polylearn_fm.py b/defs_regression/polylearn_fm.py index e0e1cd6..6cee0d9 100644 --- a/defs_regression/polylearn_fm.py +++ b/defs_regression/polylearn_fm.py @@ -1,3 +1,4 @@ +from __future__ import print_function "function (and parameter space) definitions for hyperband" "binary classification with polylearn FM" @@ -36,7 +37,7 @@ def get_params(): def try_params( n_iterations, params ): max_iter = int( round( n_iterations * iters_per_iteration )) - print "max_iter:", max_iter + print("max_iter:", max_iter) pprint( params ) if params['scaler']: diff --git a/defs_regression/polylearn_fm_pn.py b/defs_regression/polylearn_fm_pn.py index a4f1842..1f2ad45 100644 --- a/defs_regression/polylearn_fm_pn.py +++ b/defs_regression/polylearn_fm_pn.py @@ -1,3 +1,4 @@ +from __future__ import print_function "function (and parameter space) definitions for hyperband" "regression with polylearn FM/PN" @@ -47,7 +48,7 @@ def get_params(): def try_params( n_iterations, params ): max_iter = int( round( n_iterations * iters_per_iteration )) - print "max_iter:", max_iter + print("max_iter:", max_iter) if params['scaler']: scaler = eval( "{}()".format( params['scaler'] )) @@ -72,7 +73,7 @@ def try_params( n_iterations, params ): params_.pop( 'classifier' ) params_.update( local_params ) - print classifier + print(classifier) pprint( params_ ) params_.pop( 'scaler' ) diff --git a/defs_regression/polylearn_pn.py b/defs_regression/polylearn_pn.py index c1675dd..1a5ad22 100644 --- a/defs_regression/polylearn_pn.py +++ b/defs_regression/polylearn_pn.py @@ -1,3 +1,4 @@ +from __future__ import print_function "function (and parameter space) definitions for hyperband" "regression with polylearn polynomial networks" @@ -36,7 +37,7 @@ def get_params(): def try_params( n_iterations, params ): max_iter = int( round( n_iterations * iters_per_iteration )) - print "max_iter:", max_iter + print("max_iter:", max_iter) pprint( params ) if params['scaler']: diff --git a/defs_regression/rf.py b/defs_regression/rf.py index 3b9482c..7c59c09 100644 --- a/defs_regression/rf.py +++ b/defs_regression/rf.py @@ -1,3 +1,4 @@ +from __future__ import print_function "function (and parameter space) definitions for hyperband" "regression with random forest" @@ -31,7 +32,7 @@ def get_params(): def try_params( n_iterations, params ): n_estimators = int( round( n_iterations * trees_per_iteration )) - print "n_estimators:", n_estimators + print("n_estimators:", n_estimators) pprint( params ) clf = RF( n_estimators = n_estimators, verbose = 0, n_jobs = -1, **params ) diff --git a/defs_regression/rf_xt.py b/defs_regression/rf_xt.py index 82698b0..4f92a17 100644 --- a/defs_regression/rf_xt.py +++ b/defs_regression/rf_xt.py @@ -1,3 +1,4 @@ +from __future__ import print_function "function (and parameter space) definitions for hyperband" "regression with random forest / extra trees" "both have the same parameters" @@ -34,7 +35,7 @@ def get_params(): def try_params( n_iterations, params ): n_estimators = int( round( n_iterations * trees_per_iteration )) - print "n_estimators:", n_estimators + print("n_estimators:", n_estimators) pprint( params ) classifier = params['classifier'] diff --git a/defs_regression/sgd.py b/defs_regression/sgd.py index 935a1f7..e756e68 100644 --- a/defs_regression/sgd.py +++ b/defs_regression/sgd.py @@ -1,3 +1,4 @@ +from __future__ import print_function "function (and parameter space) definitions for hyperband" "regression with linear SGD regressor" @@ -37,7 +38,7 @@ def get_params(): def try_params( n_iterations, params ): n_iterations = int( round( n_iterations )) - print "n_iterations:", n_iterations + print("n_iterations:", n_iterations) pprint( params ) if params['scaler']: diff --git a/defs_regression/xt.py b/defs_regression/xt.py index 3771160..4be0fbf 100644 --- a/defs_regression/xt.py +++ b/defs_regression/xt.py @@ -1,3 +1,4 @@ +from __future__ import print_function "function (and parameter space) definitions for hyperband" "regression with extremely randomized trees" @@ -31,7 +32,7 @@ def get_params(): def try_params( n_iterations, params ): n_estimators = int( round( n_iterations * trees_per_iteration )) - print "n_estimators:", n_estimators + print("n_estimators:", n_estimators) pprint( params ) clf = XT( n_estimators = n_estimators, verbose = 0, n_jobs = -1, **params ) diff --git a/hyperband.py b/hyperband.py index f584efa..981171d 100644 --- a/hyperband.py +++ b/hyperband.py @@ -1,3 +1,4 @@ + from __future__ import print_function import numpy as np @@ -28,7 +29,7 @@ def __init__( self, get_params_function, try_params_function ): # can be called multiple times def run( self, skip_last = 0, dry_run = False ): - for s in reversed( range( self.s_max + 1 )): + for s in reversed( list(range( self.s_max + 1))): # initial number of configurations n = int( ceil( self.B / self.max_iter / ( s + 1 ) * self.eta ** s )) diff --git a/load_data.py b/load_data.py index eb5ba6e..f93571e 100644 --- a/load_data.py +++ b/load_data.py @@ -5,12 +5,12 @@ """ # this particular example loads data from a pickle file - -import cPickle as pickle +from __future__ import print_function +import pickle as pickle data_file = 'data/classification.pkl' -print "loading..." +print("loading...") with open( data_file, 'rb' ) as f: data = pickle.load( f ) diff --git a/load_data_for_regression.py b/load_data_for_regression.py index ffb5def..cab6743 100644 --- a/load_data_for_regression.py +++ b/load_data_for_regression.py @@ -1,10 +1,11 @@ import numpy as np +from __future__ import print_function train_file = 'data/kin8nm/train.csv' valid_file = 'data/kin8nm/validation.csv' test_file = 'data/kin8nm/test.csv' -print "loading data..." +print("loading data...") train = np.loadtxt( open( train_file ), delimiter = "," ) valid = np.loadtxt( open( valid_file ), delimiter = "," ) diff --git a/main.py b/main.py index 458a5eb..eeb4702 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from __future__ import print_function + """a more polished example of using Hyperband includes displaying best results and saving to a file""" @@ -18,8 +19,8 @@ #from defs.keras_mlp import get_params, try_params #from defs.polylearn_fm import get_params, try_params #from defs.polylearn_pn import get_params, try_params -from defs.xgb import get_params, try_params -#from defs.meta import get_params, try_params +#from defs.xgb import get_params, try_params +from defs.meta import get_params, try_params try: output_file = sys.argv[1] diff --git a/main_regression.py b/main_regression.py index 2cd016b..5141a3c 100644 --- a/main_regression.py +++ b/main_regression.py @@ -1,9 +1,10 @@ #!/usr/bin/env python +from __future__ import print_function "A regression example. Mostly the same, only importing from defs_regression." import sys -import cPickle as pickle +import pickle as pickle from pprint import pprint from hyperband import Hyperband @@ -16,22 +17,22 @@ except IndexError: output_file = 'results.pkl' -print "Will save results to", output_file +print("Will save results to", output_file) # hb = Hyperband( get_params, try_params ) results = hb.run( skip_last = 1 ) -print "{} total, best:\n".format( len( results )) +print("{} total, best:\n".format( len( results ))) for r in sorted( results, key = lambda x: x['loss'] )[:5]: - print "loss: {:.2%} | {} seconds | {:.1f} iterations | run {} ".format( - r['loss'], r['seconds'], r['iterations'], r['counter'] ) + print("loss: {:.2%} | {} seconds | {:.1f} iterations | run {} ".format( + r['loss'], r['seconds'], r['iterations'], r['counter'] )) pprint( r['params'] ) - print + print() -print "saving..." +print("saving...") with open( output_file, 'wb' ) as f: pickle.dump( results, f ) \ No newline at end of file diff --git a/main_simple.py b/main_simple.py index 8a46c0d..4914e2e 100644 --- a/main_simple.py +++ b/main_simple.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +from __future__ import print_function "bare-bones demonstration of using hyperband to tune sklearn GBT" From 8369d8510fe8b55b0f261f68c553814993033a94 Mon Sep 17 00:00:00 2001 From: James Winegar Date: Sat, 9 Dec 2017 00:55:16 -0600 Subject: [PATCH 4/5] fix pickle for python 2 and 3 --- defs/keras_mlp.py | 64 +++++++++++++++++++++++----------------------- load_data.py | 6 ++++- main_regression.py | 16 ++++++++---- 3 files changed, 48 insertions(+), 38 deletions(-) diff --git a/defs/keras_mlp.py b/defs/keras_mlp.py index 8642cd3..3b18359 100644 --- a/defs/keras_mlp.py +++ b/defs/keras_mlp.py @@ -17,33 +17,33 @@ # -# TODO: advanced activations - 'leakyrelu', 'prelu', 'elu', 'thresholdedrelu', 'srelu' +# TODO: advanced activations - 'leakyrelu', 'prelu', 'elu', 'thresholdedrelu', 'srelu' max_layers = 5 space = { - 'scaler': hp.choice( 's', + 'scaler': hp.choice( 's', ( None, 'StandardScaler', 'RobustScaler', 'MinMaxScaler', 'MaxAbsScaler' )), 'n_layers': hp.quniform( 'l', 1, max_layers, 1 ), #'layer_size': hp.quniform( 'ls', 5, 100, 1 ), - #'activation': hp.choice( 'a', ( 'relu', 'sigmoid', 'tanh' )), - 'init': hp.choice( 'i', ( 'uniform', 'normal', 'glorot_uniform', + #'activation': hp.choice( 'a', ( 'relu', 'sigmoid', 'tanh' )) + 'init': hp.choice( 'i', ( 'uniform', 'normal', 'glorot_uniform', 'glorot_normal', 'he_uniform', 'he_normal' )), 'batch_size': hp.choice( 'bs', ( 16, 32, 64, 128, 256 )), - 'optimizer': hp.choice( 'o', ( 'rmsprop', 'adagrad', 'adadelta', 'adam', 'adamax' )) + 'optimizer': hp.choice( 'o', ( 'rmsprop', 'adagrad', 'adadelta', 'adam', 'adamax' )) } # for each hidden layer, we choose size, activation and extras individually for i in range( 1, max_layers + 1 ): space[ 'layer_{}_size'.format( i )] = hp.quniform( 'ls{}'.format( i ), 2, 200, 1 ) - space[ 'layer_{}_activation'.format( i )] = hp.choice( 'a{}'.format( i ), + space[ 'layer_{}_activation'.format( i )] = hp.choice( 'a{}'.format( i ), ( 'relu', 'sigmoid', 'tanh' )) - space[ 'layer_{}_extras'.format( i )] = hp.choice( 'e{}'.format( i ), ( - { 'name': 'dropout', 'rate': hp.uniform( 'd{}'.format( i ), 0.1, 0.5 )}, + space[ 'layer_{}_extras'.format( i )] = hp.choice( 'e{}'.format( i ), ( + { 'name': 'dropout', 'rate': hp.uniform( 'd{}'.format( i ), 0.1, 0.5 )}, { 'name': 'batchnorm' }, - { 'name': None } )) - + { 'name': None } )) + def get_params(): params = sample( space ) @@ -57,7 +57,7 @@ def get_params(): def print_layers( params ): for i in range( 1, params['n_layers'] + 1 ): print("layer {} | size: {:>3} | activation: {:<7} | extras: {}".format( i, - params['layer_{}_size'.format( i )], + params['layer_{}_size'.format( i )], params['layer_{}_activation'.format( i )], params['layer_{}_extras'.format( i )]['name'] ), end=' ') if params['layer_{}_extras'.format( i )]['name'] == 'dropout': @@ -67,16 +67,16 @@ def print_layers( params ): def print_params( params ): pprint({ k: v for k, v in list(params.items()) if not k.startswith( 'layer_' )}) print_layers( params ) - print() + print() def try_params( n_iterations, params ): - + print("iterations:", n_iterations) print_params( params ) - + y_train = data['y_train'] y_test = data['y_test'] - + if params['scaler']: scaler = eval( "{}()".format( params['scaler'] )) x_train_ = scaler.fit_transform( data['x_train'].astype( float )) @@ -84,29 +84,29 @@ def try_params( n_iterations, params ): else: x_train_ = data['x_train'] x_test_ = data['x_test'] - + input_dim = x_train_.shape[1] model = Sequential() - model.add( Dense( params['layer_1_size'], init = params['init'], + model.add( Dense( params['layer_1_size'], init = params['init'], activation = params['layer_1_activation'], input_dim = input_dim )) - + for i in range( int( params['n_layers'] ) - 1 ): - + extras = 'layer_{}_extras'.format( i + 1 ) - + if params[extras]['name'] == 'dropout': model.add( Dropout( params[extras]['rate'] )) elif params[extras]['name'] == 'batchnorm': model.add( BatchNorm()) - - model.add( Dense( params['layer_{}_size'.format( i + 2 )], init = params['init'], + + model.add( Dense( params['layer_{}_size'.format( i + 2 )], init = params['init'], activation = params['layer_{}_activation'.format( i + 2 )])) - + model.add( Dense( 1, init = params['init'], activation = 'sigmoid' )) model.compile( optimizer = params['optimizer'], loss = 'binary_crossentropy' ) - + #print model.summary() # @@ -114,18 +114,18 @@ def try_params( n_iterations, params ): validation_data = ( x_test_, y_test ) early_stopping = EarlyStopping( monitor = 'val_loss', patience = 5, verbose = 0 ) - + history = model.fit( x_train_, y_train, nb_epoch = int( round( n_iterations )), - batch_size = params['batch_size'], - shuffle = False, - validation_data = validation_data, - callbacks = [ early_stopping ]) - + batch_size = params['batch_size'], + shuffle = False, + validation_data = validation_data, + callbacks = [ early_stopping ]) + # - + p = model.predict_proba( x_train_, batch_size = params['batch_size'] ) - + ll = log_loss( y_train, p ) auc = AUC( y_train, p ) acc = accuracy( y_train, np.round( p )) diff --git a/load_data.py b/load_data.py index f93571e..901af0c 100644 --- a/load_data.py +++ b/load_data.py @@ -6,7 +6,11 @@ # this particular example loads data from a pickle file from __future__ import print_function -import pickle as pickle +from sys import version_info +if version_info >= (3,): + import pickle as pickle +else: + import cPickle as pickle data_file = 'data/classification.pkl' diff --git a/main_regression.py b/main_regression.py index 5141a3c..fa76e8b 100644 --- a/main_regression.py +++ b/main_regression.py @@ -4,19 +4,25 @@ "A regression example. Mostly the same, only importing from defs_regression." import sys -import pickle as pickle from pprint import pprint + +from sys import version_info +if version_info >= (3,): + import pickle as pickle +else: + import cPickle as pickle + from hyperband import Hyperband from defs_regression.meta import get_params, try_params try: output_file = sys.argv[1] if not output_file.endswith( '.pkl' ): - output_file += '.pkl' + output_file += '.pkl' except IndexError: output_file = 'results.pkl' - + print("Will save results to", output_file) # @@ -27,7 +33,7 @@ print("{} total, best:\n".format( len( results ))) for r in sorted( results, key = lambda x: x['loss'] )[:5]: - print("loss: {:.2%} | {} seconds | {:.1f} iterations | run {} ".format( + print("loss: {:.2%} | {} seconds | {:.1f} iterations | run {} ".format( r['loss'], r['seconds'], r['iterations'], r['counter'] )) pprint( r['params'] ) print() @@ -35,4 +41,4 @@ print("saving...") with open( output_file, 'wb' ) as f: - pickle.dump( results, f ) \ No newline at end of file + pickle.dump( results, f ) From 312f8fa9b2ae79ff1fcc0d2c473dc6480a29dc2f Mon Sep 17 00:00:00 2001 From: James Winegar Date: Sat, 9 Dec 2017 01:11:41 -0600 Subject: [PATCH 5/5] fix newer commits for python 2/3 --- defs_regression/xgb.py | 13 ++++++------- show_results.py | 15 ++++++++++----- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/defs_regression/xgb.py b/defs_regression/xgb.py index 016b26b..cd30c1c 100644 --- a/defs_regression/xgb.py +++ b/defs_regression/xgb.py @@ -2,7 +2,7 @@ "regression with XGBoost" from common_defs import * - +from __future__ import print_function # a dict with x_train, y_train, x_test, y_test from load_data_for_regression import data @@ -25,7 +25,7 @@ 'default', hp.quniform( 'mcw_', 1, 10, 1 ) ]), - + 'subsample': hp.choice( 'ss', [ 'default', hp.uniform( 'ss_', 0.5, 1.0 ) @@ -37,7 +37,7 @@ 'colsample_bylevel': hp.choice( 'cbl', [ 'default', hp.uniform( 'cbl_', 0.5, 1.0 ) - ]), + ]), 'gamma': hp.choice( 'g', [ 'default', hp.uniform( 'g_', 0, 1 ) @@ -63,7 +63,7 @@ def get_params(): params = sample( space ) - params = { k: v for k, v in params.items() if v is not 'default' } + params = { k: v for k, v in list(params.items()) if v is not 'default' } return handle_integers( params ) # @@ -71,11 +71,10 @@ def get_params(): def try_params( n_iterations, params, get_predictions = False ): n_estimators = int( round( n_iterations * trees_per_iteration )) - print "n_estimators:", n_estimators + print("n_estimators:", n_estimators) pprint( params ) model = XGB( n_estimators = n_estimators, nthread = -1, **params ) - + return train_and_eval_sklearn_regressor( model, data ) - \ No newline at end of file diff --git a/show_results.py b/show_results.py index 20eff36..6c6d3a3 100755 --- a/show_results.py +++ b/show_results.py @@ -3,14 +3,19 @@ "load pickled results, show the best" import sys -import pickle +from __future__ import print_function +from sys import version_info +if version_info >= (3,): + import pickle as pickle +else: + import cPickle as pickle from pprint import pprint try: input_file = sys.argv[1] except IndexError: - print "Usage: python show_results.py []\n" + print("Usage: python show_results.py []\n") raise SystemExit try: @@ -22,7 +27,7 @@ results = pickle.load( i_f ) for r in sorted( results, key = lambda x: x['loss'] )[:results_to_show]: - print "loss: {:.2%} | {} seconds | {:.1f} iterations | run {} ".format( - r['loss'], r['seconds'], r['iterations'], r['counter'] ) + print("loss: {:.2%} | {} seconds | {:.1f} iterations | run {} ".format( + r['loss'], r['seconds'], r['iterations'], r['counter'] )) pprint( r['params'] ) - print \ No newline at end of file + print()