Auto-sklearn baseline v1

.dockerignore

@@ -0,0 +1,13 @@
+# Config & info files
+.pep8speaks.yml
+Dockerfile
+LICENSE
+README.md
+
+# Unnecessary files
+examples
+notebooks
+test
+
+# User data
+data/cache

.gitignore

@@ -1,3 +1,5 @@
+.idea
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -129,4 +131,4 @@ dmypy.json
 .pyre/
 
 # User data
-data/
+/data/cache

Dockerfile

@@ -0,0 +1,30 @@
+# Download base image ubuntu 20.04
+FROM ubuntu:20.04
+
+# For apt to be noninteractive
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_NONINTERACTIVE_SEEN true
+
+# Preseed tzdata, update package index, upgrade packages and install needed software
+RUN truncate -s0 /tmp/preseed.cfg; \
+    echo "tzdata tzdata/Areas select Europe" >> /tmp/preseed.cfg; \
+    echo "tzdata tzdata/Zones/Europe select Berlin" >> /tmp/preseed.cfg; \
+    debconf-set-selections /tmp/preseed.cfg && \
+    rm -f /etc/timezone /etc/localtime && \
+	apt-get update && \
+	apt-get install -y nano  && \
+	apt-get install -y mc && \
+    apt-get install -y python3.9 python3-pip && \
+	apt-get install -y git && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set the workdir
+ENV WORKDIR /home/meta-automl-research
+WORKDIR $WORKDIR
+COPY . $WORKDIR
+
+RUN pip3 install pip && \
+    pip install wheel && \
+    pip install --trusted-host pypi.python.org -r ${WORKDIR}/requirements.txt
+
+ENV PYTHONPATH $WORKDIR

baselines/auto-sklearn/auto-sklearn_baseline.py

@@ -0,0 +1,166 @@
+import csv
+import time
+
+from typing import Any, Tuple, Dict
+
+import numpy as np
+import logging
+
+import autosklearn.classification
+import autosklearn.ensembles
+
+from sklearn import model_selection, metrics
+
+from baselines.automl_baseline import AutoMLBaseline
+from meta_automl.data_preparation.datasets_loaders import OpenMLDatasetsLoader
+from meta_automl.data_preparation.models_loaders import KnowledgeBaseModelsLoader
+from autosklearn.classification import AutoSklearnClassifier
+
+
+class AutoSklearnBaseline(AutoMLBaseline):
+    def __init__(self, ensemble_type, time_limit):
+        self.estimator = AutoSklearnClassifier(
+            ensemble_class=ensemble_type,
+            time_left_for_this_task=time_limit,
+        )
+        self.knowledge_base_loader = KnowledgeBaseModelsLoader()
+
+    @staticmethod
+    def make_quality_metric_estimates(y, predictions, prediction_proba, is_multi_label):
+        """ Compute roc_auc, f1, accuracy, log_loss and precision scores. """
+        results = {
+            'roc_auc': -1 * float(
+                "{:.3f}".format(
+                    metrics.roc_auc_score(
+                        y,
+                        prediction_proba if is_multi_label else predictions,
+                        multi_class='ovr'
+                    )
+                )
+            ),
+            'f1': -1 * float(
+                "{:.3f}".format(
+                    metrics.f1_score(
+                        y,
+                        predictions,
+                        average='macro' if is_multi_label else 'binary'
+                    )
+                )
+            ),
+            'accuracy': -1 * float(
+                "{:.3f}".format(
+                    metrics.accuracy_score(
+                        y,
+                        predictions
+                    )
+                )
+            ),
+            'logloss': float(
+                "{:.3f}".format(
+                    metrics.log_loss(
+                        y,
+                        prediction_proba if is_multi_label else predictions
+                    )
+                )
+            ),
+            'precision': -1 * float(
+                "{:.3f}".format(
+                    metrics.precision_score(
+                        y,
+                        predictions,
+                        average='macro' if is_multi_label else 'binary',
+                        labels=np.unique(predictions)
+                    )
+                )
+            )
+        }
+        return results
+
+    def run(self):
+        """ Fit auto-sklearn meta-optimizer to knowledge base datasets and output a single best model. """
+        dataset_ids_to_load = [
+            dataset_id for dataset_id in self.knowledge_base_loader
+                                             .parse_datasets('test')
+                                             .loc[:, 'dataset_id']
+        ]
+        # dataset_ids_to_load = [dataset_ids_to_load[dataset_ids_to_load.index(41166)]]
+
+        loaded_datasets = OpenMLDatasetsLoader().load(dataset_ids_to_load)
+
+        for iteration, dataset in enumerate(loaded_datasets):
+            logging.log(logging.INFO, f"Loaded dataset name: {dataset.name}")
+            dataset_data = dataset.get_data()
+
+            X_train, X_test, y_train, y_test = model_selection.train_test_split(
+                dataset_data.x,
+                dataset_data.y,
+                test_size=0.2,
+                random_state=42,
+                stratify=dataset_data.y
+            )
+
+            fitting_start_time = time.time()
+            ensemble = self.estimator.fit(X_train, y_train)
+            fitting_time = time.time() - fitting_start_time
+            logging.log(logging.INFO, f"Fitting time is {fitting_time}sec")
+
+            inference_start_time = time.time()
+            predicted_results = self.estimator.predict(X_test)
+            inference_time = time.time() - inference_start_time
+            logging.log(logging.INFO, f"Inference time is {inference_time}sec")
+
+            predicted_probabilities = self.estimator.predict_proba(X_test)
+
+            best_single_model = list(ensemble.show_models().values())[0].get('sklearn_classifier')
+
+            # autosklearn_ensemble = pipeline.show_models()
+            # formatted_ensemble = {
+            #     model_id: {
+            #         'rank': autosklearn_ensemble[model_id].get('rank'),
+            #         'cost': float(f"{autosklearn_ensemble[model_id].get('cost'):.3f}"),
+            #         'ensemble_weight': autosklearn_ensemble[model_id].get('ensemble_weight'),
+            #         'model': autosklearn_ensemble[model_id].get('sklearn_classifier')
+            #     } for model_id in autosklearn_ensemble.keys()
+            # }
+
+            general_run_info = {
+                'dataset_id': dataset.id_,
+                'dataset_name': dataset.name,
+                'run_label': 'Auto-sklearn',
+            }
+
+            is_multilabel_classification = True if len(set(predicted_results)) > 2 else False
+            quality_metric_estimates = AutoSklearnBaseline.make_quality_metric_estimates(
+                y_test,
+                predicted_results,
+                predicted_probabilities,
+                is_multilabel_classification
+            )
+
+            model_dependent_run_info = {
+                'fit_time': float(f'{fitting_time:.1f}'),
+                'inference_time': float(f'{inference_time:.1f}'),
+                'model_str': repr(best_single_model)
+            }
+
+            results = {**general_run_info, **quality_metric_estimates, **model_dependent_run_info}
+
+            # for key in autosklearn_ensemble.keys():
+            #     ensemble_model = autosklearn_ensemble[key]
+            #     formatted_ensemble = results['ensemble']
+            #     for model_id in formatted_ensemble.keys():
+            #         formatted_ensemble[model_id] = ensemble_model.get("rank", None)
+
+            AutoSklearnBaseline.save_on_disk(results.valuess())
+
+            return results
+
+    @staticmethod
+    def save_on_disk(data):
+        with open('data/experimental_data.csv', 'a', newline='') as file:
+            writer = csv.writer(file, delimiter=',')
+            writer.writerow(data)
+
+
+if __name__ == '__main__':
+    AutoSklearnBaseline(autosklearn.ensembles.SingleBest, 600).run()

baselines/auto-sklearn/data/experimental_data.csv

@@ -0,0 +1,57 @@
+1461,bank-marketing,Auto-sklearn,-0.711,-0.535,-0.907,3.34,-0.648,598.0,0.1,"HistGradientBoostingClassifier(early_stopping=True,
+                               l2_regularization=1.7108930238344161e-10,
+                               learning_rate=0.010827728124541558, loss='auto',
+                               max_iter=512, max_leaf_nodes=25,
+                               min_samples_leaf=4, n_iter_no_change=19,
+                               random_state=1,
+                               validation_fraction=0.1759114608225653,
+                               warm_start=True)"
+179,adult,Auto-sklearn,-0.774,-0.91,-0.859,5.077,-0.885,595.3,0.1,"HistGradientBoostingClassifier(early_stopping=True,
+                               l2_regularization=1.7108930238344161e-10,
+                               learning_rate=0.010827728124541558, loss='auto',
+                               max_iter=512, max_leaf_nodes=25,
+                               min_samples_leaf=4, n_iter_no_change=19,
+                               random_state=1,
+                               validation_fraction=0.1759114608225653,
+                               warm_start=True)"
+1464,blood-transfusion-service-center,Auto-sklearn,-0.669,-0.5,-0.8,7.209,-0.625,597.6,0.0,"PassiveAggressiveClassifier(C=0.253246830865058, average=True, max_iter=16,
+                            random_state=1, tol=0.01676578241454229,
+                            warm_start=True)"
+991,car,Auto-sklearn,-1.0,-1.0,-1.0,0.0,-1.0,596.8,0.0,"HistGradientBoostingClassifier(early_stopping=True,
+                               l2_regularization=1.9280388598217333e-10,
+                               learning_rate=0.24233932723531437, loss='auto',
+                               max_iter=128, max_leaf_nodes=35,
+                               min_samples_leaf=17, n_iter_no_change=1,
+                               random_state=1, validation_fraction=None,
+                               warm_start=True)"
+1489,phoneme,Auto-sklearn,-0.848,-0.797,-0.887,4.068,-0.845,600.4,0.1,"AdaBoostClassifier(algorithm='SAMME',
+                   base_estimator=DecisionTreeClassifier(max_depth=10),
+                   learning_rate=1.1377640450285444, n_estimators=352,
+                   random_state=1)"
+41027,jungle_chess_2pcs_raw_endgame_complete,Auto-sklearn,-0.975,-0.816,-0.865,0.271,-0.824,595.1,0.2,"HistGradientBoostingClassifier(early_stopping=True,
+                               l2_regularization=9.674948183980905e-09,
+                               learning_rate=0.014247987845444413, loss='auto',
+                               max_iter=512, max_leaf_nodes=55,
+                               min_samples_leaf=164, n_iter_no_change=1,
+                               random_state=1,
+                               validation_fraction=0.11770489601182355,
+                               warm_start=True)"
+41166,volkert,Auto-sklearn,-0.874,-0.586,-0.644,1.829,-0.587,595.8,0.3,"LinearDiscriminantAnalysis(shrinkage='auto', solver='lsqr',
+                           tol=0.018821286956948503)"
+54,vehicle,Auto-sklearn,-0.964,-0.86,-0.859,0.408,-0.861,595.5,0.0,"MLPClassifier(activation='tanh', alpha=0.0002060405669905105, beta_1=0.999,
+              beta_2=0.9, hidden_layer_sizes=(87, 87, 87),
+              learning_rate_init=0.00040205833939989724, max_iter=256,
+              n_iter_no_change=32, random_state=1, validation_fraction=0.0,
+              verbose=0, warm_start=True)"
+40996,fashion-mnist,Auto-sklearn,-0.968,-0.864,-0.865,1.913,-0.866,296.1,1.2,"KNeighborsClassifier(n_neighbors=4, weights='distance')"
+40996,fashion-mnist,Auto-sklearn,-0.968,-0.864,-0.865,1.913,-0.866,595.5,0.8,"KNeighborsClassifier(n_neighbors=4, weights='distance')"
+42344,sf-police-incidents,Auto-sklearn,-0.574,-0.589,-0.574,15.367,-0.569,594.8,0.5,"HistGradientBoostingClassifier(early_stopping=True,
+                               l2_regularization=3.609412172481434e-10,
+                               learning_rate=0.05972079854295879, loss='auto',
+                               max_iter=512, max_leaf_nodes=4,
+                               min_samples_leaf=2, n_iter_no_change=14,
+                               random_state=1, validation_fraction=None,
+                               warm_start=True)"
+1240,airlinescodrnaadult,Auto-sklearn,-0.62,-0.683,-0.631,13.306,-0.658,594.3,0.1,"SGDClassifier(alpha=1.6992296128865824e-07, average=True, eta0=0.01, loss='log',
+              max_iter=512, penalty='l1', random_state=1,
+              tol=1.535384699341134e-05, warm_start=True)"

baselines/automl_baseline.py

@@ -0,0 +1,11 @@
+from abc import ABC
+
+
+class AutoMLBaseline(ABC):
+    def run(self):
+        raise NotImplementedError
+
+    @staticmethod
+    def save_on_disk(data):
+        raise NotImplementedError
+

examples/0_loading_data/load_list_of_datasests.py

@@ -6,9 +6,8 @@ def get_datasets():
         'nomao', 'sylvine', 'kc1', 'jungle_chess_2pcs_raw_endgame_complete', 'credit-g', 'delta_ailerons', 'pol'
     ]
     datasets_loader = OpenMLDatasetsLoader()
-    datasets = datasets_loader.load(dataset_names)
-    print(f'Datasets "{", ".join(dataset_names)}" are available at the paths:')
-    print('\n'.join(str(d) for d in datasets))
+    datasets = datasets_loader.load(dataset_names, allow_names=True)
+    print(f'Datasets "{", ".join(dataset_names)}" are downloaded.')
     return datasets
 
 

examples/2_extracting_datasets_meta_features/extract_with_load_on_demand.py

@@ -1,3 +1,5 @@
+import openml
+
 from meta_automl.data_preparation.datasets_loaders import OpenMLDatasetsLoader
 from meta_automl.data_preparation.meta_features_extractors import PymfeExtractor
 
@@ -6,8 +8,9 @@ def main():
     dataset_names = [
         'nomao', 'sylvine'
     ]
+    dataset_ids = [openml.datasets.get_dataset(name, download_data=False, download_qualities=False).dataset_id for name in dataset_names]
     extractor = PymfeExtractor(extractor_params={'groups': 'general'}, datasets_loader=OpenMLDatasetsLoader())
-    meta_features = extractor.extract(dataset_names)
+    meta_features = extractor.extract(dataset_ids)
     return meta_features
 
 

examples/2_extracting_datasets_meta_features/load_and_extract_features_sequentially.py

@@ -9,8 +9,8 @@ def main():
     loader = OpenMLDatasetsLoader()
     extractor = PymfeExtractor(extractor_params={'groups': 'general'})
 
-    cached_datasets = loader.load(dataset_names)
-    meta_features = extractor.extract(cached_datasets)
+    datasets = loader.load(dataset_names, allow_names=True)
+    meta_features = extractor.extract(datasets)
     return meta_features
 
 

examples/3_selecting_similar_datasets/select_similar_datasets_by_knn.py

@@ -2,24 +2,25 @@
 
 from meta_automl.data_preparation.datasets_loaders import OpenMLDatasetsLoader
 from meta_automl.data_preparation.meta_features_extractors import PymfeExtractor
-from meta_automl.meta_algorithm.datasets_similarity_assessors import KNNSimilarityAssessor
+from meta_automl.meta_algorithm.datasets_similarity_assessors import KNeighborsBasedSimilarityAssessor
 
 
 def main():
     # Define datasets.
     dataset_names = ['monks-problems-1', 'apsfailure', 'australian', 'bank-marketing']
+    datasets = OpenMLDatasetsLoader().load(dataset_names, allow_names=True)
     # Extract meta-features and load on demand.
-    extractor = PymfeExtractor(extractor_params={'groups': 'general'}, datasets_loader=OpenMLDatasetsLoader())
-    meta_features = extractor.extract(dataset_names)
+    extractor = PymfeExtractor(extractor_params={'groups': 'general'})
+    meta_features = extractor.extract(datasets)
     # Preprocess meta-features, as KNN does not support NaNs.
     meta_features = meta_features.dropna(axis=1, how='any')
     # Split datasets to train (preprocessing) and test (actual meta-algorithm objects).
     x_train, x_test = train_test_split(meta_features, train_size=0.75, random_state=42)
     y_train = x_train.index
-    assessor = KNNSimilarityAssessor({'n_neighbors': 1}, n_best=2)
+    assessor = KNeighborsBasedSimilarityAssessor(n_neighbors=3)
     assessor.fit(x_train, y_train)
     # Get models for the best fitting datasets from train.
-    return x_test.index, assessor.predict(x_test)
+    return x_test.index, assessor.predict(x_test, return_distance=True)
 
 
 if __name__ == '__main__':