Training_Ensemble_TL_Regressor.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os \n",
    "import torch \n",
    "#os.environment['CUDA_VISIBLE_DEVICES'] = 5\n",
    "%env CUDA_VISIBLE_DEVICES=4\n",
    "device = torch.device('cuda:4' if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('./fastai1/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "JXpOQRLHkE6V",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "import threading\n",
    "import random\n",
    "\n",
    "from rdkit import Chem\n",
    "from rdkit.Chem import Draw\n",
    "from rdkit.Chem.Draw import IPythonConsole\n",
    "from rdkit import RDLogger\n",
    "RDLogger.DisableLog('rdApp.*') # switch off RDKit warning messages\n",
    "\n",
    "\n",
    "from fastai import *\n",
    "from fastai.text import *\n",
    "from fastai.vision import *\n",
    "from fastai.imports import *\n",
    "\n",
    "import torch\n",
    "import torchvision\n",
    "import torch.nn.functional as F\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.model_selection import train_test_split\n",
    "current_path = os.getcwd()\n",
    "print(current_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Define a custom tokenizer\n",
    "\n",
    "# Don't include the defalut specific token of fastai, only keep the padding token\n",
    "BOS,EOS,FLD,UNK,PAD = 'xxbos','xxeos','xxfld','xxunk','xxpad'\n",
    "TK_MAJ,TK_UP,TK_REP,TK_WREP = 'xxmaj','xxup','xxrep','xxwrep'\n",
    "defaults.text_spec_tok = [PAD]\n",
    "\n",
    "\n",
    "\n",
    "special_tokens = ['[BOS]', '[C@H]', '[C@@H]','[C@]', '[C@@]','[C-]','[C+]', '[c-]', '[c+]','[cH-]',\n",
    "                   '[nH]', '[N+]', '[N-]', '[n+]', '[n-]' '[NH+]', '[NH2+]', '[O-]', '[S+]', '[s+]',\n",
    "                   '[S-]', '[O+]', '[SH]', '[B-]','[BH2-]', '[BH3-]','[b-]','[PH]','[P+]', '[I+]', \n",
    "                   '[Si]','[SiH2]', '[Se]','[SeH]', '[se]', '[Se+]', '[se+]','[te]','[te+]', '[Te]',\n",
    "                   '[Pd]' , '[Ag]','[Cs]','[Li]','[K]','[Na]', '[N@]', '[N@@]', '[S@+]''[K+]', '[Ni+2]',\n",
    "                   '[Mg]','[Li+]', '[Cl-]', '[Ni]','[Cs+]', '[Cu+2]', '[Zn+2]', '[Al]', '[Cu]']\n",
    "\n",
    "\n",
    "\n",
    "class MolTokenizer(BaseTokenizer):\n",
    "    def __init__(self, lang = 'en', special_tokens = special_tokens):\n",
    "        self.lang = lang\n",
    "        self.special_tokens = special_tokens\n",
    "        \n",
    "    def tokenizer(self, smiles):\n",
    "        # add specific token '[BOS]' to represetences the start of SMILES\n",
    "        smiles = '[BOS]' + smiles\n",
    "        regex = '(\\[[^\\[\\]]{1,10}\\])'\n",
    "        char_list = re.split(regex, smiles)\n",
    "        tokens = []\n",
    "        \n",
    "        if self.special_tokens:\n",
    "            for char in char_list:\n",
    "                if char.startswith('['):\n",
    "                    if char in special_tokens:\n",
    "                        tokens.append(str(char))\n",
    "                    else:\n",
    "                        tokens.append('[UNK]')\n",
    "                else:\n",
    "                    chars = [unit for unit in char]\n",
    "                    [tokens.append(i) for i in chars]                    \n",
    "        \n",
    "        if not self.special_tokens:\n",
    "            for char in char_list:\n",
    "                if char.startswith('['):\n",
    "                    tokens.append(str(char))\n",
    "                else:\n",
    "                    chars = [unit for unit in char]\n",
    "                    [tokens.append(i) for i in chars]\n",
    "                \n",
    "        #fix the 'Br' be splited into 'B' and 'r'\n",
    "        if 'B' in tokens:\n",
    "            for index, tok in enumerate(tokens):\n",
    "                if tok == 'B':\n",
    "                    if index < len(tokens)-1: # make sure 'B' is not the last character\n",
    "                        if tokens[index+1] == 'r':\n",
    "                            tokens[index: index+2] = [reduce(lambda i, j: i + j, tokens[index : index+2])]\n",
    "        \n",
    "        #fix the 'Cl' be splited into 'C' and 'l'\n",
    "        if 'l' in tokens:\n",
    "            for index, tok in enumerate(tokens):\n",
    "                if tok == 'l':\n",
    "                    if tokens[index-1] == 'C':\n",
    "                            tokens[index-1: index+1] = [reduce(lambda i, j: i + j, tokens[index-1 : index+1])]\n",
    "        return tokens    \n",
    "    \n",
    "    def add_special_cases(self, toks):\n",
    "        pass\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def randomize_smiles(smiles):\n",
    "    m = Chem.MolFromSmiles(smiles)\n",
    "    ans = list(range(m.GetNumAtoms()))\n",
    "    np.random.shuffle(ans)\n",
    "    nm = Chem.RenumberAtoms(m,ans)\n",
    "    return Chem.MolToSmiles(nm, canonical=False, isomericSmiles=True, kekuleSmiles=False)\n",
    "\n",
    "def ee_smiles_augmentation(df, N_rounds, noise):\n",
    "    '''\n",
    "    noise: add gaussion noise to the label\n",
    "    '''\n",
    "    dist_aug = {col_name: [] for col_name in df}\n",
    "\n",
    "    for i in range(df.shape[0]):\n",
    "        for j in range(N_rounds):\n",
    "            dist_aug['smiles'].append(randomize_smiles(df.iloc[i].smiles))\n",
    "            dist_aug['ee'].append(df.iloc[i]['ee'] + np.random.normal(0,noise))\n",
    "    df_aug = pd.DataFrame.from_dict(dist_aug)\n",
    "    df_aug = df_aug.append(df, ignore_index=True)\n",
    "    return df_aug.drop_duplicates('smiles')\n",
    "\n",
    "def test_smiles_augmentation(df, N_rounds):\n",
    "    dist_aug = {col_name: [] for col_name in df}\n",
    "\n",
    "    for i in range(df.shape[0]):\n",
    "        for j in range(N_rounds):\n",
    "            dist_aug['smiles'].append(randomize_smiles(df.iloc[i].smiles))\n",
    "            dist_aug['ee'].append(df.iloc[i]['ee'])\n",
    "    df_aug = pd.DataFrame.from_dict(dist_aug)\n",
    "\n",
    "    return pd.DataFrame.from_dict(dist_aug)    \n",
    "    \n",
    "def random_seed(seed_value, use_cuda):\n",
    "    np.random.seed(seed_value) # cpu vars\n",
    "    torch.manual_seed(seed_value) # cpu  vars\n",
    "    random.seed(seed_value) # Python\n",
    "    if use_cuda: \n",
    "        torch.cuda.manual_seed(seed_value)\n",
    "        torch.cuda.manual_seed_all(seed_value) # gpu vars\n",
    "        torch.backends.cudnn.deterministic = True  #needed\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "\n",
    "\n",
    "def pred_init(seed, batch_size, filename, current_path, augm, drp_out, sigm_g):\n",
    "\n",
    "\n",
    "    # Create a path to save the results\n",
    "\n",
    "    data_path = Path(current_path)\n",
    "    name = 'regressor'\n",
    "    path = data_path/name\n",
    "    path.mkdir(exist_ok=True, parents=True)\n",
    "\n",
    "    data = filename\n",
    "    print('Dataset:', data.shape)\n",
    "\n",
    "\n",
    "    \"\"\"### Target task regressor fine-tuning on target task LM\n",
    "\n",
    "    Train-validation-test splits\n",
    "\n",
    "    - Split the data into train-validation-test sets\n",
    "    - Validation set is used for hyperparameter tuning\n",
    "    - Test set is used for the final performance evaluation\n",
    "    \"\"\"\n",
    "\n",
    "    random_seed(seed, True)\n",
    "\n",
    "    train_ , test = train_test_split(data, test_size=0.20, shuffle=False)\n",
    "    train, valid = train_test_split(train_, test_size=0.125, shuffle=False)\n",
    "\n",
    "    \n",
    "    print(train_.shape)\n",
    "    print(train.shape)\n",
    "    print(test.shape)\n",
    "    print(valid.shape)\n",
    "\n",
    "\n",
    "    \"\"\"### SMILES augmentation for regression task\n",
    "\n",
    "    - For the regression task, a gaussian noise (with mean zero and standard deviation, σg_noise) is added to the labels of the augmented SMILES during the training\n",
    "    - The number of augmented SMILES and σg_noise is tuned on the validation set\n",
    "    \"\"\"\n",
    "\n",
    "    random_seed(seed, True)\n",
    "\n",
    "    train_aug = ee_smiles_augmentation(train, augm, noise=sigm_g)\n",
    "    print(\"Train_aug: \", train_aug.shape)\n",
    "\n",
    "    train_aug\n",
    "\n",
    "    ### Data pre-processing\n",
    "\n",
    "   \n",
    "    bs = batch_size\n",
    "    tok = Tokenizer(partial(MolTokenizer, special_tokens = special_tokens), n_cpus=6, pre_rules=[], post_rules=[])\n",
    "\n",
    "    \"\"\"Adpot the encoder of the pre-trained LM according to the target dataset\n",
    "\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    np.warnings.filterwarnings('ignore', category=np.VisibleDeprecationWarning) \n",
    "    random_seed(seed, True)\n",
    "\n",
    "    lm_vocab = TextLMDataBunch.from_df(path, train_aug, valid, bs=bs, tokenizer=tok, \n",
    "                                  chunksize=50000, text_cols=0, label_cols=1, max_vocab=60000, include_bos=False, min_freq=1, num_workers=0)\n",
    "    print(f'Vocab Size: {len(lm_vocab.vocab.itos)}')\n",
    "\n",
    "\n",
    "\n",
    "    pretrained_model_path = Path('./pre_trained_model_checkpoint/')\n",
    "\n",
    "\n",
    "\n",
    "    #pretrained_model_path = Path(current_path)\n",
    "    pretrained_fnames = ['pre_trained_wt', 'pre_trained_vocab']\n",
    "    fnames = [pretrained_model_path/f'{fn}.{ext}' for fn,ext in zip(pretrained_fnames, ['pth', 'pkl'])]\n",
    "\n",
    "    random_seed(seed, True)\n",
    "\n",
    "    lm_learner = language_model_learner(lm_vocab, AWD_LSTM, config = dict(emb_sz=400, n_hid=1152, n_layers=3, pad_token=1, bidir=False, output_p=0.1,\n",
    "                              hidden_p=0.15, input_p=0.25, embed_p=0.02, weight_p=0.2, tie_weights=True, out_bias=True), drop_mult=drp_out, pretrained=False)\n",
    "    lm_learner = lm_learner.load_pretrained(*fnames)\n",
    "    lm_learner.freeze()\n",
    "    lm_learner.save_encoder(f'lm_encoder')\n",
    "\n",
    "    lm_learner.model\n",
    "\n",
    "    \"\"\"Create a text databunch for regression:\n",
    "\n",
    "    - It takes as input the train and validation data\n",
    "    - Pass the vocab of the pre-trained LM as defined in the previous step\n",
    "    - Specify the column containing text data and output\n",
    "    - Define the batch size according to the GPU memory available\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    random_seed(seed, True)\n",
    "\n",
    "    data_clas = TextClasDataBunch.from_df(path, train_aug, valid, bs=bs, tokenizer=tok, \n",
    "                                              chunksize=50000, text_cols='smiles',label_cols='ee', \n",
    "                                              vocab=lm_vocab.vocab, max_vocab=60000, include_bos=False, min_freq=1, num_workers=0)\n",
    "\n",
    "    print(f'Vocab Size: {len(data_clas.vocab.itos)}')\n",
    "\n",
    "    \"\"\"### Training the regression model\n",
    "\n",
    "    Create a learner for regression:\n",
    "\n",
    "    - Pass the databunch\n",
    "    - Load the encoder of the pre-trained LM\n",
    "    - The drop_mult hyperparameter can be tuned\n",
    "    - The model is evaluated using RMSE and R-squared value as error metric\n",
    "    \"\"\"\n",
    "\n",
    "    random_seed(seed, True)\n",
    "\n",
    "    reg_learner = text_classifier_learner(data_clas, AWD_LSTM,  config = dict(emb_sz=400, n_hid=1152, n_layers=3, pad_token=1, bidir=False, output_p=0.4,\n",
    "                                hidden_p=0.3, input_p=0.4, embed_p=0.05, weight_p=0.5), pretrained=False, drop_mult=drp_out, metrics = [r2_score, rmse])\n",
    "    reg_learner.load_encoder(f'lm_encoder')\n",
    "    reg_learner.freeze()\n",
    "    \n",
    "    return reg_learner, train_aug, valid\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Training the regressor with stepwise unfreezing \n",
    "\n",
    "def train_reg(unf1, unf2, unf3, unf4, reg_learner_tr):\n",
    "    reg_learner_tr.fit_one_cycle(unf1, 3e-2, moms=(0.8,0.7))\n",
    "\n",
    "    reg_learner_tr.freeze_to(-2)\n",
    "    reg_learner_tr.fit_one_cycle(unf2, slice(5e-3/(2.6**4),5e-3), moms=(0.8,0.7))\n",
    "\n",
    "    reg_learner_tr.freeze_to(-3)\n",
    "    reg_learner_tr.fit_one_cycle(unf3, slice(5e-4/(2.6**4),5e-4), moms=(0.8,0.7))\n",
    "\n",
    "    \"\"\"The regressor is fine-tuned all at once without any frozen weights (i.e., no gradual unfreezing)\"\"\"\n",
    "\n",
    "    reg_learner_tr.unfreeze()\n",
    "    reg_learner_tr.fit_one_cycle(unf4, slice(5e-5/(2.6**4),5e-5), moms=(0.8,0.7))\n",
    "    \n",
    "    return reg_learner_tr\n",
    "\n",
    "\n",
    "def train_reg1(unf1, unf2, unf3, unf4, lr1, lr2, lr3, lr4, reg_learner_tr):\n",
    "    reg_learner_tr.fit_one_cycle(unf1, lr1, moms=(0.8,0.7))\n",
    "\n",
    "    reg_learner_tr.freeze_to(-2)\n",
    "    reg_learner_tr.fit_one_cycle(unf2, lr2, moms=(0.8,0.7))\n",
    "\n",
    "    reg_learner_tr.freeze_to(-3)\n",
    "    reg_learner_tr.fit_one_cycle(unf3, lr3, moms=(0.8,0.7))\n",
    "\n",
    "    \"\"\"The regressor is fine-tuned all at once without any frozen weights (i.e., no gradual unfreezing)\"\"\"\n",
    "\n",
    "    reg_learner_tr.unfreeze()\n",
    "    reg_learner_tr.fit_one_cycle(unf4, lr4, moms=(0.8,0.7))\n",
    "    \n",
    "    return reg_learner_tr\n",
    "\n",
    "\n",
    "def test_performance(seed, batch_size, filename, train_aug, valid, current_path, drp_out, sigm_g,trained_model_name):\n",
    "    \n",
    "    data_path = Path(current_path)\n",
    "    name = 'regressor'\n",
    "    path = data_path/name\n",
    "    path.mkdir(exist_ok=True, parents=True)\n",
    "\n",
    "    \n",
    "    random_seed(seed, True)\n",
    "\n",
    "    bs = batch_size\n",
    "    tok = Tokenizer(partial(MolTokenizer, special_tokens = special_tokens), n_cpus=6, pre_rules=[], post_rules=[])\n",
    "    lm_vocab = TextLMDataBunch.from_df(path, train_aug, valid, bs=bs, tokenizer=tok, \n",
    "                                  chunksize=50000, text_cols=0, label_cols=1, max_vocab=60000, include_bos=False, min_freq=1, num_workers=0)\n",
    "    print(f'Vocab Size: {len(lm_vocab.vocab.itos)}')\n",
    "\n",
    "\n",
    "    tok_new = TokenizeProcessor(tokenizer=tok, chunksize=50000, include_bos=False)\n",
    "    num_new = NumericalizeProcessor(vocab=lm_vocab.vocab, max_vocab=60000, min_freq=1) \n",
    "\n",
    "\n",
    "    train_ , test_ = train_test_split(filename, test_size=0.2, shuffle=False)\n",
    "    train , test = train_test_split(filename, test_size=0.2, shuffle=False)\n",
    "\n",
    "    train_.insert(2, 'valid', False)\n",
    "    test_.insert(2, 'valid', True)\n",
    "    df = pd.concat([train_, test_])\n",
    "\n",
    "    random_seed(seed, True)\n",
    "\n",
    "    preds = []\n",
    "\n",
    "    # Randomized SMILES Predictions\n",
    "    for i in range(4):\n",
    "        np.random.seed(12*i)\n",
    "        test_aug = test_smiles_augmentation(test, 1)\n",
    "\n",
    "        #model\n",
    "\n",
    "        test_aug.insert(2, 'valid', True)\n",
    "        df_aug = test_aug\n",
    "        #train.insert(2, 'valid', False)\n",
    "        #df_aug = pd.concat([train, test_aug])\n",
    "        test_db = (TextList.from_df(df_aug, path, cols='smiles', processor=[tok_new, num_new]).split_from_df(col='valid').label_from_df(cols='ee', label_cls=FloatList).databunch(bs=bs))\n",
    "\n",
    "        learner = text_classifier_learner(test_db, AWD_LSTM, config=None, pretrained=False, drop_mult=drp_out, metrics = [r2_score, rmse])\n",
    "\n",
    "        learner.load(trained_model_name); \n",
    "\n",
    "        #get predictions\n",
    "        pred,lbl = learner.get_preds(ordered=True)\n",
    "\n",
    "        print(len(pred),len(lbl), 'augmented')\n",
    "        #print(pred,lbl)\n",
    "        preds.append(pred)\n",
    "\n",
    "    # Canonical SMILES Predictions\n",
    "\n",
    "    test_db = (TextList.from_df(df, path, cols='smiles', processor=[tok_new, num_new]).split_from_df(col='valid').label_from_df(cols='ee', label_cls=FloatList).databunch(bs=bs))\n",
    "\n",
    "    learner = text_classifier_learner(test_db, AWD_LSTM, config=None, pretrained=False, drop_mult=drp_out, metrics = [r2_score, rmse])\n",
    "\n",
    "    learner.load(trained_model_name);\n",
    "\n",
    "\n",
    "    #get predictions\n",
    "    pred_canonical,lbl = learner.get_preds(ordered=True)\n",
    "    print(len(pred_canonical),len(lbl), 'canonical')    \n",
    "    preds.append(pred_canonical)\n",
    "    list_test = list(test['smiles'])\n",
    "\n",
    "\n",
    "    \"\"\"The test set performance is evaluated using the predictions based on the canonical SMILES as well as that employing test-time augmentation\"\"\"\n",
    "\n",
    "    print('Test Set (Canonical)')\n",
    "    print('RMSE:', root_mean_squared_error(pred_canonical,lbl))\n",
    "    print('MAE:', mean_absolute_error(pred_canonical,lbl))\n",
    "    print('R2:', r2_score(pred_canonical,lbl))\n",
    "\n",
    "    avg_preds = sum(preds)/len(preds)\n",
    "    #print('\\n')\n",
    "    print('Test Set (Average)')\n",
    "    print('RMSE:', root_mean_squared_error(avg_preds,lbl))\n",
    "    print('MAE:', mean_absolute_error(avg_preds,lbl))\n",
    "    print('R2:', r2_score(avg_preds,lbl))\n",
    "\n",
    "    #return root_mean_squared_error(pred_canonical,lbl),root_mean_squared_error(avg_preds,lbl),list_test,lbl,pred_canonical\n",
    "    return root_mean_squared_error(pred_canonical,lbl),root_mean_squared_error(avg_preds,lbl)\n",
    "\n",
    "def predictor(smiles, seed, batch_size, filename, train_aug, valid, current_path, drp_out, sigm_g, trained_model_name):\n",
    "\n",
    "    data_path = Path(current_path)\n",
    "    name = 'regressor'\n",
    "    path = data_path/name\n",
    "    path.mkdir(exist_ok=True, parents=True)\n",
    "\n",
    "    \n",
    "    random_seed(seed, True)\n",
    "\n",
    "    bs = batch_size\n",
    "    tok = Tokenizer(partial(MolTokenizer, special_tokens = special_tokens), n_cpus=6, pre_rules=[], post_rules=[])\n",
    "    lm_vocab = TextLMDataBunch.from_df(path, train_aug, valid, bs=bs, tokenizer=tok, \n",
    "                                  chunksize=50000, text_cols=0, label_cols=1, max_vocab=60000, include_bos=False, min_freq=1, num_workers=0)\n",
    "    #print(f'Vocab Size: {len(lm_vocab.vocab.itos)}')\n",
    "\n",
    "\n",
    "    tok_new = TokenizeProcessor(tokenizer=tok, chunksize=50000, include_bos=False)\n",
    "    num_new = NumericalizeProcessor(vocab=lm_vocab.vocab, max_vocab=60000, min_freq=1) \n",
    "\n",
    "\n",
    "    \n",
    "    train_ , test_ = train_test_split(filename, test_size=0.2, shuffle=False)\n",
    "    \n",
    "    test_ = test_[:1]\n",
    "    train , test = train_test_split(filename, test_size=0.2, shuffle=False)\n",
    "\n",
    "    train_.insert(2, 'valid', False)\n",
    "    test_.insert(2, 'valid', True)\n",
    "    df = pd.concat([train_, test_])\n",
    "    \n",
    "    gen_smile = smiles\n",
    "    df_gen_smile = pd.DataFrame(gen_smile, columns = ['smiles'])\n",
    "   \n",
    "    df_gen_smile.insert(1, 'ee', '0')\n",
    "    df_gen_smile.insert(2, 'valid', True)\n",
    "   \n",
    "    df_smile = pd.concat([df,df_gen_smile])\n",
    "    \n",
    "    test_db = (TextList.from_df(df_smile, path, cols='smiles', processor=[tok_new, num_new]).split_from_df(col='valid').label_from_df(cols='ee', label_cls=FloatList).databunch(bs=bs))\n",
    "\n",
    "    learner = text_classifier_learner(test_db, AWD_LSTM, config=None, pretrained=False, drop_mult=drp_out, metrics = [r2_score, rmse])\n",
    "\n",
    "    learner.load(trained_model_name);\n",
    "\n",
    "\n",
    "    #get predictions\n",
    "    pred_canonical,lbl = learner.get_preds(ordered=True)\n",
    "    #print(len(pred_canonical),len(lbl), 'canonical')    \n",
    "    \n",
    "\n",
    "    return gen_smile, pred_canonical[1:]\n",
    "\n",
    "def tensor_to_array(prediction):\n",
    "    a = np.zeros([len(prediction)])\n",
    "    for k in range(len(prediction)):\n",
    "        a[k] = float(prediction[k][0])\n",
    "    return a\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "NYpTQSh9YgVY"
   },
   "source": [
    "**Test of random state**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Best parameter\n",
    "seed = 1234\n",
    "batch_size = 128\n",
    "data_file_path = './data/Betach_ee_220.xlsx'\n",
    "augm = 100\n",
    "drp_out =  0.0\n",
    "sigm_g = 0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "CV_list_name = ['FullCV_1','FullCV_2','FullCV_3','FullCV_4','FullCV_5','FullCV_6','FullCV_7','FullCV_8','FullCV_9',\n",
    "'FullCV_10','FullCV_11','FullCV_12','FullCV_13','FullCV_14','FullCV_15','FullCV_16','FullCV_17',\n",
    "'FullCV_18','FullCV_19','FullCV_20','FullCV_21','FullCV_22','FullCV_23','FullCV_24','FullCV_25',\n",
    "'FullCV_26','FullCV_27','FullCV_28','FullCV_29','FullCV_30']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Prediction with the trained model Just original test set performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def CV_state_rep(data_file_name,CV_list):\n",
    "    val_rmse_app = []\n",
    "    canon_test_rmse_app = []\n",
    "    tta_test_rmse_app = []\n",
    "    for CV_state in CV_list:\n",
    "        data_file = pd.read_excel(data_file_name, sheet_name=CV_state)\n",
    "        reg_learner_pre, train_aug , valid = pred_init(seed, batch_size, data_file, current_path, augm, drp_out, sigm_g)\n",
    "\n",
    "        lr1 = 3e-2\n",
    "        lr2 = slice(5e-4/(2.6**4),5e-3)\n",
    "        lr3 = slice(5e-4/(2.6**4),5e-3)\n",
    "        lr4 = slice(5e-6/(2.6**4),5e-5)\n",
    "\n",
    "        unf1 = 6\n",
    "        unf2 = 7\n",
    "        unf3 = 7\n",
    "        unf4 = 7\n",
    "\n",
    "        reg_learner_trained = train_reg1(unf1, unf2, unf3, unf4, lr1, lr2, lr3, lr4, reg_learner_pre)\n",
    "        trained_model_name = 'reg_learner_betach_fl_' + CV_state\n",
    "        reg_learner_trained.save(trained_model_name)\n",
    "        val_result = reg_learner_trained.validate(reg_learner_trained.data.valid_dl)\n",
    "        val_rmse = float(val_result[-1])\n",
    "        val_rmse_app.append(val_rmse)\n",
    "        canon_test_rmse,tta_test_rmse,list_test,real,pred_canon = test_performance(seed, batch_size, data_file, train_aug, valid, current_path,drp_out, sigm_g, trained_model_name)\n",
    "        canon_test_rmse_app.append(canon_test_rmse)\n",
    "        tta_test_rmse_app.append(tta_test_rmse)\n",
    "\n",
    "    return val_rmse_app, canon_test_rmse_app, tta_test_rmse_app"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "val_rmse_app, canon_test_rmse_app, tta_test_rmse_app = CV_state_rep(data_file_path, CV_list_name)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "re-explore",
   "language": "python",
   "name": "re-explore"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}