Molecule_Design.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Read data \n",
    "\n",
    "2. Prefix instructs\n",
    "\n",
    "3. ICL Similar examples\n",
    "\n",
    "4. Suffix Descriptions\n",
    "\n",
    "Evaluation Method:\n",
    "\n",
    "Validation: Sample 50\n",
    "\n",
    "Test: Sample 500 repeat 5 times : mean std\n",
    "\n",
    "\n",
    "1. Format: IUPAC name \n",
    "\n",
    "2. Similarity Methods\n",
    "\n",
    "3. Temperature / Sample_N\n",
    "\n",
    "Utils: Save during training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-06T19:44:25.929956Z",
     "start_time": "2023-05-06T19:44:24.707596Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from rdkit import Chem\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "import openai\n",
    "import datetime\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "def molToCanonical(smiles):\n",
    "    mol = Chem.MolFromSmiles(smiles)\n",
    "    canonical_smiles = Chem.MolToSmiles(mol)\n",
    "    return canonical_smiles\n",
    "\n",
    "from steamship import Steamship"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-06T19:44:25.935386Z",
     "start_time": "2023-05-06T19:44:25.933110Z"
    }
   },
   "outputs": [],
   "source": [
    "# Set up the OpenAI API client\n",
    "\n",
    "openai.api_key = 'Your API Key'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-06T19:44:26.083528Z",
     "start_time": "2023-05-06T19:44:25.938220Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[36mresults\u001b[m\u001b[m/        \u001b[1m\u001b[36mresults_test\u001b[m\u001b[m/   test.txt        train.txt       validation.txt\r\n"
     ]
    }
   ],
   "source": [
    "ls Molecule_Design/ChEBI-20_data/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T22:52:36.801966Z",
     "start_time": "2023-05-14T22:52:36.797296Z"
    }
   },
   "outputs": [],
   "source": [
    "folder = 'Molecule_Design/ChEBI-20_data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-06T19:44:26.254765Z",
     "start_time": "2023-05-06T19:44:26.095443Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[36mresults\u001b[m\u001b[m/        \u001b[1m\u001b[36mresults_test\u001b[m\u001b[m/   test.txt        train.txt       validation.txt\r\n"
     ]
    }
   ],
   "source": [
    "ls Molecule_Design/ChEBI-20_data/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-06T19:44:28.612388Z",
     "start_time": "2023-05-06T19:44:28.609296Z"
    }
   },
   "outputs": [],
   "source": [
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-06T19:44:29.058981Z",
     "start_time": "2023-05-06T19:44:29.054884Z"
    }
   },
   "outputs": [],
   "source": [
    "def read_data(filename):\n",
    "    # Open the file for reading\n",
    "    with open(filename, 'r') as f:\n",
    "        # Create a CSV reader with tab delimiter\n",
    "        reader = csv.reader(f, delimiter='\\t')\n",
    "        # Read the data into a list of tuples\n",
    "        data = [tuple(row) for row in reader]\n",
    "    df = pd.DataFrame(data[1:], columns=data[0])\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-06T19:44:49.477060Z",
     "start_time": "2023-05-06T19:44:49.241747Z"
    }
   },
   "outputs": [],
   "source": [
    "train = read_data(folder + \"train.txt\")\n",
    "valid = read_data(folder + \"validation.txt\")\n",
    "test = read_data(folder + \"test.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-06T19:44:49.483765Z",
     "start_time": "2023-05-06T19:44:49.479918Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26407"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T17:03:18.802154Z",
     "start_time": "2023-04-25T17:03:18.785323Z"
    }
   },
   "outputs": [],
   "source": [
    "valid = valid.sample(30, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T17:03:18.811249Z",
     "start_time": "2023-04-25T17:03:18.804506Z"
    }
   },
   "outputs": [],
   "source": [
    "test = test.sample(500, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T17:03:18.825395Z",
     "start_time": "2023-04-25T17:03:18.815401Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['CID', 'SMILES', 'description'], dtype='object')"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T17:03:19.940077Z",
     "start_time": "2023-04-25T17:03:19.933713Z"
    }
   },
   "outputs": [],
   "source": [
    "from rdkit import Chem\n",
    "from rdkit.Chem.Scaffolds import MurckoScaffold\n",
    "from rdkit.Chem import DataStructs\n",
    "from rdkit.Chem import rdMolDescriptors\n",
    "from tqdm import tqdm\n",
    "\n",
    "def get_scaffold_fp(x):\n",
    "    mol = Chem.MolFromSmiles(x)\n",
    "    scaffold = MurckoScaffold.GetScaffoldForMol(mol)\n",
    "    scaffold_fp = rdMolDescriptors.GetMorganFingerprint(scaffold, 2)\n",
    "    return scaffold_fp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T18:13:03.793582Z",
     "start_time": "2023-04-25T18:13:03.789304Z"
    }
   },
   "outputs": [],
   "source": [
    "def molToCanonical(smiles):\n",
    "    mol = Chem.MolFromSmiles(smiles)\n",
    "    canonical_smiles = Chem.MolToSmiles(mol)\n",
    "    return canonical_smiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T18:15:09.094342Z",
     "start_time": "2023-04-25T18:14:55.873376Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[14:14:56] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:14:56] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:14:56] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:14:56] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:14:57] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:14:57] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:14:57] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:14:57] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:14:57] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:14:58] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:14:58] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:14:59] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:14:59] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:14:59] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:14:59] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:00] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:00] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:00] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:00] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:00] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:00] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:00] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:00] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:00] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:00] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:01] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:01] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:01] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:01] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:02] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:02] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:02] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:02] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:02] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:02] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:03] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:03] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:03] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:03] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:03] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:03] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:03] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:03] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:03] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:03] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:03] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:03] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:03] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:04] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:04] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:04] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:04] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:04] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:04] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:04] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:04] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:04] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:04] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:04] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:04] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:04] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:04] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:05] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:05] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:05] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:05] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:05] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:06] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:06] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:06] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:06] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:07] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:07] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:07] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:08] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:08] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:08] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:08] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:08] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:08] WARNING: not removing hydrogen atom without neighbors\n"
     ]
    }
   ],
   "source": [
    "train['SMILES'] = train['SMILES'].apply(molToCanonical)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T18:15:42.971909Z",
     "start_time": "2023-04-25T18:15:42.953268Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "valid['SMILES'] = valid['SMILES'].apply(molToCanonical)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T18:15:46.816786Z",
     "start_time": "2023-04-25T18:15:46.592328Z"
    }
   },
   "outputs": [],
   "source": [
    "test['SMILES'] = test['SMILES'].apply(molToCanonical)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T18:16:15.562022Z",
     "start_time": "2023-04-25T18:15:50.177818Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[14:15:50] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:50] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:50] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:51] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:52] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:52] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:52] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:53] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:53] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:54] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:54] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:56] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:56] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:57] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:57] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:58] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:58] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:58] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:59] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:59] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:59] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:59] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:59] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:59] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:15:59] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:00] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:00] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:01] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:01] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:02] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:02] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:02] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:02] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:02] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:02] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:03] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:03] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:04] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:04] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:04] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:04] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:05] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:05] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:05] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:05] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:05] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:05] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:05] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:05] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:05] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:05] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:06] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:06] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:06] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:06] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:06] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:06] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:06] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:07] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:07] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:07] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:07] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:08] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:08] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:08] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:09] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:09] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:10] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:10] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:11] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:11] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:11] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:12] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:13] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:13] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:13] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:14] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:14] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:14] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:16:15] WARNING: not removing hydrogen atom without neighbors\n"
     ]
    }
   ],
   "source": [
    "train['scaffold_fp'] = train['SMILES'].apply(lambda x: get_scaffold_fp(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T17:03:48.771475Z",
     "start_time": "2023-04-25T17:03:48.763715Z"
    }
   },
   "outputs": [],
   "source": [
    "def top_n_scaffold_similar_molecules(target_smiles, molecule_scaffold_list, molecule_smiles_list, n=5):\n",
    "    target_mol = Chem.MolFromSmiles(target_smiles)\n",
    "    target_scaffold = MurckoScaffold.GetScaffoldForMol(target_mol)\n",
    "    target_fp = rdMolDescriptors.GetMorganFingerprint(target_scaffold, 2)\n",
    "\n",
    "    similarities = []\n",
    "\n",
    "    for idx, scaffold_fp in enumerate(molecule_scaffold_list):\n",
    "        try:\n",
    "            tanimoto_similarity = DataStructs.TanimotoSimilarity(target_fp, scaffold_fp)\n",
    "            similarities.append((idx, tanimoto_similarity))\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            continue\n",
    "\n",
    "    similarities.sort(key=lambda x: x[1], reverse=True)\n",
    "    top_5_similar_molecules = similarities[:n]\n",
    "\n",
    "    return [molecule_smiles_list[i[0]] for i in top_5_similar_molecules]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T17:03:48.782417Z",
     "start_time": "2023-04-25T17:03:48.777403Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# top_n_scaffold_similar_molecules(train.iloc[0]['reactants_smiles'], list(train['scaffold_fp']), list(train['reactants_smiles']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T17:52:43.593554Z",
     "start_time": "2023-04-25T17:52:43.591154Z"
    }
   },
   "outputs": [],
   "source": [
    "# 1. prompt\n",
    "# 2. performance csv\n",
    "# 3. sleep "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T17:55:35.357842Z",
     "start_time": "2023-04-25T17:55:35.353251Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_prompt_molecule_design(input_text, examples):\n",
    "    prompt = \"You are an expert chemist. Given the molecular requirements description, your task is to design a new molecule using your experienced chemical Molecular Design knowledge. \\n\\\n",
    "Please strictly follow the format, no other information can be provided. You should only reply with SMILES \\\n",
    "string notations to represent the designed molecule. The SMILES must be valid and chemically reasonable. \\n\"\n",
    "    \n",
    "    for example in examples:\n",
    "        prompt += f\"Molecular requirements description: {example[0]}\\nMolecular SMILES: {example[1]}\\n\"\n",
    "    prompt += f\"Molecular requirements description: {input_text}\\nMolecular SMILES:\"\n",
    "    return prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T17:55:35.606385Z",
     "start_time": "2023-04-25T17:55:35.603081Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_prompt_molecule_captioning(input_text, examples):\n",
    "    prompt = \"You are an expert chemist. Given the molecular SMILES, your task is to provide the detailed description of the molecule using your experienced chemical Molecular knowledge. \\n\\\n",
    "Please strictly follow the format, no other information can be provided.\\n\"\n",
    "    \n",
    "    for example in examples:\n",
    "        prompt += f\"Molecule SMILES: {example[0]}\\nMolecular Description: {example[1]}\\n\"\n",
    "    prompt += f\"Molecule SMILES: {input_text}\\nMolecular Description:\"\n",
    "    return prompt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T17:52:45.743540Z",
     "start_time": "2023-04-25T17:52:45.740801Z"
    }
   },
   "outputs": [],
   "source": [
    "from prompt_utils.icl_examples import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T17:56:35.300956Z",
     "start_time": "2023-04-25T17:56:35.297962Z"
    }
   },
   "outputs": [],
   "source": [
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T17:56:36.634278Z",
     "start_time": "2023-04-25T17:56:36.626199Z"
    }
   },
   "outputs": [],
   "source": [
    "def generate_response_by_davinci(prompt, model_engine = 'text-davinci-003'):\n",
    "    time.sleep(2)\n",
    "    completion = openai.Completion.create(\n",
    "      engine=model_engine,\n",
    "      prompt=prompt,\n",
    "      temperature=0.5,\n",
    "      max_tokens=256,\n",
    "#       top_p=1.0,\n",
    "      frequency_penalty=0.0,\n",
    "      presence_penalty=0.0, \n",
    "      n=5,\n",
    "    )\n",
    "#     message = completion.choices[0]['text'].strip()\n",
    "    \n",
    "    message = completion.choices\n",
    "    message = [i['text'].strip() for i in message]\n",
    "    return message\n",
    "\n",
    "def generate_response_by_gpt35(prompt, model_engine = \"gpt-3.5-turbo\"):\n",
    "    time.sleep(2)\n",
    "    completion = openai.ChatCompletion.create(\n",
    "        model=model_engine, temperature=0.5, n=5, \n",
    "        messages=[{\"role\": \"user\", \"content\": prompt}],\n",
    "    )\n",
    "#     message = completion.choices[0].message.content.strip()\n",
    "\n",
    "    message = completion.choices\n",
    "    message = [i.message.content.strip() for i in message]\n",
    "    return message\n",
    "\n",
    "def generate_response_by_gpt4(prompt):\n",
    "    time.sleep(2)\n",
    "    # Create a Steamship client\n",
    "    # NOTE: When developing a package, just use `self.client`\n",
    "    client = Steamship(workspace=\"gpt-4-92g\")\n",
    "\n",
    "    # Create an instance of this generator\n",
    "    generator = client.use_plugin('gpt-4', config={\"temperature\":0.5, \"n\": 5})\n",
    "\n",
    "    # Generate text\n",
    "    task = generator.generate(text=prompt)\n",
    "    # Wait for completion of the task.\n",
    "    task.wait()\n",
    "    # Print the output\n",
    "#     message = task.output.blocks[0].text.strip()\n",
    "    message = task.output.blocks\n",
    "    message = [i.text.strip() for i in message]\n",
    "    return message"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T17:52:48.131589Z",
     "start_time": "2023-04-25T17:52:48.127834Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_input_output_columns_by_task(task):\n",
    "    if task == 'smiles2iupac':\n",
    "        return \"smiles\", \"iupac\"\n",
    "    elif task == 'smiles2formula':\n",
    "        return \"smiles\", \"formula\"\n",
    "    elif task == 'iupac2smiles':\n",
    "        return \"iupac\", \"smiles\"\n",
    "    elif task == 'formula2smiles':\n",
    "        return \"formula\", 'smiles'\n",
    "    elif task == 'molecule_design':\n",
    "        return \"description\", \"SMILES\"\n",
    "    elif task == 'molecule_captioning':\n",
    "        return \"SMILES\", \"description\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T17:52:48.314614Z",
     "start_time": "2023-04-25T17:52:48.310518Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_prompt(reactant, examples, task):\n",
    "    if task == 'smiles2iupac':\n",
    "        return create_prompt_smiles2iupac(reactant, examples)\n",
    "    elif task == 'smiles2formula':\n",
    "        return create_prompt_smiles2formula(reactant, examples)\n",
    "    elif task == 'iupac2smiles':\n",
    "        return create_prompt_iupac2smiles(reactant, examples)\n",
    "    elif task == 'formula2smiles':\n",
    "        return create_prompt_formula2smiles(reactant, examples)\n",
    "    elif task == 'molecule_design':\n",
    "        return create_prompt_molecule_design(reactant, examples)\n",
    "    elif task == 'molecule_captioning':\n",
    "        return create_prompt_molecule_captioning(reactant, examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-25T17:51:30.320683Z",
     "start_time": "2023-04-25T17:51:30.293110Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8333333333333334\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['python', 'java']"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import difflib\n",
    "\n",
    "def similarity_ratio(s1, s2):\n",
    "    # Calculate the similarity ratio between the two strings\n",
    "    ratio = difflib.SequenceMatcher(None, s1, s2).ratio()\n",
    "    \n",
    "    # Return the similarity ratio\n",
    "    return ratio\n",
    "\n",
    "# Example usage:\n",
    "s1 = \"python\"\n",
    "s2 = \"pathon\"\n",
    "ratio = similarity_ratio(s1, s2)\n",
    "print(ratio)  # Output: 0.0\n",
    "\n",
    "\n",
    "def top_n_similar_strings(query, candidates, n=5):\n",
    "    # Calculate the Levenshtein distance between the query and each candidate\n",
    "    distances = [(c, similarity_ratio(query, c)) for c in candidates]\n",
    "    \n",
    "    # Sort the candidates by their Levenshtein distance to the query\n",
    "    sorted_distances = sorted(distances, key=lambda x: x[1], reverse=True)\n",
    "    \n",
    "    # Get the top n candidates with the smallest Levenshtein distance\n",
    "    top_candidates = [d[0] for d in sorted_distances[:n]]\n",
    "    \n",
    "    # Return the top n candidates\n",
    "    return top_candidates\n",
    "top_n_similar_strings(\"pathon\", ['python', 'java'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-26T02:39:05.974768Z",
     "start_time": "2023-04-26T01:27:01.410497Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Molecule_Design/ChEBI-20_data//results/valid_molecule_captioning_gpt35_SMILES_5_Fixed_ICL.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30it [06:54, 13.80s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15\n",
      "Molecule_Design/ChEBI-20_data//results/valid_molecule_design_davinci003_SMILES_5_Fixed_ICL.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30it [05:45, 11.51s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16\n",
      "Molecule_Design/ChEBI-20_data//results/valid_molecule_design_gpt4_SMILES_5_Fixed_ICL.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30it [06:55, 13.85s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17\n",
      "Molecule_Design/ChEBI-20_data//results/valid_molecule_design_gpt35_SMILES_5_Fixed_ICL.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30it [04:31,  9.05s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18\n",
      "Molecule_Design/ChEBI-20_data//results/valid_molecule_captioning_davinci003_SMILES_10_Fixed_ICL.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30it [07:41, 15.40s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19\n",
      "Molecule_Design/ChEBI-20_data//results/valid_molecule_captioning_gpt4_SMILES_10_Fixed_ICL.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30it [11:05, 22.19s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\n",
      "Molecule_Design/ChEBI-20_data//results/valid_molecule_captioning_gpt35_SMILES_10_Fixed_ICL.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30it [07:17, 14.59s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21\n",
      "Molecule_Design/ChEBI-20_data//results/valid_molecule_design_davinci003_SMILES_10_Fixed_ICL.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30it [06:21, 12.72s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22\n",
      "Molecule_Design/ChEBI-20_data//results/valid_molecule_design_gpt4_SMILES_10_Fixed_ICL.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30it [07:02, 14.08s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23\n",
      "Molecule_Design/ChEBI-20_data//results/valid_molecule_design_gpt35_SMILES_10_Fixed_ICL.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30it [08:28, 16.96s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# format the running grid search \n",
    "# evaluate and save\n",
    "\n",
    "Models = ['davinci003', 'gpt4', 'gpt35']\n",
    "ICL_Sample_methods = ['Scaffold_SIM', 'Fixed_ICL']\n",
    "ICL_Samples = [5, 10]\n",
    "NAME_Format = ['SMILES']\n",
    "TASK_Types = ['molecule_captioning', 'molecule_design']\n",
    "detail_save_folder = folder + '/results/'\n",
    "\n",
    "params_idx = 0\n",
    "\n",
    "for sample_method in ICL_Sample_methods:\n",
    "    for sample_num in ICL_Samples:\n",
    "        for name in NAME_Format:\n",
    "            for task in TASK_Types:\n",
    "                for model in Models:\n",
    "                    \n",
    "                    details_results = []\n",
    "                    performance_results = []\n",
    "                                      \n",
    "                    if params_idx < 14:\n",
    "                        params_idx +=1\n",
    "                        continue\n",
    "                    \n",
    "                    detail_predict_file = detail_save_folder + 'valid_{}_{}_{}_{}_{}.csv'.format(task, model, name, sample_num, sample_method)\n",
    "                    log_file = detail_save_folder + 'valid_{}_{}_{}_{}_{}.log'.format(task, model, name, sample_num, sample_method)\n",
    "                    print(detail_predict_file)\n",
    "\n",
    "                    # append new date\n",
    "                    # Get the current date and time\n",
    "                    now = datetime.datetime.now()\n",
    "                    # Convert the date and time to a string\n",
    "                    date_time_str = now.strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "                    with open(log_file, \"w+\") as file:\n",
    "                        file.write(\"=\" * 30 + date_time_str + \"=\" * 30 + \"\\n\")\n",
    "\n",
    "                    # init var\n",
    "                    predicted_products = []\n",
    "                    input_col, output_col = get_input_output_columns_by_task(task)\n",
    "\n",
    "                    # query\n",
    "                    for idx, row in tqdm(valid.iterrows()):\n",
    "                        reactant = row[input_col]\n",
    "                        product = row[output_col]\n",
    "\n",
    "                        # ICL examples\n",
    "                        if sample_method == 'Fixed_ICL':\n",
    "                            chunk = train.sample(sample_num, random_state=42)\n",
    "                        elif sample_method == 'Scaffold_SIM':\n",
    "                            if input_col == 'SMILES':\n",
    "                                sim = top_n_scaffold_similar_molecules(reactant, list(train['scaffold_fp']), list(train['SMILES']), n=sample_num)\n",
    "                            else:\n",
    "                                # similarity by leven\n",
    "                                sim = top_n_similar_strings(reactant, list(train[input_col]), n=sample_num)\n",
    "                            chunk = train[train[input_col].isin(sim)]\n",
    "                            \n",
    "                        examples = list(zip(chunk[input_col].values, chunk[output_col].values))\n",
    "\n",
    "                        # build prompt and save\n",
    "                        prompt = create_prompt(reactant, examples, task)\n",
    "                        \n",
    "                        with open(log_file, \"a\") as file:\n",
    "                            file.write(prompt + \"\\n\")\n",
    "                            file.write(\"=\" * 50 + \"\\n\")\n",
    "\n",
    "                        # different model \n",
    "                        if model == 'davinci003':\n",
    "                            predicted_product = generate_response_by_davinci(prompt)\n",
    "                        elif model == 'gpt35':\n",
    "                            predicted_product = generate_response_by_gpt35(prompt)\n",
    "                        elif model == 'gpt4':\n",
    "                            predicted_product = generate_response_by_gpt4(prompt)\n",
    "\n",
    "                        predicted_products.append(predicted_product)\n",
    "                        details_results.append([reactant] + [product] + predicted_product)\n",
    "\n",
    "#                     # evaluate\n",
    "#                     acc_list = []\n",
    "#                     for repeat in range(5):\n",
    "#                         tpredicted_products = [i[repeat] for i in predicted_products]\n",
    "#                         correct = 0\n",
    "#                         all_sample_num = len(valid)\n",
    "#                         for idx, gt in enumerate(list(valid['products_smiles'])):\n",
    "#                             pred = tpredicted_products[idx]\n",
    "#                             try:\n",
    "#                                 mol = Chem.MolFromSmiles(pred)\n",
    "#                                 pred = Chem.MolToSmiles(mol)\n",
    "#                             except Exception as e:\n",
    "#                                 continue\n",
    "\n",
    "#                             pred_list = pred.split(\".\")\n",
    "#                             if gt in pred_list:\n",
    "#                                 correct += 1\n",
    "#                         acc = correct / all_sample_num\n",
    "#                         acc_list.append(acc)\n",
    "\n",
    "                    # save to file\n",
    "#                     performance_results.append([task, model, name, sample_num, sample_method, np.mean(acc_list)] + acc_list)\n",
    "\n",
    "                    details_df = pd.DataFrame(details_results, columns=[input_col, output_col, 'pred_1', 'pred_2', 'pred_3', 'pred_4', 'pred_5'])\n",
    "                    details_df.to_csv(detail_predict_file, index=False)\n",
    "           \n",
    "                    params_idx += 1\n",
    "                    print(params_idx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-26T16:02:42.951078Z",
     "start_time": "2023-04-26T16:02:41.837234Z"
    },
    "code_folding": [
     2,
     97,
     141
    ]
   },
   "outputs": [],
   "source": [
    "from nltk.translate.bleu_score import corpus_bleu\n",
    "from Levenshtein import distance as lev\n",
    "def evaluate1(outputs, verbose=False):\n",
    "#     outputs = []\n",
    "\n",
    "#     with open(osp.join(input_fp)) as f:\n",
    "#         reader = csv.DictReader(f, delimiter=\"\\t\", quoting=csv.QUOTE_NONE)\n",
    "#         for n, line in enumerate(reader):\n",
    "#             gt_smi = line['ground truth']\n",
    "#             ot_smi = line['output']\n",
    "#             outputs.append((line['description'], gt_smi, ot_smi))\n",
    "\n",
    "\n",
    "    bleu_scores = []\n",
    "    #meteor_scores = []\n",
    "\n",
    "    references = []\n",
    "    hypotheses = []\n",
    "\n",
    "    for i, (smi, gt, out) in enumerate(outputs):\n",
    "\n",
    "        if i % 100 == 0:\n",
    "            if verbose:\n",
    "                print(i, 'processed.')\n",
    "\n",
    "\n",
    "        gt_tokens = [c for c in gt]\n",
    "\n",
    "        out_tokens = [c for c in out]\n",
    "\n",
    "        references.append([gt_tokens])\n",
    "        hypotheses.append(out_tokens)\n",
    "\n",
    "        # mscore = meteor_score([gt], out)\n",
    "        # meteor_scores.append(mscore)\n",
    "\n",
    "    # BLEU score\n",
    "    bleu_score = corpus_bleu(references, hypotheses)\n",
    "    if verbose: print('BLEU score:', bleu_score)\n",
    "\n",
    "    # Meteor score\n",
    "    # _meteor_score = np.mean(meteor_scores)\n",
    "    # print('Average Meteor score:', _meteor_score)\n",
    "\n",
    "    rouge_scores = []\n",
    "\n",
    "    references = []\n",
    "    hypotheses = []\n",
    "\n",
    "    levs = []\n",
    "\n",
    "    num_exact = 0\n",
    "\n",
    "    bad_mols = 0\n",
    "\n",
    "    for i, (smi, gt, out) in enumerate(outputs):\n",
    "\n",
    "        hypotheses.append(out)\n",
    "        references.append(gt)\n",
    "\n",
    "        try:\n",
    "            m_out = Chem.MolFromSmiles(out)\n",
    "            m_gt = Chem.MolFromSmiles(gt)\n",
    "\n",
    "            if Chem.MolToInchi(m_out) == Chem.MolToInchi(m_gt): num_exact += 1\n",
    "            #if gt == out: num_exact += 1 #old version that didn't standardize strings\n",
    "        except:\n",
    "            bad_mols += 1\n",
    "\n",
    "        \n",
    "\n",
    "        levs.append(lev(out, gt))\n",
    "\n",
    "\n",
    "    # Exact matching score\n",
    "    exact_match_score = num_exact/(i+1)\n",
    "    if verbose:\n",
    "        print('Exact Match:')\n",
    "        print(exact_match_score)\n",
    "\n",
    "    # Levenshtein score\n",
    "    levenshtein_score = np.mean(levs)\n",
    "    if verbose:\n",
    "        print('Levenshtein:')\n",
    "        print(levenshtein_score)\n",
    "        \n",
    "    validity_score = 1 - bad_mols/len(outputs)\n",
    "    if verbose:\n",
    "        print('validity:', validity_score)\n",
    "\n",
    "    return bleu_score, exact_match_score, levenshtein_score, validity_score\n",
    "\n",
    "from rdkit import Chem\n",
    "from rdkit.Chem import MACCSkeys\n",
    "from rdkit import DataStructs\n",
    "from rdkit.Chem import AllChem\n",
    "\n",
    "def evaluate2(raw_outputs, morgan_r=2, verbose=False):\n",
    "    bad_mols = 0\n",
    "    outputs = []\n",
    "\n",
    "    for desc, gt_smi, ot_smi in raw_outputs:\n",
    "        try:\n",
    "            gt_m = Chem.MolFromSmiles(gt_smi)\n",
    "            ot_m = Chem.MolFromSmiles(ot_smi)\n",
    "\n",
    "            if ot_m == None: raise ValueError('Bad SMILES')\n",
    "            outputs.append((desc, gt_m, ot_m))\n",
    "        except:\n",
    "            bad_mols += 1\n",
    "    validity_score = len(outputs)/(len(outputs)+bad_mols)\n",
    "    if verbose:\n",
    "        print('validity:', validity_score)\n",
    "\n",
    "\n",
    "    MACCS_sims = []\n",
    "    morgan_sims = []\n",
    "    RDK_sims = []\n",
    "\n",
    "    enum_list = outputs\n",
    "\n",
    "    for i, (desc, gt_m, ot_m) in enumerate(enum_list):\n",
    "\n",
    "        if i % 100 == 0:\n",
    "            if verbose: print(i, 'processed.')\n",
    "\n",
    "        MACCS_sims.append(DataStructs.FingerprintSimilarity(MACCSkeys.GenMACCSKeys(gt_m), MACCSkeys.GenMACCSKeys(ot_m), metric=DataStructs.TanimotoSimilarity))\n",
    "        RDK_sims.append(DataStructs.FingerprintSimilarity(Chem.RDKFingerprint(gt_m), Chem.RDKFingerprint(ot_m), metric=DataStructs.TanimotoSimilarity))\n",
    "        morgan_sims.append(DataStructs.TanimotoSimilarity(AllChem.GetMorganFingerprint(gt_m,morgan_r), AllChem.GetMorganFingerprint(ot_m, morgan_r)))\n",
    "\n",
    "    maccs_sims_score = np.mean(MACCS_sims)\n",
    "    rdk_sims_score = np.mean(RDK_sims)\n",
    "    morgan_sims_score = np.mean(morgan_sims)\n",
    "    if verbose:\n",
    "        print('Average MACCS Similarity:', maccs_sims_score)\n",
    "        print('Average RDK Similarity:', rdk_sims_score)\n",
    "        print('Average Morgan Similarity:', morgan_sims_score)\n",
    "    return validity_score, maccs_sims_score, rdk_sims_score, morgan_sims_score\n",
    "\n",
    "from fcd import get_fcd, load_ref_model, canonical_smiles\n",
    "\n",
    "def evaluate3(gt_smis, ot_smis, verbose=False):\n",
    "#     gt_smis = []\n",
    "#     ot_smis = []\n",
    "\n",
    "#     with open(osp.join(input_file)) as f:\n",
    "#         reader = csv.DictReader(f, delimiter=\"\\t\", quoting=csv.QUOTE_NONE)\n",
    "#         for n, line in enumerate(reader):\n",
    "#             gt_smi = line['ground truth']\n",
    "#             ot_smi = line['output']\n",
    "#             if len(ot_smi) == 0: ot_smi = '[]'\n",
    "\n",
    "#             gt_smis.append(gt_smi)\n",
    "#             ot_smis.append(ot_smi)\n",
    "\n",
    "\n",
    "    model = load_ref_model()\n",
    "\n",
    "    canon_gt_smis = [w for w in canonical_smiles(gt_smis) if w is not None]\n",
    "    canon_ot_smis = [w for w in canonical_smiles(ot_smis) if w is not None]\n",
    "\n",
    "    fcd_sim_score = get_fcd(canon_gt_smis, canon_ot_smis, model)\n",
    "    if verbose:\n",
    "        print('FCD Similarity:', fcd_sim_score)\n",
    "\n",
    "    return fcd_sim_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Molecule Design"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-15T02:31:49.244221Z",
     "start_time": "2023-05-15T02:31:49.198491Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def evaluate_one_molecule_design(details_df):\n",
    "    bleu_scores = []\n",
    "    exact_match_scores = []\n",
    "    levenshtein_scores = []\n",
    "    validity_scores = []\n",
    "    maccs_sims_scores = []\n",
    "    rdk_sims_scores = []\n",
    "    morgan_sims_scores = []\n",
    "    fcd_scores = []\n",
    "\n",
    "    fcds = []\n",
    "    \n",
    "    for i in range(1, 6):\n",
    "        tem = list(zip(details_df['description'], details_df['SMILES'], details_df['pred_{}'.format(i)]))\n",
    "#         tem2 = list(zip(details_df['SMILES'], details_df['pred_{}'.format(i)]))\n",
    "        \n",
    "        bleu_score, exact_match_score, levenshtein_score, _ = evaluate1(tem)\n",
    "        validity_score, maccs_sims_score, rdk_sims_score, morgan_sims_score = evaluate2(tem)\n",
    "        fcd = evaluate3(list(details_df['SMILES']), list(details_df['pred_{}'.format(i)]))\n",
    "        \n",
    "        bleu_scores.append(bleu_score)\n",
    "        exact_match_scores.append(exact_match_score)\n",
    "        levenshtein_scores.append(levenshtein_score)\n",
    "        \n",
    "        validity_scores.append(validity_score)\n",
    "        maccs_sims_scores.append(maccs_sims_score)\n",
    "        rdk_sims_scores.append(rdk_sims_score)\n",
    "        morgan_sims_scores.append(morgan_sims_score)\n",
    "        \n",
    "        fcd_scores.append(fcd)\n",
    "    \n",
    "    bleu_mean = np.mean(bleu_scores)\n",
    "    exact_match_mean = np.mean(exact_match_scores)\n",
    "    levenshtein_mean = np.mean(levenshtein_scores)\n",
    "    validity_mean = np.mean(validity_scores)\n",
    "    maccs_sims_mean = np.mean(maccs_sims_scores)\n",
    "    rdk_sims_mean = np.mean(rdk_sims_scores)\n",
    "    morgan_sims_mean = np.mean(morgan_sims_scores)\n",
    "    fcd_mean = np.mean(fcd_scores)\n",
    "    \n",
    "\n",
    "    # cal std\n",
    "    bleu_variance = statistics.stdev(bleu_scores)\n",
    "    exact_match_variance = statistics.stdev(exact_match_scores) \n",
    "    levenshtein_variance = statistics.stdev(levenshtein_scores)\n",
    "    validity_variance = statistics.stdev(validity_scores)\n",
    "    maccs_sims_variance = statistics.stdev(maccs_sims_scores) \n",
    "    rdk_sims_variance = statistics.stdev(rdk_sims_scores) \n",
    "    morgan_sims_variance = statistics.stdev(morgan_sims_scores) \n",
    "    fcd_variance = statistics.stdev(fcd_scores) \n",
    "    \n",
    "    stds = [bleu_variance, exact_match_variance, levenshtein_variance, \n",
    "             validity_variance, maccs_sims_variance, rdk_sims_variance, \n",
    "             morgan_sims_variance, fcd_variance]\n",
    "    \n",
    "    return [bleu_mean, exact_match_mean, levenshtein_mean, validity_mean, maccs_sims_mean, rdk_sims_mean, morgan_sims_mean, fcd_mean], stds "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T21:47:12.938360Z",
     "start_time": "2023-05-14T21:47:08.244002Z"
    },
    "code_folding": [
     6
    ]
   },
   "outputs": [],
   "source": [
    "from transformers import BertTokenizerFast\n",
    "\n",
    "from nltk.translate.bleu_score import corpus_bleu\n",
    "from nltk.translate.meteor_score import meteor_score\n",
    "from rouge_score import rouge_scorer\n",
    "\n",
    "def evaluate4(raw_outputs, text_model='allenai/scibert_scivocab_uncased', text_trunc_length=512):\n",
    "    outputs = []\n",
    "    \n",
    "    for smiles, gt, output in raw_outputs:\n",
    "        out_tmp = output[6:] if output.startswith('[CLS] ') else output\n",
    "        outputs.append((smiles, gt, out_tmp))\n",
    "\n",
    "    text_tokenizer = BertTokenizerFast.from_pretrained(text_model)\n",
    "\n",
    "    bleu_scores = []\n",
    "    meteor_scores = []\n",
    "\n",
    "    references = []\n",
    "    hypotheses = []\n",
    "\n",
    "    for i, (smi, gt, out) in enumerate(outputs):\n",
    "\n",
    "        if i % 100 == 0: pass\n",
    "\n",
    "\n",
    "        gt_tokens = text_tokenizer.tokenize(gt, truncation=True, max_length=text_trunc_length,\n",
    "                                            padding='max_length')\n",
    "        gt_tokens = list(filter(('[PAD]').__ne__, gt_tokens))\n",
    "        gt_tokens = list(filter(('[CLS]').__ne__, gt_tokens))\n",
    "        gt_tokens = list(filter(('[SEP]').__ne__, gt_tokens))\n",
    "\n",
    "        out_tokens = text_tokenizer.tokenize(out, truncation=True, max_length=text_trunc_length,\n",
    "                                            padding='max_length')\n",
    "        out_tokens = list(filter(('[PAD]').__ne__, out_tokens))\n",
    "        out_tokens = list(filter(('[CLS]').__ne__, out_tokens))\n",
    "        out_tokens = list(filter(('[SEP]').__ne__, out_tokens))\n",
    "\n",
    "        references.append([gt_tokens])\n",
    "        hypotheses.append(out_tokens)\n",
    "\n",
    "        mscore = meteor_score([gt_tokens], out_tokens)\n",
    "        meteor_scores.append(mscore)\n",
    "\n",
    "    bleu2 = corpus_bleu(references, hypotheses, weights=(.5,.5))\n",
    "    bleu4 = corpus_bleu(references, hypotheses, weights=(.25,.25,.25,.25))\n",
    "\n",
    "#     print('BLEU-2 score:', bleu2)\n",
    "#     print('BLEU-4 score:', bleu4)\n",
    "    _meteor_score = np.mean(meteor_scores)\n",
    "#     print('Average Meteor score:', _meteor_score)\n",
    "\n",
    "    scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'])\n",
    "\n",
    "    rouge_scores = []\n",
    "\n",
    "    references = []\n",
    "    hypotheses = []\n",
    "\n",
    "    for i, (smi, gt, out) in enumerate(outputs):\n",
    "\n",
    "        rs = scorer.score(out, gt)\n",
    "        rouge_scores.append(rs)\n",
    "\n",
    "#     print('ROUGE score:')\n",
    "    rouge_1 = np.mean([rs['rouge1'].fmeasure for rs in rouge_scores])\n",
    "    rouge_2 = np.mean([rs['rouge2'].fmeasure for rs in rouge_scores])\n",
    "    rouge_l = np.mean([rs['rougeL'].fmeasure for rs in rouge_scores])\n",
    "#     print('rouge1:', rouge_1)\n",
    "#     print('rouge2:', rouge_2)\n",
    "#     print('rougeL:', rouge_l)\n",
    "    return bleu2, bleu4, rouge_1, rouge_2, rouge_l, _meteor_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T21:47:12.943276Z",
     "start_time": "2023-05-14T21:47:12.941015Z"
    }
   },
   "outputs": [],
   "source": [
    "import statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T21:47:12.953842Z",
     "start_time": "2023-05-14T21:47:12.945134Z"
    },
    "code_folding": [],
    "run_control": {
     "marked": false
    }
   },
   "outputs": [],
   "source": [
    "def evaluate_one_molecule_captioning(details_df):\n",
    "    bleu2_scores = []\n",
    "    bleu4_scores = []\n",
    "    rouge1_scores = []\n",
    "    rouge2_scores = []\n",
    "    rougel_scores = []\n",
    "    meteor_scores = []\n",
    "    \n",
    "    \n",
    "    for i in range(1, 6):\n",
    "        tem = list(zip(details_df['SMILES'], details_df['description'], details_df['pred_{}'.format(i)]))\n",
    "        bleu2, bleu4, rouge_1, rouge_2, rouge_l, _meteor_score = evaluate4(tem)\n",
    "        \n",
    "        bleu2_scores.append(bleu2)\n",
    "        bleu4_scores.append(bleu4)\n",
    "        rouge1_scores.append(rouge_1)\n",
    "        rouge2_scores.append(rouge_2)\n",
    "        rougel_scores.append(rouge_l)\n",
    "        meteor_scores.append(_meteor_score)\n",
    "    \n",
    "    mean_bleu2 = np.mean(bleu2_scores)\n",
    "    mean_bleu4 = np.mean(bleu4_scores)\n",
    "    mean_rouge1 = np.mean(rouge1_scores)\n",
    "    mean_rouge2 = np.mean(rouge2_scores)\n",
    "    mean_rougel = np.mean(rougel_scores)\n",
    "    mean_meteor = np.mean(meteor_scores)\n",
    "    \n",
    "    std_bleu2 = statistics.stdev(bleu2_scores)\n",
    "    std_bleu4 = statistics.stdev(bleu4_scores)\n",
    "    std_rouge1 = statistics.stdev(rouge1_scores)\n",
    "    std_rouge2 = statistics.stdev(rouge2_scores)\n",
    "    std_rougel = statistics.stdev(rougel_scores)\n",
    "    std_meteor = statistics.stdev(meteor_scores)\n",
    "    \n",
    "    stds = [std_bleu2, std_bleu4, std_rouge1, std_rouge2, std_rougel, std_meteor]\n",
    "    return [mean_bleu2, mean_bleu4, mean_rouge1, mean_rouge2, mean_rougel, mean_meteor], stds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-26T18:44:34.444637Z",
     "start_time": "2023-04-26T18:44:34.440935Z"
    }
   },
   "outputs": [],
   "source": [
    "performance_results_mol_design = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "performance_results_mol_captioning = []\n",
    "\n",
    "Models = ['davinci003', 'gpt4', 'gpt35']\n",
    "ICL_Sample_methods = ['Scaffold_SIM', 'Fixed_ICL']\n",
    "ICL_Samples = [5, 10]\n",
    "NAME_Format = ['SMILES']\n",
    "TASK_Types = ['molecule_design']\n",
    "detail_save_folder = folder + '/results/'\n",
    "\n",
    "params_idx = 0\n",
    "\n",
    "for sample_method in ICL_Sample_methods:\n",
    "    for sample_num in ICL_Samples:\n",
    "        for name in NAME_Format:\n",
    "            for task in TASK_Types:\n",
    "                for model in Models:\n",
    "                    params_idx += 1\n",
    "                    print(params_idx)\n",
    "                \n",
    "                    detail_predict_file = detail_save_folder + 'valid_{}_{}_{}_{}_{}.csv'.format(task, model, name, sample_num, sample_method)\n",
    "                    details_df = pd.read_csv(detail_predict_file)\n",
    "                    \n",
    "                    if task == 'molecule_design':\n",
    "                        metrics = evaluate_one_molecule_design(details_df)\n",
    "                        performance_results_mol_design.append([task, model, name, sample_num, sample_method] + metrics)\n",
    "                    elif task == 'molecule_captioning':\n",
    "                        metrics = evaluate_one_molecule_captioning(details_df)\n",
    "                        performance_results_mol_captioning.append([task, model, name, sample_num, sample_method] + metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-26T18:51:51.058571Z",
     "start_time": "2023-04-26T18:51:51.049470Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(performance_results_mol_design)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-26T18:55:00.217646Z",
     "start_time": "2023-04-26T18:55:00.210033Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['molecule_design',\n",
       "  'davinci003',\n",
       "  'SMILES',\n",
       "  5,\n",
       "  'Scaffold_SIM',\n",
       "  0.72453314670545,\n",
       "  0.07333333333333333,\n",
       "  22.833333333333332,\n",
       "  0.9066666666666666,\n",
       "  0.7305471568603197,\n",
       "  0.5952062533148453,\n",
       "  0.48357744668311353,\n",
       "  14.48228529190319]]"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "performance_results_mol_design"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-26T19:41:34.384946Z",
     "start_time": "2023-04-26T19:41:34.361542Z"
    }
   },
   "outputs": [],
   "source": [
    "valid.to_csv(\"mol_design_valid.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-26T18:21:00.440345Z",
     "start_time": "2023-04-26T18:21:00.342814Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(performance_results_mol_captioning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-26T18:41:37.453253Z",
     "start_time": "2023-04-26T18:41:37.445101Z"
    }
   },
   "outputs": [],
   "source": [
    "caption_cols = [\n",
    "    'task', 'model', 'name', 'sample_num', 'sample_method',\n",
    "    'bleu2', 'bleu4', 'rouge1', 'rouge2', 'rougel', 'meteor'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-26T18:41:43.130320Z",
     "start_time": "2023-04-26T18:41:43.118570Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "performance_results_mol_captioning_df = pd.DataFrame(performance_results_mol_captioning, columns=caption_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-26T18:42:33.629131Z",
     "start_time": "2023-04-26T18:42:33.261593Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "performance_results_mol_captioning_df.to_clipboard()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-26T19:40:00.875379Z",
     "start_time": "2023-04-26T19:40:00.755239Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CID</th>\n",
       "      <th>SMILES</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>11703004</td>\n",
       "      <td>C=C1CC[C@@H](C(C)=O)[C@@H](OC(=O)/C=C/C(C)(C)O...</td>\n",
       "      <td>The molecule is a sesquiterpenoid that is a ca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>679</th>\n",
       "      <td>5280825</td>\n",
       "      <td>COc1cccc(C/C=C(\\C)CC/C=C(\\C)CC/C=C(\\C)CC/C=C(\\...</td>\n",
       "      <td>The molecule is a monomethoxybenzene that is 6...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1253</th>\n",
       "      <td>3126</td>\n",
       "      <td>CCCCCCCCCCCCCCCC(O)C(N)CO</td>\n",
       "      <td>The molecule is an aminodiol that is octadecan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1618</th>\n",
       "      <td>8640</td>\n",
       "      <td>Nc1cccc2ccccc12</td>\n",
       "      <td>The molecule is a naphthylamine that is naphth...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203</th>\n",
       "      <td>16061139</td>\n",
       "      <td>CC/C=C\\C[C@H](O)/C=C/C=C\\C/C=C\\C=C\\[C@@H](O)/C...</td>\n",
       "      <td>The molecule is a member of the class of resol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2452</th>\n",
       "      <td>13785026</td>\n",
       "      <td>CC(C)c1cc2c(cc1O)[C@@]1(C=O)CCCC(C)(C)[C@@H]1CC2</td>\n",
       "      <td>The molecule is an abietane diterpenoid that i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2093</th>\n",
       "      <td>24014</td>\n",
       "      <td>Cl[Cu]Cl</td>\n",
       "      <td>The molecule is an inorganic chloride of coppe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2481</th>\n",
       "      <td>72276</td>\n",
       "      <td>Oc1cc(O)c2c(c1)O[C@H](c1ccc(O)c(O)c1)[C@H](O)C2</td>\n",
       "      <td>The molecule is a catechin with (2R,3R)-config...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1666</th>\n",
       "      <td>497299</td>\n",
       "      <td>CCCCCCCCCCCCCCCCCC(=O)OC[C@@H](O)COP(=O)([O-])...</td>\n",
       "      <td>The molecule is a lysophosphatidylcholine 18:0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485</th>\n",
       "      <td>46224585</td>\n",
       "      <td>CC(=O)N[C@H]1[C@H]([C@H](O)[C@H](O)COC(C)=O)O[...</td>\n",
       "      <td>The molecule is an amino disaccharide consisti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1642</th>\n",
       "      <td>71464635</td>\n",
       "      <td>CC(C)C[C@H](N)C(=O)N[C@@H](Cc1ccccc1)C(=O)N[C@...</td>\n",
       "      <td>The molecule is a tripeptide composed of L-leu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>511</th>\n",
       "      <td>5281416</td>\n",
       "      <td>O=c1ccc2cc(O)c(O)cc2o1</td>\n",
       "      <td>The molecule is a hydroxycoumarin that is umbe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015</th>\n",
       "      <td>68624</td>\n",
       "      <td>CCn1cc(C(=O)O)c(=O)c2cc(F)c(N3CCNC(C)C3)c(F)c2...</td>\n",
       "      <td>The molecule is the hydrochloride salt of lome...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>734</th>\n",
       "      <td>11338033</td>\n",
       "      <td>O=C(NC1CCNCC1)c1[nH]ncc1NC(=O)c1c(Cl)cccc1Cl</td>\n",
       "      <td>The molecule is a member of the class of pryra...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>101689</td>\n",
       "      <td>COc1ccc2c(c1)OC[C@]1(O)c3cc4c(cc3O[C@H]21)OCO4</td>\n",
       "      <td>The molecule is a member of the class of ptero...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1381</th>\n",
       "      <td>10167704</td>\n",
       "      <td>CN1/C(=C/C=C/c2cc[n+](CCC[N+](C)(C)C)cc2)Oc2cc...</td>\n",
       "      <td>The molecule is a cyanine dye and an organic i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2667</th>\n",
       "      <td>71296156</td>\n",
       "      <td>[NH3+]C[C@H]1O[C@H](O[C@H]2[C@H](O)[C@@H](O[C@...</td>\n",
       "      <td>The molecule is an organic cation obtained by ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2781</th>\n",
       "      <td>656867</td>\n",
       "      <td>C[N+](C)(C)CCOC(=O)CCC(=O)OCC[N+](C)(C)C.O.O.[...</td>\n",
       "      <td>The molecule is a hydrate that is the dihydrat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2858</th>\n",
       "      <td>5461069</td>\n",
       "      <td>CCCCCC/C=C\\CCCCCCCCCC(=O)[O-]</td>\n",
       "      <td>The molecule is a vaccenate(1-) that is the co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1233</th>\n",
       "      <td>44263322</td>\n",
       "      <td>CC(C)CCC[C@@H](C)[C@H]1CC[C@H]2C3=C(CC[C@]12C)...</td>\n",
       "      <td>The molecule is a 3beta-sterol that is 5alpha-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1497</th>\n",
       "      <td>165337</td>\n",
       "      <td>CC(C)C(=O)[O-]</td>\n",
       "      <td>The molecule is a branched, short-chain, satur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1412</th>\n",
       "      <td>23724475</td>\n",
       "      <td>C[C@H]1O[C@H](OP(=O)(O)OP(=O)(O)OC[C@H]2O[C@@H...</td>\n",
       "      <td>The molecule is uDP-4-dehydro-6-deoxy-D-glucos...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2816</th>\n",
       "      <td>72193634</td>\n",
       "      <td>CC(=O)O[C@H]1C(=O)/C=C/C(=O)[C@@]23C(=O)N[C@@H...</td>\n",
       "      <td>The molecule is a cytochalasan alkaloid that i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1178</th>\n",
       "      <td>135761165</td>\n",
       "      <td>Nc1nc2ncc(CNc3ccc(C(=O)N[C@@H](CCC(=O)[O-])C(=...</td>\n",
       "      <td>The molecule is the dicarboxylic acid dianion ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2396</th>\n",
       "      <td>946</td>\n",
       "      <td>O=N[O-]</td>\n",
       "      <td>The molecule is the nitrogen oxoanion formed b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1764</th>\n",
       "      <td>86583485</td>\n",
       "      <td>CCCCCCCCCCCCCCC(C(=O)[O-])C(=O)SCCNC(=O)CCNC(=...</td>\n",
       "      <td>The molecule is a 2-carboxyacyl CoA(5-) arisin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1514</th>\n",
       "      <td>51351814</td>\n",
       "      <td>CC(=O)N[C@H]1[C@H](O[C@@H]2[C@H](O)[C@@H](O)[C...</td>\n",
       "      <td>The molecule is an amino disaccharide consisti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>864</th>\n",
       "      <td>10563682</td>\n",
       "      <td>O=C1OC(O)C(Cc2ccccc2)=C1c1ccccc1</td>\n",
       "      <td>The molecule is a butenolide that is furan-2(5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>72193728</td>\n",
       "      <td>CC(C)(COP(=O)(O)OP(=O)(O)OC[C@H]1O[C@@H](n2cnc...</td>\n",
       "      <td>The molecule is an alpha,omega dicarboxyacyl-C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1706</th>\n",
       "      <td>4338370</td>\n",
       "      <td>Nc1ccc(N)c([N+](=O)[O-])c1</td>\n",
       "      <td>The molecule is a primary amino compound that ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            CID                                             SMILES  \\\n",
       "52     11703004  C=C1CC[C@@H](C(C)=O)[C@@H](OC(=O)/C=C/C(C)(C)O...   \n",
       "679     5280825  COc1cccc(C/C=C(\\C)CC/C=C(\\C)CC/C=C(\\C)CC/C=C(\\...   \n",
       "1253       3126                          CCCCCCCCCCCCCCCC(O)C(N)CO   \n",
       "1618       8640                                    Nc1cccc2ccccc12   \n",
       "203    16061139  CC/C=C\\C[C@H](O)/C=C/C=C\\C/C=C\\C=C\\[C@@H](O)/C...   \n",
       "2452   13785026   CC(C)c1cc2c(cc1O)[C@@]1(C=O)CCCC(C)(C)[C@@H]1CC2   \n",
       "2093      24014                                           Cl[Cu]Cl   \n",
       "2481      72276    Oc1cc(O)c2c(c1)O[C@H](c1ccc(O)c(O)c1)[C@H](O)C2   \n",
       "1666     497299  CCCCCCCCCCCCCCCCCC(=O)OC[C@@H](O)COP(=O)([O-])...   \n",
       "485    46224585  CC(=O)N[C@H]1[C@H]([C@H](O)[C@H](O)COC(C)=O)O[...   \n",
       "1642   71464635  CC(C)C[C@H](N)C(=O)N[C@@H](Cc1ccccc1)C(=O)N[C@...   \n",
       "511     5281416                             O=c1ccc2cc(O)c(O)cc2o1   \n",
       "2015      68624  CCn1cc(C(=O)O)c(=O)c2cc(F)c(N3CCNC(C)C3)c(F)c2...   \n",
       "734    11338033       O=C(NC1CCNCC1)c1[nH]ncc1NC(=O)c1c(Cl)cccc1Cl   \n",
       "70       101689     COc1ccc2c(c1)OC[C@]1(O)c3cc4c(cc3O[C@H]21)OCO4   \n",
       "1381   10167704  CN1/C(=C/C=C/c2cc[n+](CCC[N+](C)(C)C)cc2)Oc2cc...   \n",
       "2667   71296156  [NH3+]C[C@H]1O[C@H](O[C@H]2[C@H](O)[C@@H](O[C@...   \n",
       "2781     656867  C[N+](C)(C)CCOC(=O)CCC(=O)OCC[N+](C)(C)C.O.O.[...   \n",
       "2858    5461069                      CCCCCC/C=C\\CCCCCCCCCC(=O)[O-]   \n",
       "1233   44263322  CC(C)CCC[C@@H](C)[C@H]1CC[C@H]2C3=C(CC[C@]12C)...   \n",
       "1497     165337                                     CC(C)C(=O)[O-]   \n",
       "1412   23724475  C[C@H]1O[C@H](OP(=O)(O)OP(=O)(O)OC[C@H]2O[C@@H...   \n",
       "2816   72193634  CC(=O)O[C@H]1C(=O)/C=C/C(=O)[C@@]23C(=O)N[C@@H...   \n",
       "1178  135761165  Nc1nc2ncc(CNc3ccc(C(=O)N[C@@H](CCC(=O)[O-])C(=...   \n",
       "2396        946                                            O=N[O-]   \n",
       "1764   86583485  CCCCCCCCCCCCCCC(C(=O)[O-])C(=O)SCCNC(=O)CCNC(=...   \n",
       "1514   51351814  CC(=O)N[C@H]1[C@H](O[C@@H]2[C@H](O)[C@@H](O)[C...   \n",
       "864    10563682                   O=C1OC(O)C(Cc2ccccc2)=C1c1ccccc1   \n",
       "80     72193728  CC(C)(COP(=O)(O)OP(=O)(O)OC[C@H]1O[C@@H](n2cnc...   \n",
       "1706    4338370                         Nc1ccc(N)c([N+](=O)[O-])c1   \n",
       "\n",
       "                                            description  \n",
       "52    The molecule is a sesquiterpenoid that is a ca...  \n",
       "679   The molecule is a monomethoxybenzene that is 6...  \n",
       "1253  The molecule is an aminodiol that is octadecan...  \n",
       "1618  The molecule is a naphthylamine that is naphth...  \n",
       "203   The molecule is a member of the class of resol...  \n",
       "2452  The molecule is an abietane diterpenoid that i...  \n",
       "2093  The molecule is an inorganic chloride of coppe...  \n",
       "2481  The molecule is a catechin with (2R,3R)-config...  \n",
       "1666  The molecule is a lysophosphatidylcholine 18:0...  \n",
       "485   The molecule is an amino disaccharide consisti...  \n",
       "1642  The molecule is a tripeptide composed of L-leu...  \n",
       "511   The molecule is a hydroxycoumarin that is umbe...  \n",
       "2015  The molecule is the hydrochloride salt of lome...  \n",
       "734   The molecule is a member of the class of pryra...  \n",
       "70    The molecule is a member of the class of ptero...  \n",
       "1381  The molecule is a cyanine dye and an organic i...  \n",
       "2667  The molecule is an organic cation obtained by ...  \n",
       "2781  The molecule is a hydrate that is the dihydrat...  \n",
       "2858  The molecule is a vaccenate(1-) that is the co...  \n",
       "1233  The molecule is a 3beta-sterol that is 5alpha-...  \n",
       "1497  The molecule is a branched, short-chain, satur...  \n",
       "1412  The molecule is uDP-4-dehydro-6-deoxy-D-glucos...  \n",
       "2816  The molecule is a cytochalasan alkaloid that i...  \n",
       "1178  The molecule is the dicarboxylic acid dianion ...  \n",
       "2396  The molecule is the nitrogen oxoanion formed b...  \n",
       "1764  The molecule is a 2-carboxyacyl CoA(5-) arisin...  \n",
       "1514  The molecule is an amino disaccharide consisti...  \n",
       "864   The molecule is a butenolide that is furan-2(5...  \n",
       "80    The molecule is an alpha,omega dicarboxyacyl-C...  \n",
       "1706  The molecule is a primary amino compound that ...  "
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "icl: samples - performance\n",
    "\n",
    "5 samples - 2/10   7/30\n",
    "\n",
    "10 samples - 3/10  10/30\n",
    "\n",
    "20 samples - 4/10  10/30\n",
    "\n",
    "30 samples - 2/10\n",
    "\n",
    "40 samples - X\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test (Run and save)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T23:41:21.639090Z",
     "start_time": "2023-04-30T23:41:21.406514Z"
    }
   },
   "outputs": [],
   "source": [
    "test = test.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T23:41:25.186876Z",
     "start_time": "2023-04-30T23:41:25.180719Z"
    }
   },
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T23:45:40.276843Z",
     "start_time": "2023-04-30T23:45:40.269580Z"
    }
   },
   "outputs": [],
   "source": [
    "params = [\n",
    "    ('Scaffold_SIM', 10, 'SMILES', 'gpt4'),\n",
    "    ('Scaffold_SIM', 10, 'SMILES', 'davinci003'),\n",
    "    ('Scaffold_SIM', 10, 'SMILES', 'gpt35'),\n",
    "    \n",
    "    ('Scaffold_SIM', 5, 'SMILES', 'gpt4'),\n",
    "    ('Fixed_ICL', 10, 'SMILES', 'gpt4'),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-30T23:41:32.063873Z",
     "start_time": "2023-04-30T23:41:32.056872Z"
    }
   },
   "outputs": [],
   "source": [
    "test = test.head(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-01T14:01:13.429907Z",
     "start_time": "2023-05-01T13:57:53.049505Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Molecule_Design/ChEBI-20_data//results_test/valid_molecule_captioning_gpt4_SMILES_10_Scaffold_SIM.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [00:00, 10908.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Molecule_Design/ChEBI-20_data//results_test/valid_molecule_captioning_davinci003_SMILES_10_Scaffold_SIM.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [00:00, 16868.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Molecule_Design/ChEBI-20_data//results_test/valid_molecule_captioning_gpt35_SMILES_10_Scaffold_SIM.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [00:00, 14488.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Molecule_Design/ChEBI-20_data//results_test/valid_molecule_captioning_gpt4_SMILES_5_Scaffold_SIM.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [00:00, 14118.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Molecule_Design/ChEBI-20_data//results_test/valid_molecule_captioning_gpt4_SMILES_10_Fixed_ICL.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [00:00, 9556.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Molecule_Design/ChEBI-20_data//results_test/valid_molecule_design_gpt4_SMILES_10_Scaffold_SIM.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [00:00, 11499.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Molecule_Design/ChEBI-20_data//results_test/valid_molecule_design_davinci003_SMILES_10_Scaffold_SIM.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [00:00, 11047.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Molecule_Design/ChEBI-20_data//results_test/valid_molecule_design_gpt35_SMILES_10_Scaffold_SIM.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [00:00, 15736.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Molecule_Design/ChEBI-20_data//results_test/valid_molecule_design_gpt4_SMILES_5_Scaffold_SIM.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [00:00, 14039.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Molecule_Design/ChEBI-20_data//results_test/valid_molecule_design_gpt4_SMILES_10_Fixed_ICL.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [03:19,  2.00s/it]\n"
     ]
    }
   ],
   "source": [
    "# format the running grid search \n",
    "# evaluate and save\n",
    "\n",
    "detail_save_folder = folder + '/results_test/'\n",
    "TASK_Types = ['molecule_captioning', 'molecule_design']\n",
    "# scaffold search names \n",
    "scaffold_col_smiles_name = 'SMILES'\n",
    "\n",
    "for task in TASK_Types:\n",
    "    for sample_method, sample_num, name, model in params:\n",
    "        \n",
    "        details_results = []\n",
    "        performance_results = []\n",
    "\n",
    "        detail_predict_file = detail_save_folder + 'valid_{}_{}_{}_{}_{}.csv'.format(task, model, name, sample_num, sample_method)\n",
    "        log_file = detail_save_folder + 'valid_{}_{}_{}_{}_{}.log'.format(task, model, name, sample_num, sample_method)\n",
    "        performance_file = detail_save_folder + 'valid_performance_{}_{}_{}_{}_{}.csv'.format(task, model, name, sample_num, sample_method)\n",
    "        print(detail_predict_file)\n",
    "        \n",
    "        \n",
    "        if os.path.exists(performance_file):\n",
    "            print(\"{} exits, continue to next params\".format(performance_file))\n",
    "            continue\n",
    "        \n",
    "\n",
    "        # append new date\n",
    "        # Get the current date and time\n",
    "        now = datetime.datetime.now()\n",
    "        # Convert the date and time to a string\n",
    "        date_time_str = now.strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "        with open(log_file, \"a\") as file:\n",
    "            file.write(\"=\" * 30 + date_time_str + \"=\" * 30 + \"\\n\")\n",
    "\n",
    "        # init var\n",
    "        predicted_products = []\n",
    "        \n",
    "        # restore previous data and skip \n",
    "        previous_index = []\n",
    "        if os.path.exists(detail_predict_file):\n",
    "            previous = pd.read_csv(detail_predict_file)\n",
    "            previous_index = list(previous['index'])\n",
    "\n",
    "            # previous data => details_results \n",
    "            details_results = previous.values.tolist()\n",
    "            predicted_products = previous[['pred_1', 'pred_2', 'pred_3', 'pred_4', 'pred_5']].values.tolist()\n",
    "\n",
    "        input_col, output_col = get_input_output_columns_by_task(task)\n",
    "\n",
    "        # query\n",
    "        for idx, row in tqdm(test.iterrows()):\n",
    "            reactant = row[input_col]\n",
    "            product = row[output_col]\n",
    "            index = row['index']\n",
    "        \n",
    "            if index in previous_index:\n",
    "                continue\n",
    "\n",
    "            # ICL examples\n",
    "            if sample_method == 'Fixed_ICL':\n",
    "                chunk = train.sample(sample_num, random_state=42)\n",
    "            elif sample_method == 'Scaffold_SIM':\n",
    "                if input_col == scaffold_col_smiles_name:\n",
    "                    print(\"using scaffold\")\n",
    "                    sim = top_n_scaffold_similar_molecules(reactant, list(train['scaffold_fp']), list(train[scaffold_col_smiles_name]), n=sample_num)\n",
    "                else:\n",
    "                    # similarity by leven\n",
    "                    print(\"using difflib\")\n",
    "                    sim = top_n_similar_strings(reactant, list(train[input_col]), n=sample_num)\n",
    "                chunk = train[train[input_col].isin(sim)]\n",
    "\n",
    "            examples = list(zip(chunk[input_col].values, chunk[output_col].values))\n",
    "\n",
    "            # build prompt and save\n",
    "            prompt = create_prompt(reactant, examples, task)\n",
    "\n",
    "            with open(log_file, \"a\") as file:\n",
    "                file.write(prompt + \"\\n\")\n",
    "                file.write(\"=\" * 50 + \"\\n\")\n",
    "                \n",
    "            try:\n",
    "                # different model \n",
    "                if model == 'davinci003':\n",
    "                    predicted_product = generate_response_by_davinci(prompt)\n",
    "                elif model == 'gpt35':\n",
    "                    predicted_product = generate_response_by_gpt35(prompt)\n",
    "                elif model == 'gpt4':\n",
    "                    predicted_product = generate_response_by_gpt4(prompt)\n",
    "            except Exception as e:\n",
    "                print(e)\n",
    "                if \"Please reduce your prompt; or completion length.\" in str(e) or \"Please reduce the length of the messages.\" in str(e):\n",
    "                    \n",
    "                    try:\n",
    "                        fixed_sample_num1 = sample_num - 5\n",
    "                        print(\"using {} icl examples\".format(fixed_sample_num1))\n",
    "                        \n",
    "                        # ICL examples\n",
    "                        if sample_method == 'Fixed_ICL':\n",
    "                            chunk = train.sample(fixed_sample_num1, random_state=42)\n",
    "                        elif sample_method == 'Scaffold_SIM':\n",
    "                            if input_col == scaffold_col_smiles_name:\n",
    "                                sim = top_n_scaffold_similar_molecules(reactant, list(train['scaffold_fp']), list(train[scaffold_col_smiles_name]), n=fixed_sample_num1)\n",
    "                            else:\n",
    "                                # similarity by leven\n",
    "                                sim = top_n_similar_strings(reactant, list(train[input_col]), n=fixed_sample_num1)\n",
    "                            chunk = train[train[input_col].isin(sim)]\n",
    "\n",
    "                        examples = list(zip(chunk[input_col].values, chunk[output_col].values))\n",
    "\n",
    "                        # build prompt and save\n",
    "                        prompt = create_prompt(reactant, examples, task)\n",
    "\n",
    "                        with open(log_file, \"a\") as file:\n",
    "                            file.write(prompt + \"\\n\")\n",
    "                            file.write(\"=\" * 50 + \"\\n\")\n",
    "                        \n",
    "                        # different model \n",
    "                        if model == 'davinci003':\n",
    "                            predicted_product = generate_response_by_davinci(prompt)\n",
    "                        elif model == 'gpt35':\n",
    "                            predicted_product = generate_response_by_gpt35(prompt)\n",
    "                        elif model == 'gpt4':\n",
    "                            predicted_product = generate_response_by_gpt4(prompt)\n",
    "                            \n",
    "                    except Exception as e:\n",
    "                        print(e)\n",
    "                        if \"Please reduce your prompt; or completion length.\" in str(e):\n",
    "                            fixed_sample_num2 = sample_num - 10\n",
    "                            print(\"using {} icl examples\".format(fixed_sample_num2))\n",
    "                            \n",
    "                            # ICL examples\n",
    "                            if sample_method == 'Fixed_ICL':\n",
    "                                chunk = train.sample(fixed_sample_num2, random_state=42)\n",
    "                            elif sample_method == 'Scaffold_SIM':\n",
    "                                if input_col == scaffold_col_smiles_name:\n",
    "                                    sim = top_n_scaffold_similar_molecules(reactant, list(train['scaffold_fp']), list(train[scaffold_col_smiles_name]), n=fixed_sample_num2)\n",
    "                                else:\n",
    "                                    # similarity by leven\n",
    "                                    sim = top_n_similar_strings(reactant, list(train[input_col]), n=fixed_sample_num2)\n",
    "                                chunk = train[train[input_col].isin(sim)]\n",
    "\n",
    "                            examples = list(zip(chunk[input_col].values, chunk[output_col].values))\n",
    "\n",
    "                            # build prompt and save\n",
    "                            prompt = create_prompt(reactant, examples, task)\n",
    "\n",
    "                            with open(log_file, \"a\") as file:\n",
    "                                file.write(prompt + \"\\n\")\n",
    "                                file.write(\"=\" * 50 + \"\\n\")\n",
    "\n",
    "                            # different model \n",
    "                            if model == 'davinci003':\n",
    "                                predicted_product = generate_response_by_davinci(prompt)\n",
    "                            elif model == 'gpt35':\n",
    "                                predicted_product = generate_response_by_gpt35(prompt)\n",
    "                            elif model == 'gpt4':\n",
    "                                predicted_product = generate_response_by_gpt4(prompt)\n",
    "\n",
    "                        else:\n",
    "                            raise RuntimeError(\"err\")\n",
    "                \n",
    "                else:\n",
    "                    raise RuntimeError(\"err\")\n",
    "\n",
    "            predicted_products.append(predicted_product)\n",
    "            details_results.append([index, reactant] + [product] + predicted_product)\n",
    "            \n",
    "            details_df = pd.DataFrame(details_results, columns=['index', input_col, output_col, 'pred_1', 'pred_2', 'pred_3', 'pred_4', 'pred_5'])\n",
    "            details_df.to_csv(detail_predict_file, index=False)\n",
    "            \n",
    "#             # evaluate\n",
    "#             if len(predicted_products) % 50 == 0:         \n",
    "#                 # evaluate\n",
    "#                 acc_list = []\n",
    "#                 for repeat in range(5):\n",
    "#                     tpredicted_products = [i[repeat] for i in predicted_products]\n",
    "#                     correct = 0\n",
    "#                     all_sample_num = len(predicted_products)\n",
    "#                     for idx, gt in enumerate(list(test.head(all_sample_num)[output_col])):\n",
    "#                         pred = tpredicted_products[idx]\n",
    "#                         if task in ['retro']:\n",
    "#                             try:\n",
    "#                                 mol = Chem.MolFromSmiles(pred)\n",
    "#                                 pred = Chem.MolToSmiles(mol)\n",
    "#                             except Exception as e:\n",
    "#                                 continue\n",
    "\n",
    "#                         if gt == pred:\n",
    "#                             correct += 1\n",
    "#                     acc = correct / all_sample_num\n",
    "#                     acc_list.append(acc)\n",
    "#                 print(np.mean(acc_list))\n",
    "            \n",
    "#         # evaluate\n",
    "#         acc_list = []\n",
    "#         for repeat in range(5):\n",
    "#             tpredicted_products = [i[repeat] for i in predicted_products]\n",
    "#             correct = 0\n",
    "#             all_sample_num = len(test)\n",
    "#             for idx, gt in enumerate(list(test[output_col])):\n",
    "#                 pred = tpredicted_products[idx]\n",
    "#                 if task in ['retro']:\n",
    "#                     try:\n",
    "#                         mol = Chem.MolFromSmiles(pred)\n",
    "#                         pred = Chem.MolToSmiles(mol)\n",
    "#                     except Exception as e:\n",
    "#                         continue\n",
    "\n",
    "#                 if gt == pred:\n",
    "#                     correct += 1\n",
    "#             acc = correct / all_sample_num\n",
    "#             acc_list.append(acc)\n",
    "\n",
    "        # save to file\n",
    "#         performance_results.append([task, model, name, sample_num, sample_method, np.mean(acc_list)] + acc_list)\n",
    "#         print(performance_results)\n",
    "\n",
    "#         # performance save based on the task\n",
    "#         tem = pd.DataFrame(performance_results, columns=['task', 'model', 'name', 'sample_num', 'sample_method', 'avg_metric'] + ['metric_{}'.format(i) for i in range(5)])\n",
    "#         tem.to_csv(performance_file, index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-01T15:57:52.764946Z",
     "start_time": "2023-05-01T15:57:52.714755Z"
    }
   },
   "outputs": [],
   "source": [
    "test.to_csv(\"mol_design_test.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T21:46:57.670721Z",
     "start_time": "2023-05-14T21:46:57.663304Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T21:46:57.888408Z",
     "start_time": "2023-05-14T21:46:57.881422Z"
    }
   },
   "outputs": [],
   "source": [
    "params = [\n",
    "    ('Scaffold_SIM', 10, 'SMILES', 'gpt4'),\n",
    "    ('Scaffold_SIM', 10, 'SMILES', 'davinci003'),\n",
    "    ('Scaffold_SIM', 10, 'SMILES', 'gpt35'),\n",
    "    \n",
    "    ('Scaffold_SIM', 5, 'SMILES', 'gpt4'),\n",
    "    ('Fixed_ICL', 10, 'SMILES', 'gpt4'),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T21:47:50.538968Z",
     "start_time": "2023-05-14T21:47:29.318351Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n"
     ]
    }
   ],
   "source": [
    "performance_results_mol_captioning = []\n",
    "\n",
    "task = 'molecule_captioning'\n",
    "\n",
    "params_idx = 0\n",
    "\n",
    "for sample_method, sample_num, name, model in params:\n",
    "    params_idx += 1\n",
    "    print(params_idx)\n",
    "\n",
    "    detail_predict_file = detail_save_folder + 'valid_{}_{}_{}_{}_{}.csv'.format(task, model, name, sample_num, sample_method)\n",
    "    details_df = pd.read_csv(detail_predict_file)\n",
    "\n",
    "    if task == 'molecule_design':\n",
    "        metrics = evaluate_one_molecule_design(details_df)\n",
    "        performance_results_mol_design.append([task, model, name, sample_num, sample_method] + metrics)\n",
    "    elif task == 'molecule_captioning':\n",
    "        metrics, stds = evaluate_one_molecule_captioning(details_df)\n",
    "        performance_results_mol_captioning.append([task, model, name, sample_num, sample_method] + metrics + stds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T21:48:15.841697Z",
     "start_time": "2023-05-14T21:48:15.837457Z"
    }
   },
   "outputs": [],
   "source": [
    "metrics_cols = [\n",
    "    'bleu2', 'bleu4', 'rouge1', 'rouge2', 'rougel', 'meteor', \n",
    "]\n",
    "\n",
    "caption_cols = [\n",
    "    'task', 'model', 'name', 'sample_num', 'sample_method',\n",
    "]\n",
    "\n",
    "std_cols = [i + \"_std\" for i in metrics_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T21:57:25.440948Z",
     "start_time": "2023-05-14T21:57:25.347215Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "performance_results_mol_captioning_df = pd.DataFrame(performance_results_mol_captioning, columns=caption_cols + metrics_cols + std_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T21:57:27.742593Z",
     "start_time": "2023-05-14T21:57:27.707251Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Molecule_Design/ChEBI-20_data//results_test/'"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "detail_save_folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T22:45:25.836330Z",
     "start_time": "2023-05-14T22:45:25.810813Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task</th>\n",
       "      <th>model</th>\n",
       "      <th>name</th>\n",
       "      <th>sample_num</th>\n",
       "      <th>sample_method</th>\n",
       "      <th>bleu2</th>\n",
       "      <th>bleu4</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougel</th>\n",
       "      <th>meteor</th>\n",
       "      <th>bleu2_std</th>\n",
       "      <th>bleu4_std</th>\n",
       "      <th>rouge1_std</th>\n",
       "      <th>rouge2_std</th>\n",
       "      <th>rougel_std</th>\n",
       "      <th>meteor_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.464337</td>\n",
       "      <td>0.365360</td>\n",
       "      <td>0.545179</td>\n",
       "      <td>0.362488</td>\n",
       "      <td>0.459131</td>\n",
       "      <td>0.518980</td>\n",
       "      <td>0.007529</td>\n",
       "      <td>0.008039</td>\n",
       "      <td>0.003315</td>\n",
       "      <td>0.002863</td>\n",
       "      <td>0.007351</td>\n",
       "      <td>0.005048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>davinci003</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.488202</td>\n",
       "      <td>0.391264</td>\n",
       "      <td>0.532238</td>\n",
       "      <td>0.358805</td>\n",
       "      <td>0.464974</td>\n",
       "      <td>0.478031</td>\n",
       "      <td>0.011140</td>\n",
       "      <td>0.011708</td>\n",
       "      <td>0.008313</td>\n",
       "      <td>0.009813</td>\n",
       "      <td>0.007998</td>\n",
       "      <td>0.010948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt35</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.467592</td>\n",
       "      <td>0.368161</td>\n",
       "      <td>0.533540</td>\n",
       "      <td>0.354869</td>\n",
       "      <td>0.457351</td>\n",
       "      <td>0.496697</td>\n",
       "      <td>0.009957</td>\n",
       "      <td>0.010420</td>\n",
       "      <td>0.005442</td>\n",
       "      <td>0.006771</td>\n",
       "      <td>0.005710</td>\n",
       "      <td>0.005346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>5</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.456031</td>\n",
       "      <td>0.357472</td>\n",
       "      <td>0.540109</td>\n",
       "      <td>0.354757</td>\n",
       "      <td>0.454905</td>\n",
       "      <td>0.505097</td>\n",
       "      <td>0.003267</td>\n",
       "      <td>0.004230</td>\n",
       "      <td>0.004614</td>\n",
       "      <td>0.006798</td>\n",
       "      <td>0.004748</td>\n",
       "      <td>0.004574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Fixed_ICL</td>\n",
       "      <td>0.259944</td>\n",
       "      <td>0.140320</td>\n",
       "      <td>0.392945</td>\n",
       "      <td>0.180456</td>\n",
       "      <td>0.308746</td>\n",
       "      <td>0.320015</td>\n",
       "      <td>0.007126</td>\n",
       "      <td>0.006514</td>\n",
       "      <td>0.004061</td>\n",
       "      <td>0.005691</td>\n",
       "      <td>0.004215</td>\n",
       "      <td>0.006952</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  task       model    name  sample_num sample_method  \\\n",
       "0  molecule_captioning        gpt4  SMILES          10  Scaffold_SIM   \n",
       "1  molecule_captioning  davinci003  SMILES          10  Scaffold_SIM   \n",
       "2  molecule_captioning       gpt35  SMILES          10  Scaffold_SIM   \n",
       "3  molecule_captioning        gpt4  SMILES           5  Scaffold_SIM   \n",
       "4  molecule_captioning        gpt4  SMILES          10     Fixed_ICL   \n",
       "\n",
       "      bleu2     bleu4    rouge1    rouge2    rougel    meteor  bleu2_std  \\\n",
       "0  0.464337  0.365360  0.545179  0.362488  0.459131  0.518980   0.007529   \n",
       "1  0.488202  0.391264  0.532238  0.358805  0.464974  0.478031   0.011140   \n",
       "2  0.467592  0.368161  0.533540  0.354869  0.457351  0.496697   0.009957   \n",
       "3  0.456031  0.357472  0.540109  0.354757  0.454905  0.505097   0.003267   \n",
       "4  0.259944  0.140320  0.392945  0.180456  0.308746  0.320015   0.007126   \n",
       "\n",
       "   bleu4_std  rouge1_std  rouge2_std  rougel_std  meteor_std  \n",
       "0   0.008039    0.003315    0.002863    0.007351    0.005048  \n",
       "1   0.011708    0.008313    0.009813    0.007998    0.010948  \n",
       "2   0.010420    0.005442    0.006771    0.005710    0.005346  \n",
       "3   0.004230    0.004614    0.006798    0.004748    0.004574  \n",
       "4   0.006514    0.004061    0.005691    0.004215    0.006952  "
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "performance_results_mol_captioning_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T22:16:19.079334Z",
     "start_time": "2023-05-14T22:16:19.017460Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task</th>\n",
       "      <th>model</th>\n",
       "      <th>name</th>\n",
       "      <th>sample_num</th>\n",
       "      <th>sample_method</th>\n",
       "      <th>bleu2</th>\n",
       "      <th>bleu4</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougel</th>\n",
       "      <th>meteor</th>\n",
       "      <th>bleu2_std</th>\n",
       "      <th>bleu4_std</th>\n",
       "      <th>rouge1_std</th>\n",
       "      <th>rouge2_std</th>\n",
       "      <th>rougel_std</th>\n",
       "      <th>meteor_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.464337</td>\n",
       "      <td>0.365360</td>\n",
       "      <td>0.545179</td>\n",
       "      <td>0.362488</td>\n",
       "      <td>0.459131</td>\n",
       "      <td>0.518980</td>\n",
       "      <td>0.007529</td>\n",
       "      <td>0.008039</td>\n",
       "      <td>0.003315</td>\n",
       "      <td>0.002863</td>\n",
       "      <td>0.007351</td>\n",
       "      <td>0.005048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>davinci003</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.488202</td>\n",
       "      <td>0.391264</td>\n",
       "      <td>0.532238</td>\n",
       "      <td>0.358805</td>\n",
       "      <td>0.464974</td>\n",
       "      <td>0.478031</td>\n",
       "      <td>0.011140</td>\n",
       "      <td>0.011708</td>\n",
       "      <td>0.008313</td>\n",
       "      <td>0.009813</td>\n",
       "      <td>0.007998</td>\n",
       "      <td>0.010948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt35</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.467592</td>\n",
       "      <td>0.368161</td>\n",
       "      <td>0.533540</td>\n",
       "      <td>0.354869</td>\n",
       "      <td>0.457351</td>\n",
       "      <td>0.496697</td>\n",
       "      <td>0.009957</td>\n",
       "      <td>0.010420</td>\n",
       "      <td>0.005442</td>\n",
       "      <td>0.006771</td>\n",
       "      <td>0.005710</td>\n",
       "      <td>0.005346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>5</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.456031</td>\n",
       "      <td>0.357472</td>\n",
       "      <td>0.540109</td>\n",
       "      <td>0.354757</td>\n",
       "      <td>0.454905</td>\n",
       "      <td>0.505097</td>\n",
       "      <td>0.003267</td>\n",
       "      <td>0.004230</td>\n",
       "      <td>0.004614</td>\n",
       "      <td>0.006798</td>\n",
       "      <td>0.004748</td>\n",
       "      <td>0.004574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Fixed_ICL</td>\n",
       "      <td>0.259944</td>\n",
       "      <td>0.140320</td>\n",
       "      <td>0.392945</td>\n",
       "      <td>0.180456</td>\n",
       "      <td>0.308746</td>\n",
       "      <td>0.320015</td>\n",
       "      <td>0.007126</td>\n",
       "      <td>0.006514</td>\n",
       "      <td>0.004061</td>\n",
       "      <td>0.005691</td>\n",
       "      <td>0.004215</td>\n",
       "      <td>0.006952</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  task       model    name  sample_num sample_method  \\\n",
       "0  molecule_captioning        gpt4  SMILES          10  Scaffold_SIM   \n",
       "1  molecule_captioning  davinci003  SMILES          10  Scaffold_SIM   \n",
       "2  molecule_captioning       gpt35  SMILES          10  Scaffold_SIM   \n",
       "3  molecule_captioning        gpt4  SMILES           5  Scaffold_SIM   \n",
       "4  molecule_captioning        gpt4  SMILES          10     Fixed_ICL   \n",
       "\n",
       "      bleu2     bleu4    rouge1    rouge2    rougel    meteor  bleu2_std  \\\n",
       "0  0.464337  0.365360  0.545179  0.362488  0.459131  0.518980   0.007529   \n",
       "1  0.488202  0.391264  0.532238  0.358805  0.464974  0.478031   0.011140   \n",
       "2  0.467592  0.368161  0.533540  0.354869  0.457351  0.496697   0.009957   \n",
       "3  0.456031  0.357472  0.540109  0.354757  0.454905  0.505097   0.003267   \n",
       "4  0.259944  0.140320  0.392945  0.180456  0.308746  0.320015   0.007126   \n",
       "\n",
       "   bleu4_std  rouge1_std  rouge2_std  rougel_std  meteor_std  \n",
       "0   0.008039    0.003315    0.002863    0.007351    0.005048  \n",
       "1   0.011708    0.008313    0.009813    0.007998    0.010948  \n",
       "2   0.010420    0.005442    0.006771    0.005710    0.005346  \n",
       "3   0.004230    0.004614    0.006798    0.004748    0.004574  \n",
       "4   0.006514    0.004061    0.005691    0.004215    0.006952  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# avg + std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T14:54:26.195416Z",
     "start_time": "2023-05-16T14:54:26.182389Z"
    }
   },
   "outputs": [],
   "source": [
    "performance_results_mol_captioning_df.to_csv(detail_save_folder + \"molecule_captioning_summary.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-13T17:08:17.717869Z",
     "start_time": "2023-05-13T17:08:17.188438Z"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'performance_results_mol_captioning_df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[16], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mperformance_results_mol_captioning_df\u001b[49m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'performance_results_mol_captioning_df' is not defined"
     ]
    }
   ],
   "source": [
    "performance_results_mol_captioning_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-13T17:08:35.865317Z",
     "start_time": "2023-05-13T17:08:35.855362Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-13T17:09:37.239540Z",
     "start_time": "2023-05-13T17:09:37.235644Z"
    }
   },
   "outputs": [],
   "source": [
    "detail_save_folder = folder + '/results_test/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T14:54:21.001078Z",
     "start_time": "2023-05-16T14:54:20.950231Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task</th>\n",
       "      <th>model</th>\n",
       "      <th>name</th>\n",
       "      <th>sample_num</th>\n",
       "      <th>sample_method</th>\n",
       "      <th>bleu2</th>\n",
       "      <th>bleu4</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougel</th>\n",
       "      <th>meteor</th>\n",
       "      <th>bleu2_std</th>\n",
       "      <th>bleu4_std</th>\n",
       "      <th>rouge1_std</th>\n",
       "      <th>rouge2_std</th>\n",
       "      <th>rougel_std</th>\n",
       "      <th>meteor_std</th>\n",
       "      <th>Method</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>GPT-4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold</td>\n",
       "      <td>0.464 $\\pm$ 0.008</td>\n",
       "      <td>0.365 $\\pm$ 0.008</td>\n",
       "      <td>0.545 $\\pm$ 0.003</td>\n",
       "      <td>0.362 $\\pm$ 0.003</td>\n",
       "      <td>0.459 $\\pm$ 0.007</td>\n",
       "      <td>0.519 $\\pm$ 0.005</td>\n",
       "      <td>0.007529</td>\n",
       "      <td>0.008039</td>\n",
       "      <td>0.003315</td>\n",
       "      <td>0.002863</td>\n",
       "      <td>0.007351</td>\n",
       "      <td>0.005048</td>\n",
       "      <td>GPT-4 (Scaffold, #ICL examples: 10)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>Davinci-003</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold</td>\n",
       "      <td>0.488 $\\pm$ 0.011</td>\n",
       "      <td>0.391 $\\pm$ 0.012</td>\n",
       "      <td>0.532 $\\pm$ 0.008</td>\n",
       "      <td>0.359 $\\pm$ 0.010</td>\n",
       "      <td>0.465 $\\pm$ 0.008</td>\n",
       "      <td>0.478 $\\pm$ 0.011</td>\n",
       "      <td>0.011140</td>\n",
       "      <td>0.011708</td>\n",
       "      <td>0.008313</td>\n",
       "      <td>0.009813</td>\n",
       "      <td>0.007998</td>\n",
       "      <td>0.010948</td>\n",
       "      <td>Davinci-003 (Scaffold, #ICL examples: 10)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>GPT-3.5</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold</td>\n",
       "      <td>0.468 $\\pm$ 0.010</td>\n",
       "      <td>0.368 $\\pm$ 0.010</td>\n",
       "      <td>0.534 $\\pm$ 0.005</td>\n",
       "      <td>0.355 $\\pm$ 0.007</td>\n",
       "      <td>0.457 $\\pm$ 0.006</td>\n",
       "      <td>0.497 $\\pm$ 0.005</td>\n",
       "      <td>0.009957</td>\n",
       "      <td>0.010420</td>\n",
       "      <td>0.005442</td>\n",
       "      <td>0.006771</td>\n",
       "      <td>0.005710</td>\n",
       "      <td>0.005346</td>\n",
       "      <td>GPT-3.5 (Scaffold, #ICL examples: 10)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>GPT-4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>5</td>\n",
       "      <td>Scaffold</td>\n",
       "      <td>0.456 $\\pm$ 0.003</td>\n",
       "      <td>0.357 $\\pm$ 0.004</td>\n",
       "      <td>0.540 $\\pm$ 0.005</td>\n",
       "      <td>0.355 $\\pm$ 0.007</td>\n",
       "      <td>0.455 $\\pm$ 0.005</td>\n",
       "      <td>0.505 $\\pm$ 0.005</td>\n",
       "      <td>0.003267</td>\n",
       "      <td>0.004230</td>\n",
       "      <td>0.004614</td>\n",
       "      <td>0.006798</td>\n",
       "      <td>0.004748</td>\n",
       "      <td>0.004574</td>\n",
       "      <td>GPT-4 (Scaffold, #ICL examples: 5)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>GPT-4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Random</td>\n",
       "      <td>0.260 $\\pm$ 0.007</td>\n",
       "      <td>0.140 $\\pm$ 0.007</td>\n",
       "      <td>0.393 $\\pm$ 0.004</td>\n",
       "      <td>0.180 $\\pm$ 0.006</td>\n",
       "      <td>0.309 $\\pm$ 0.004</td>\n",
       "      <td>0.320 $\\pm$ 0.007</td>\n",
       "      <td>0.007126</td>\n",
       "      <td>0.006514</td>\n",
       "      <td>0.004061</td>\n",
       "      <td>0.005691</td>\n",
       "      <td>0.004215</td>\n",
       "      <td>0.006952</td>\n",
       "      <td>GPT-4 (Random, #ICL examples: 10)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>zero-shot</td>\n",
       "      <td>Zero-shot (GPT-4)</td>\n",
       "      <td>zero-shot</td>\n",
       "      <td>zero-shot</td>\n",
       "      <td>Zero-shot (GPT-4)</td>\n",
       "      <td>0.062 $\\pm$ 0.001</td>\n",
       "      <td>0.013 $\\pm$ 0.001</td>\n",
       "      <td>0.192 $\\pm$ 0.002</td>\n",
       "      <td>0.040 $\\pm$ 0.002</td>\n",
       "      <td>0.125 $\\pm$ 0.002</td>\n",
       "      <td>0.209 $\\pm$ 0.002</td>\n",
       "      <td>0.001142</td>\n",
       "      <td>0.001294</td>\n",
       "      <td>0.002219</td>\n",
       "      <td>0.001523</td>\n",
       "      <td>0.001522</td>\n",
       "      <td>0.001517</td>\n",
       "      <td>Zero-shot (GPT-4) (Zero-shot (GPT-4), #ICL exa...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  task              model       name sample_num  \\\n",
       "0  molecule_captioning              GPT-4     SMILES         10   \n",
       "1  molecule_captioning        Davinci-003     SMILES         10   \n",
       "2  molecule_captioning            GPT-3.5     SMILES         10   \n",
       "3  molecule_captioning              GPT-4     SMILES          5   \n",
       "4  molecule_captioning              GPT-4     SMILES         10   \n",
       "5            zero-shot  Zero-shot (GPT-4)  zero-shot  zero-shot   \n",
       "\n",
       "       sample_method              bleu2              bleu4             rouge1  \\\n",
       "0           Scaffold  0.464 $\\pm$ 0.008  0.365 $\\pm$ 0.008  0.545 $\\pm$ 0.003   \n",
       "1           Scaffold  0.488 $\\pm$ 0.011  0.391 $\\pm$ 0.012  0.532 $\\pm$ 0.008   \n",
       "2           Scaffold  0.468 $\\pm$ 0.010  0.368 $\\pm$ 0.010  0.534 $\\pm$ 0.005   \n",
       "3           Scaffold  0.456 $\\pm$ 0.003  0.357 $\\pm$ 0.004  0.540 $\\pm$ 0.005   \n",
       "4             Random  0.260 $\\pm$ 0.007  0.140 $\\pm$ 0.007  0.393 $\\pm$ 0.004   \n",
       "5  Zero-shot (GPT-4)  0.062 $\\pm$ 0.001  0.013 $\\pm$ 0.001  0.192 $\\pm$ 0.002   \n",
       "\n",
       "              rouge2             rougel             meteor  bleu2_std  \\\n",
       "0  0.362 $\\pm$ 0.003  0.459 $\\pm$ 0.007  0.519 $\\pm$ 0.005   0.007529   \n",
       "1  0.359 $\\pm$ 0.010  0.465 $\\pm$ 0.008  0.478 $\\pm$ 0.011   0.011140   \n",
       "2  0.355 $\\pm$ 0.007  0.457 $\\pm$ 0.006  0.497 $\\pm$ 0.005   0.009957   \n",
       "3  0.355 $\\pm$ 0.007  0.455 $\\pm$ 0.005  0.505 $\\pm$ 0.005   0.003267   \n",
       "4  0.180 $\\pm$ 0.006  0.309 $\\pm$ 0.004  0.320 $\\pm$ 0.007   0.007126   \n",
       "5  0.040 $\\pm$ 0.002  0.125 $\\pm$ 0.002  0.209 $\\pm$ 0.002   0.001142   \n",
       "\n",
       "   bleu4_std  rouge1_std  rouge2_std  rougel_std  meteor_std  \\\n",
       "0   0.008039    0.003315    0.002863    0.007351    0.005048   \n",
       "1   0.011708    0.008313    0.009813    0.007998    0.010948   \n",
       "2   0.010420    0.005442    0.006771    0.005710    0.005346   \n",
       "3   0.004230    0.004614    0.006798    0.004748    0.004574   \n",
       "4   0.006514    0.004061    0.005691    0.004215    0.006952   \n",
       "5   0.001294    0.002219    0.001523    0.001522    0.001517   \n",
       "\n",
       "                                              Method  \n",
       "0                GPT-4 (Scaffold, #ICL examples: 10)  \n",
       "1          Davinci-003 (Scaffold, #ICL examples: 10)  \n",
       "2              GPT-3.5 (Scaffold, #ICL examples: 10)  \n",
       "3                 GPT-4 (Scaffold, #ICL examples: 5)  \n",
       "4                  GPT-4 (Random, #ICL examples: 10)  \n",
       "5  Zero-shot (GPT-4) (Zero-shot (GPT-4), #ICL exa...  "
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "performance_results_mol_captioning_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-13T17:09:37.518185Z",
     "start_time": "2023-05-13T17:09:37.500835Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "performance_results_mol_captioning_df = pd.read_csv(detail_save_folder + \"molecule_captioning_summary.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-13T17:18:54.041895Z",
     "start_time": "2023-05-13T17:18:54.009393Z"
    }
   },
   "outputs": [],
   "source": [
    "import statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-13T17:19:12.404079Z",
     "start_time": "2023-05-13T17:19:12.398822Z"
    }
   },
   "outputs": [],
   "source": [
    "mapmodel = {\n",
    "    \"gpt4\": \"GPT-4\",\n",
    "    \"davinci003\": 'Davinci-003',\n",
    "    \"gpt35\": 'GPT-3.5',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-13T17:19:16.388653Z",
     "start_time": "2023-05-13T17:19:16.385924Z"
    }
   },
   "outputs": [],
   "source": [
    "simmap = {\n",
    "    \"Scaffold_SIM\": \"Scaffold\",\n",
    "    \"Fixed_ICL\": \"Random\",\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-13T17:33:07.185571Z",
     "start_time": "2023-05-13T17:33:07.173205Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task</th>\n",
       "      <th>model</th>\n",
       "      <th>name</th>\n",
       "      <th>sample_num</th>\n",
       "      <th>sample_method</th>\n",
       "      <th>bleu2</th>\n",
       "      <th>bleu4</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougel</th>\n",
       "      <th>meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.464337</td>\n",
       "      <td>0.365360</td>\n",
       "      <td>0.545179</td>\n",
       "      <td>0.362488</td>\n",
       "      <td>0.459131</td>\n",
       "      <td>0.518980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>davinci003</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.488202</td>\n",
       "      <td>0.391264</td>\n",
       "      <td>0.532238</td>\n",
       "      <td>0.358805</td>\n",
       "      <td>0.464974</td>\n",
       "      <td>0.478031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt35</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.467592</td>\n",
       "      <td>0.368161</td>\n",
       "      <td>0.533540</td>\n",
       "      <td>0.354869</td>\n",
       "      <td>0.457351</td>\n",
       "      <td>0.496697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>5</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.456031</td>\n",
       "      <td>0.357472</td>\n",
       "      <td>0.540109</td>\n",
       "      <td>0.354757</td>\n",
       "      <td>0.454905</td>\n",
       "      <td>0.505097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Fixed_ICL</td>\n",
       "      <td>0.259944</td>\n",
       "      <td>0.140320</td>\n",
       "      <td>0.392945</td>\n",
       "      <td>0.180456</td>\n",
       "      <td>0.308746</td>\n",
       "      <td>0.320015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>zero-shot</td>\n",
       "      <td>zero-shot</td>\n",
       "      <td>zero-shot</td>\n",
       "      <td>zero-shot</td>\n",
       "      <td>zero-shot</td>\n",
       "      <td>0.062159</td>\n",
       "      <td>0.012904</td>\n",
       "      <td>0.192246</td>\n",
       "      <td>0.040152</td>\n",
       "      <td>0.124792</td>\n",
       "      <td>0.209018</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  task       model       name sample_num sample_method  \\\n",
       "0  molecule_captioning        gpt4     SMILES         10  Scaffold_SIM   \n",
       "1  molecule_captioning  davinci003     SMILES         10  Scaffold_SIM   \n",
       "2  molecule_captioning       gpt35     SMILES         10  Scaffold_SIM   \n",
       "3  molecule_captioning        gpt4     SMILES          5  Scaffold_SIM   \n",
       "4  molecule_captioning        gpt4     SMILES         10     Fixed_ICL   \n",
       "5            zero-shot   zero-shot  zero-shot  zero-shot     zero-shot   \n",
       "\n",
       "      bleu2     bleu4    rouge1    rouge2    rougel    meteor  \n",
       "0  0.464337  0.365360  0.545179  0.362488  0.459131  0.518980  \n",
       "1  0.488202  0.391264  0.532238  0.358805  0.464974  0.478031  \n",
       "2  0.467592  0.368161  0.533540  0.354869  0.457351  0.496697  \n",
       "3  0.456031  0.357472  0.540109  0.354757  0.454905  0.505097  \n",
       "4  0.259944  0.140320  0.392945  0.180456  0.308746  0.320015  \n",
       "5  0.062159  0.012904  0.192246  0.040152  0.124792  0.209018  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "performance_results_mol_captioning_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-13T17:18:41.994313Z",
     "start_time": "2023-05-13T17:18:41.975021Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task</th>\n",
       "      <th>model</th>\n",
       "      <th>name</th>\n",
       "      <th>sample_num</th>\n",
       "      <th>sample_method</th>\n",
       "      <th>bleu2</th>\n",
       "      <th>bleu4</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougel</th>\n",
       "      <th>meteor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.464337</td>\n",
       "      <td>0.365360</td>\n",
       "      <td>0.545179</td>\n",
       "      <td>0.362488</td>\n",
       "      <td>0.459131</td>\n",
       "      <td>0.518980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>davinci003</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.488202</td>\n",
       "      <td>0.391264</td>\n",
       "      <td>0.532238</td>\n",
       "      <td>0.358805</td>\n",
       "      <td>0.464974</td>\n",
       "      <td>0.478031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt35</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.467592</td>\n",
       "      <td>0.368161</td>\n",
       "      <td>0.533540</td>\n",
       "      <td>0.354869</td>\n",
       "      <td>0.457351</td>\n",
       "      <td>0.496697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>5</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.456031</td>\n",
       "      <td>0.357472</td>\n",
       "      <td>0.540109</td>\n",
       "      <td>0.354757</td>\n",
       "      <td>0.454905</td>\n",
       "      <td>0.505097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Fixed_ICL</td>\n",
       "      <td>0.259944</td>\n",
       "      <td>0.140320</td>\n",
       "      <td>0.392945</td>\n",
       "      <td>0.180456</td>\n",
       "      <td>0.308746</td>\n",
       "      <td>0.320015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>zero-shot</td>\n",
       "      <td>zero-shot</td>\n",
       "      <td>zero-shot</td>\n",
       "      <td>zero-shot</td>\n",
       "      <td>zero-shot</td>\n",
       "      <td>0.062159</td>\n",
       "      <td>0.012904</td>\n",
       "      <td>0.192246</td>\n",
       "      <td>0.040152</td>\n",
       "      <td>0.124792</td>\n",
       "      <td>0.209018</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  task       model       name sample_num sample_method  \\\n",
       "0  molecule_captioning        gpt4     SMILES         10  Scaffold_SIM   \n",
       "1  molecule_captioning  davinci003     SMILES         10  Scaffold_SIM   \n",
       "2  molecule_captioning       gpt35     SMILES         10  Scaffold_SIM   \n",
       "3  molecule_captioning        gpt4     SMILES          5  Scaffold_SIM   \n",
       "4  molecule_captioning        gpt4     SMILES         10     Fixed_ICL   \n",
       "5            zero-shot   zero-shot  zero-shot  zero-shot     zero-shot   \n",
       "\n",
       "      bleu2     bleu4    rouge1    rouge2    rougel    meteor  \n",
       "0  0.464337  0.365360  0.545179  0.362488  0.459131  0.518980  \n",
       "1  0.488202  0.391264  0.532238  0.358805  0.464974  0.478031  \n",
       "2  0.467592  0.368161  0.533540  0.354869  0.457351  0.496697  \n",
       "3  0.456031  0.357472  0.540109  0.354757  0.454905  0.505097  \n",
       "4  0.259944  0.140320  0.392945  0.180456  0.308746  0.320015  \n",
       "5  0.062159  0.012904  0.192246  0.040152  0.124792  0.209018  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "performance_results_mol_captioning_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-01T15:26:13.122492Z",
     "start_time": "2023-05-01T15:26:13.043434Z"
    }
   },
   "outputs": [],
   "source": [
    "def top_n_scaffold_similar_molecules_sim(target_smiles, molecule_scaffold_list, molecule_smiles_list, n=5):\n",
    "    target_mol = Chem.MolFromSmiles(target_smiles)\n",
    "    target_scaffold = MurckoScaffold.GetScaffoldForMol(target_mol)\n",
    "    target_fp = rdMolDescriptors.GetMorganFingerprint(target_scaffold, 2)\n",
    "\n",
    "    similarities = []\n",
    "\n",
    "    for idx, scaffold_fp in enumerate(molecule_scaffold_list):\n",
    "        try:\n",
    "            tanimoto_similarity = DataStructs.TanimotoSimilarity(target_fp, scaffold_fp)\n",
    "            similarities.append((idx, tanimoto_similarity))\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            continue\n",
    "\n",
    "    similarities.sort(key=lambda x: x[1], reverse=True)\n",
    "    top_5_similar_molecules = similarities[:n]\n",
    "\n",
    "    return [molecule_smiles_list[i[0]] for i in top_5_similar_molecules], [i[1] for i in top_5_similar_molecules]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-01T15:26:23.011741Z",
     "start_time": "2023-05-01T15:26:23.004818Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'CC(C)C[C@H]1C(=O)N2CCC[C@H]2[C@]2(O)O[C@](NC(=O)[C@@H]3C=C4c5cccc6[nH]c(Br)c(c56)C[C@H]4N(C)C3)(C(C)C)C(=O)N12'"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.iloc[0]['SMILES']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-01T15:26:37.061347Z",
     "start_time": "2023-05-01T15:26:35.842410Z"
    }
   },
   "outputs": [],
   "source": [
    "a, b = top_n_scaffold_similar_molecules_sim(test.iloc[0]['SMILES'], list(train['scaffold_fp']), list(train['SMILES']), n=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-01T15:27:30.578737Z",
     "start_time": "2023-05-01T15:27:30.568418Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"The molecule is ergotaman bearing a hydroxy group at the 12' position, isopropyl groups at the 2' and 5'alpha positions, and oxo groups at positions 3', 6', and 18. It is a natural ergot alkaloid. It derives from a hydride of an ergotaman.\",\n",
       " 'The molecule is ergocornine in which a single bond replaces the double bond between positions 9 and 10. It derives from an ergocornine. It derives from a hydride of an ergotaman.',\n",
       " 'The molecule is the methanesulfonic acid salt of 9,10-dihydroergocornine. It is a component of ergoloid mesylate (codergocrine mesilate), a mixture of ergot alkaloid derivatives that is used as a vasodilator and has shown mild benefits in the treatment of vascular dementia. It contains a dihydroergocornine. It derives from an ergocornine.',\n",
       " 'The molecule is alpha-Ergocryptine in which a single bond replaces the double bond between positions 9 and 10. It derives from an alpha-ergocryptine. It derives from a hydride of an ergotaman.',\n",
       " 'The molecule is ergocristine in which a single bond replaces the double bond between positions 9 and 10. It is used as the mesylate salt for the symptomatic treatment of mental deterioration associated with cerebrovascular insufficiency and in peripheral vascular disease. It has a role as an adrenergic antagonist and a vasodilator agent. It derives from an ergocristine. It derives from a hydride of an ergotaman.',\n",
       " 'The molecule is the methanesulfonic acid salt of dihydroergocristine. It has been used as the for the symptomatic treatment of mental deterioration associated with cerebrovascular insufficiency and in peripheral vascular disease. It is also a component of ergoloid mesylate (codergocrine mesilate), a mixture of ergot alkaloid derivatives that is used as a vasodilator and has shown mild benefits in the treatment of vascular dementia. It has a role as a vasodilator agent and an alpha-adrenergic antagonist. It contains a dihydroergocristine.',\n",
       " 'The molecule is beta-Ergocryptine in which a single bond replaces the double bond between positions 9 and 10. It derives from a beta-ergocryptine. It derives from a hydride of an ergotaman.',\n",
       " \"The molecule is ergotaman bearing benzyl, hydroxy, and isopropyl groups at the 5', 12' and 2' positions, respectively, and oxo groups at positions 3', 6', and 18. It is a natural ergot alkaloid. It derives from a hydride of an ergotaman.\",\n",
       " 'The molecule is an ergot alkaloid isolated from the fungus Epichloe typhina. It has a role as a fungal metabolite. It derives from a hydride of an ergotaman.',\n",
       " 'The molecule is alpha-Ergocryptine in which the isobutyl (2-methylpropyl) substituent is replaced by sec-butyl (1-methylpropyl). It is a natural ergot alkaloid. Note that ergocryptine discussed in the literature prior to 1967, when beta-ergocryptine was separated from alpha-ergocryptine, is now referred to as alpha-ergocryptine. It derives from a hydride of an ergotaman.']"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(train[train['SMILES'].isin(a)]['description'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-01T15:29:24.476892Z",
     "start_time": "2023-05-01T15:29:24.471130Z"
    }
   },
   "outputs": [],
   "source": [
    "def top_n_similar_strings_sim(query, candidates, n=5):\n",
    "    # Calculate the Levenshtein distance between the query and each candidate\n",
    "    distances = [(c, similarity_ratio(query, c)) for c in candidates]\n",
    "    \n",
    "    # Sort the candidates by their Levenshtein distance to the query\n",
    "    sorted_distances = sorted(distances, key=lambda x: x[1], reverse=True)\n",
    "    \n",
    "    # Get the top n candidates with the smallest Levenshtein distance\n",
    "    top_candidates = [d[0] for d in sorted_distances[:n]]\n",
    "    \n",
    "    # Return the top n candidates\n",
    "    return top_candidates, [d[1] for d in sorted_distances[:n]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-01T15:30:10.467970Z",
     "start_time": "2023-05-01T15:29:58.417443Z"
    }
   },
   "outputs": [],
   "source": [
    "a, b = top_n_similar_strings_sim(test.iloc[0]['description'], list(train['description']), n=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Zero-shot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-01T16:55:18.868376Z",
     "start_time": "2023-05-01T16:55:18.862015Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_zero_shot_prompt(input_text, task):\n",
    "    if task == 'smiles2iupac':\n",
    "        \n",
    "        prompt = \"You are an expert chemist. Given the molecular SMILES: {}, predict the molecular IUPAC name using your experienced chemical molecular SMILES and IUPAC name knowledge. No explanations and other information. \\\n",
    "Only return the molecular IUPAC name.\".format(input_text)\n",
    "        \n",
    "    elif task == 'smiles2formula':\n",
    "        \n",
    "        prompt = \"You are an expert chemist. Given the molecular SMILES: {}, predict the chemical molecular formula using your experienced chemical molecular SMILES and formula knowledge. No explanations and other information. \\\n",
    "Only return the molecular formula.\".format(input_text)\n",
    "        \n",
    "    elif task == 'iupac2smiles':\n",
    "        \n",
    "        prompt = \"You are an expert chemist. Given the molecular IUPAC name: {}, predict the molecular SMILES using your experienced chemical molecular IUPAC name and SMILES knowledge. No explanations and other information. \\\n",
    "Only return the molecular SMILES.\".format(input_text)\n",
    "        \n",
    "    elif task == 'formula2smiles':\n",
    "        \n",
    "        prompt = \"You are an expert chemist. Given the molecular formula: {}, predict the molecular SMILES using your experienced chemical molecular formula and SMILES knowledge. No explanations and other information. \\\n",
    "Only return the molecular SMILES.\".format(input_text)\n",
    "    \n",
    "    elif task == 'molecule_design':\n",
    "        prompt = \"You are an expert chemist. Given the molecular requirements description: '{}', design the molecular SMILES using your experienced chemical molecule design knowledge. No explanations and other information. \\\n",
    "Only return the designed molecular SMILES. The SMILES must be valid and chemically reasonable.\".format(input_text)\n",
    "        \n",
    "    elif task == 'molecule_captioning':\n",
    "        prompt = \"You are an expert chemist. Given the molecular SMILES: {}, provide the detailed molecule description using your experienced chemical Molecular knowledge.\".format(input_text)\n",
    "    \n",
    "    return prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-01T16:55:19.399802Z",
     "start_time": "2023-05-01T16:55:19.396404Z"
    }
   },
   "outputs": [],
   "source": [
    "model = 'gpt4'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-01T18:42:31.945268Z",
     "start_time": "2023-05-01T17:11:27.417911Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Molecule_Design/ChEBI-20_data//results_test/molecule_design_gpt4_zero_shot.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [26:42, 16.02s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Molecule_Design/ChEBI-20_data//results_test/molecule_captioning_gpt4_zero_shot.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [1:04:22, 38.62s/it]\n"
     ]
    }
   ],
   "source": [
    "# format the running grid search \n",
    "# evaluate and save\n",
    "\n",
    "detail_save_folder = folder + '/results_test/'\n",
    "TASK_Types = ['molecule_design', 'molecule_captioning']\n",
    "# scaffold search names \n",
    "scaffold_col_smiles_name = 'SMILES'\n",
    "\n",
    "for task in TASK_Types:\n",
    "        \n",
    "    details_results = []\n",
    "    performance_results = []\n",
    "\n",
    "    detail_predict_file = detail_save_folder + '{}_{}_zero_shot.csv'.format(task, model)\n",
    "    log_file = detail_save_folder + '{}_{}_zero_shot.log'.format(task, model)\n",
    "    performance_file = detail_save_folder + '{}_{}_zero_shot_performance.csv'.format(task, model)\n",
    "    print(detail_predict_file)\n",
    "\n",
    "\n",
    "    if os.path.exists(performance_file):\n",
    "        print(\"{} exits, continue to next params\".format(performance_file))\n",
    "        continue\n",
    "\n",
    "\n",
    "    # append new date\n",
    "    # Get the current date and time\n",
    "    now = datetime.datetime.now()\n",
    "    # Convert the date and time to a string\n",
    "    date_time_str = now.strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "    with open(log_file, \"a\") as file:\n",
    "        file.write(\"=\" * 30 + date_time_str + \"=\" * 30 + \"\\n\")\n",
    "\n",
    "    # init var\n",
    "    predicted_products = []\n",
    "\n",
    "    # restore previous data and skip \n",
    "    previous_index = []\n",
    "    if os.path.exists(detail_predict_file):\n",
    "        previous = pd.read_csv(detail_predict_file)\n",
    "        previous_index = list(previous['index'])\n",
    "\n",
    "        # previous data => details_results \n",
    "        details_results = previous.values.tolist()\n",
    "        predicted_products = previous[['pred_1', 'pred_2', 'pred_3', 'pred_4', 'pred_5']].values.tolist()\n",
    "\n",
    "    input_col, output_col = get_input_output_columns_by_task(task)\n",
    "\n",
    "    # query\n",
    "    for idx, row in tqdm(test.iterrows()):\n",
    "        reactant = row[input_col]\n",
    "        product = row[output_col]\n",
    "        index = row['index']\n",
    "\n",
    "        if index in previous_index:\n",
    "            continue\n",
    "\n",
    "        prompt = create_zero_shot_prompt(reactant, task)\n",
    "        with open(log_file, \"a\") as file:\n",
    "            file.write(prompt + \"\\n\")\n",
    "            file.write(\"=\" * 50 + \"\\n\")\n",
    "\n",
    "        # different model \n",
    "        if model == 'davinci003':\n",
    "            predicted_product = generate_response_by_davinci(prompt)\n",
    "        elif model == 'gpt35':\n",
    "            predicted_product = generate_response_by_gpt35(prompt)\n",
    "        elif model == 'gpt4':\n",
    "            predicted_product = generate_response_by_gpt4(prompt)\n",
    "\n",
    "        predicted_products.append(predicted_product)\n",
    "        details_results.append([index, reactant] + [product] + predicted_product)\n",
    "\n",
    "        details_df = pd.DataFrame(details_results, columns=['index', input_col, output_col, 'pred_1', 'pred_2', 'pred_3', 'pred_4', 'pred_5'])\n",
    "        details_df.to_csv(detail_predict_file, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T22:54:07.812501Z",
     "start_time": "2023-05-14T22:54:07.779604Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Molecule_Design/ChEBI-20_data/'"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T22:54:10.582211Z",
     "start_time": "2023-05-14T22:54:10.578550Z"
    }
   },
   "outputs": [],
   "source": [
    "detail_save_folder = folder + '/results_test/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T23:05:18.123025Z",
     "start_time": "2023-05-14T23:05:17.927292Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "details_df = pd.read_csv(detail_save_folder + 'molecule_captioning_gpt4_zero_shot.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T23:05:48.857137Z",
     "start_time": "2023-05-14T23:05:38.558256Z"
    }
   },
   "outputs": [],
   "source": [
    "metrics, stds = evaluate_one_molecule_captioning(details_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T23:05:48.863924Z",
     "start_time": "2023-05-14T23:05:48.859630Z"
    }
   },
   "outputs": [],
   "source": [
    "performance_results_mol_captioning.append(['zero-shot'] * 5 + metrics + stds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T23:06:10.486005Z",
     "start_time": "2023-05-14T23:06:10.459256Z"
    }
   },
   "outputs": [],
   "source": [
    "performance_results_mol_captioning_df = pd.DataFrame(performance_results_mol_captioning, columns=caption_cols + metrics_cols + std_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T23:11:23.052502Z",
     "start_time": "2023-05-14T23:11:22.988820Z"
    }
   },
   "outputs": [],
   "source": [
    "performance_results_mol_captioning_df.to_csv(detail_save_folder + \"molecule_captioning_summary.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T23:11:23.087828Z",
     "start_time": "2023-05-14T23:11:23.059218Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task</th>\n",
       "      <th>model</th>\n",
       "      <th>name</th>\n",
       "      <th>sample_num</th>\n",
       "      <th>sample_method</th>\n",
       "      <th>bleu2</th>\n",
       "      <th>bleu4</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougel</th>\n",
       "      <th>meteor</th>\n",
       "      <th>bleu2_std</th>\n",
       "      <th>bleu4_std</th>\n",
       "      <th>rouge1_std</th>\n",
       "      <th>rouge2_std</th>\n",
       "      <th>rougel_std</th>\n",
       "      <th>meteor_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.464337</td>\n",
       "      <td>0.365360</td>\n",
       "      <td>0.545179</td>\n",
       "      <td>0.362488</td>\n",
       "      <td>0.459131</td>\n",
       "      <td>0.518980</td>\n",
       "      <td>0.007529</td>\n",
       "      <td>0.008039</td>\n",
       "      <td>0.003315</td>\n",
       "      <td>0.002863</td>\n",
       "      <td>0.007351</td>\n",
       "      <td>0.005048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>davinci003</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.488202</td>\n",
       "      <td>0.391264</td>\n",
       "      <td>0.532238</td>\n",
       "      <td>0.358805</td>\n",
       "      <td>0.464974</td>\n",
       "      <td>0.478031</td>\n",
       "      <td>0.011140</td>\n",
       "      <td>0.011708</td>\n",
       "      <td>0.008313</td>\n",
       "      <td>0.009813</td>\n",
       "      <td>0.007998</td>\n",
       "      <td>0.010948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt35</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.467592</td>\n",
       "      <td>0.368161</td>\n",
       "      <td>0.533540</td>\n",
       "      <td>0.354869</td>\n",
       "      <td>0.457351</td>\n",
       "      <td>0.496697</td>\n",
       "      <td>0.009957</td>\n",
       "      <td>0.010420</td>\n",
       "      <td>0.005442</td>\n",
       "      <td>0.006771</td>\n",
       "      <td>0.005710</td>\n",
       "      <td>0.005346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>5</td>\n",
       "      <td>Scaffold_SIM</td>\n",
       "      <td>0.456031</td>\n",
       "      <td>0.357472</td>\n",
       "      <td>0.540109</td>\n",
       "      <td>0.354757</td>\n",
       "      <td>0.454905</td>\n",
       "      <td>0.505097</td>\n",
       "      <td>0.003267</td>\n",
       "      <td>0.004230</td>\n",
       "      <td>0.004614</td>\n",
       "      <td>0.006798</td>\n",
       "      <td>0.004748</td>\n",
       "      <td>0.004574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>molecule_captioning</td>\n",
       "      <td>gpt4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Fixed_ICL</td>\n",
       "      <td>0.259944</td>\n",
       "      <td>0.140320</td>\n",
       "      <td>0.392945</td>\n",
       "      <td>0.180456</td>\n",
       "      <td>0.308746</td>\n",
       "      <td>0.320015</td>\n",
       "      <td>0.007126</td>\n",
       "      <td>0.006514</td>\n",
       "      <td>0.004061</td>\n",
       "      <td>0.005691</td>\n",
       "      <td>0.004215</td>\n",
       "      <td>0.006952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>zero-shot</td>\n",
       "      <td>zero-shot</td>\n",
       "      <td>zero-shot</td>\n",
       "      <td>zero-shot</td>\n",
       "      <td>zero-shot</td>\n",
       "      <td>0.062159</td>\n",
       "      <td>0.012904</td>\n",
       "      <td>0.192246</td>\n",
       "      <td>0.040152</td>\n",
       "      <td>0.124792</td>\n",
       "      <td>0.209018</td>\n",
       "      <td>0.001142</td>\n",
       "      <td>0.001294</td>\n",
       "      <td>0.002219</td>\n",
       "      <td>0.001523</td>\n",
       "      <td>0.001522</td>\n",
       "      <td>0.001517</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  task       model       name sample_num sample_method  \\\n",
       "0  molecule_captioning        gpt4     SMILES         10  Scaffold_SIM   \n",
       "1  molecule_captioning  davinci003     SMILES         10  Scaffold_SIM   \n",
       "2  molecule_captioning       gpt35     SMILES         10  Scaffold_SIM   \n",
       "3  molecule_captioning        gpt4     SMILES          5  Scaffold_SIM   \n",
       "4  molecule_captioning        gpt4     SMILES         10     Fixed_ICL   \n",
       "5            zero-shot   zero-shot  zero-shot  zero-shot     zero-shot   \n",
       "\n",
       "      bleu2     bleu4    rouge1    rouge2    rougel    meteor  bleu2_std  \\\n",
       "0  0.464337  0.365360  0.545179  0.362488  0.459131  0.518980   0.007529   \n",
       "1  0.488202  0.391264  0.532238  0.358805  0.464974  0.478031   0.011140   \n",
       "2  0.467592  0.368161  0.533540  0.354869  0.457351  0.496697   0.009957   \n",
       "3  0.456031  0.357472  0.540109  0.354757  0.454905  0.505097   0.003267   \n",
       "4  0.259944  0.140320  0.392945  0.180456  0.308746  0.320015   0.007126   \n",
       "5  0.062159  0.012904  0.192246  0.040152  0.124792  0.209018   0.001142   \n",
       "\n",
       "   bleu4_std  rouge1_std  rouge2_std  rougel_std  meteor_std  \n",
       "0   0.008039    0.003315    0.002863    0.007351    0.005048  \n",
       "1   0.011708    0.008313    0.009813    0.007998    0.010948  \n",
       "2   0.010420    0.005442    0.006771    0.005710    0.005346  \n",
       "3   0.004230    0.004614    0.006798    0.004748    0.004574  \n",
       "4   0.006514    0.004061    0.005691    0.004215    0.006952  \n",
       "5   0.001294    0.002219    0.001523    0.001522    0.001517  "
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "performance_results_mol_captioning_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T23:19:16.202796Z",
     "start_time": "2023-05-14T23:19:16.199188Z"
    }
   },
   "outputs": [],
   "source": [
    "mapmodel = {\n",
    "    \"gpt4\": \"GPT-4\",\n",
    "    \"davinci003\": 'Davinci-003',\n",
    "    \"gpt35\": 'GPT-3.5',\n",
    "    'zero-shot': 'Zero-shot (GPT-4)'\n",
    "}\n",
    "\n",
    "simmap = {\n",
    "    \"Scaffold_SIM\": \"Scaffold\",\n",
    "    \"Fixed_ICL\": \"Random\",\n",
    "    'zero-shot': 'Zero-shot (GPT-4)'\n",
    "}\n",
    "\n",
    "performance_results_mol_captioning_df['model'] = performance_results_mol_captioning_df['model'].apply(lambda x: mapmodel[x])\n",
    "performance_results_mol_captioning_df['sample_method'] = performance_results_mol_captioning_df['sample_method'].apply(lambda x: simmap[x])\n",
    "performance_results_mol_captioning_df['Method'] = performance_results_mol_captioning_df.apply(lambda row: \"{} ({}, #ICL examples: {})\".format(row['model'], row['sample_method'], row['sample_num']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T23:19:37.562831Z",
     "start_time": "2023-05-14T23:19:37.552720Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['bleu2', 'bleu4', 'rouge1', 'rouge2', 'rougel', 'meteor']"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-14T23:33:51.552212Z",
     "start_time": "2023-05-14T23:33:51.459940Z"
    }
   },
   "outputs": [],
   "source": [
    "for metric in metrics_cols:\n",
    "    metric_std = metric + \"_\" + \"std\"\n",
    "    performance_results_mol_captioning_df[metric]=performance_results_mol_captioning_df.apply(lambda row: \"{:.3f} $\\pm$ {:.3f}\".format(row[metric], row[metric_std]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-15T00:02:49.154745Z",
     "start_time": "2023-05-15T00:02:49.129785Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['bleu2', 'bleu4', 'rouge1', 'rouge2', 'rougel', 'meteor']"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-15T00:03:21.263371Z",
     "start_time": "2023-05-15T00:03:20.789436Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "performance_results_mol_captioning_df[['Method'] + metrics_cols].to_clipboardboard(index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-15T14:45:37.471377Z",
     "start_time": "2023-05-15T14:45:37.137679Z"
    }
   },
   "outputs": [],
   "source": [
    "design_performance = pd.read_csv(\"Molecule_Design/ChEBI-20_data/results_test/moldesign.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-15T14:49:46.917234Z",
     "start_time": "2023-05-15T14:49:46.910302Z"
    }
   },
   "outputs": [],
   "source": [
    "metrics_cols = ['bleu', \n",
    "     'exact_match', \n",
    "    'levenshtein', \n",
    "    'validity', \n",
    "    'maccs_sims', \n",
    "    'rdk_sims', \n",
    "    'morgan_sims', \n",
    "    'fcd']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-15T14:49:59.411606Z",
     "start_time": "2023-05-15T14:49:59.312147Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "design_performance[['Method'] + metrics_cols].to_clipboard(index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T14:00:02.737330Z",
     "start_time": "2023-05-16T14:00:00.195591Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task</th>\n",
       "      <th>model</th>\n",
       "      <th>name</th>\n",
       "      <th>sample_num</th>\n",
       "      <th>sample_method</th>\n",
       "      <th>bleu</th>\n",
       "      <th>exact_match</th>\n",
       "      <th>levenshtein</th>\n",
       "      <th>validity</th>\n",
       "      <th>maccs_sims</th>\n",
       "      <th>...</th>\n",
       "      <th>fcd</th>\n",
       "      <th>bleu_std</th>\n",
       "      <th>exact_match_std</th>\n",
       "      <th>levenshtein_std</th>\n",
       "      <th>validity_std</th>\n",
       "      <th>maccs_sims_std</th>\n",
       "      <th>rdk_sims_std</th>\n",
       "      <th>morgan_sims_std</th>\n",
       "      <th>fcd_std</th>\n",
       "      <th>Method</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>molecule_design</td>\n",
       "      <td>GPT-4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold</td>\n",
       "      <td>0.816 $\\pm$ 0.004</td>\n",
       "      <td>0.174 $\\pm$ 0.029</td>\n",
       "      <td>21.160 $\\pm$ 0.600</td>\n",
       "      <td>0.888 $\\pm$ 0.023</td>\n",
       "      <td>0.867 $\\pm$ 0.005</td>\n",
       "      <td>...</td>\n",
       "      <td>6.224 $\\pm$ 0.449</td>\n",
       "      <td>0.003625</td>\n",
       "      <td>0.028810</td>\n",
       "      <td>0.599792</td>\n",
       "      <td>0.022804</td>\n",
       "      <td>0.005329</td>\n",
       "      <td>0.009698</td>\n",
       "      <td>0.013001</td>\n",
       "      <td>0.449094</td>\n",
       "      <td>GPT-4 (Scaffold, #ICL examples: 10)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>molecule_design</td>\n",
       "      <td>Davinci-003</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold</td>\n",
       "      <td>0.741 $\\pm$ 0.011</td>\n",
       "      <td>0.100 $\\pm$ 0.010</td>\n",
       "      <td>25.648 $\\pm$ 2.186</td>\n",
       "      <td>0.936 $\\pm$ 0.009</td>\n",
       "      <td>0.783 $\\pm$ 0.014</td>\n",
       "      <td>...</td>\n",
       "      <td>8.335 $\\pm$ 0.310</td>\n",
       "      <td>0.011309</td>\n",
       "      <td>0.010000</td>\n",
       "      <td>2.186211</td>\n",
       "      <td>0.008944</td>\n",
       "      <td>0.014157</td>\n",
       "      <td>0.004082</td>\n",
       "      <td>0.009790</td>\n",
       "      <td>0.310120</td>\n",
       "      <td>Davinci-003 (Scaffold, #ICL examples: 10)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>molecule_design</td>\n",
       "      <td>GPT-3.5</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Scaffold</td>\n",
       "      <td>0.479 $\\pm$ 0.156</td>\n",
       "      <td>0.094 $\\pm$ 0.011</td>\n",
       "      <td>82.008 $\\pm$ 40.354</td>\n",
       "      <td>0.854 $\\pm$ 0.059</td>\n",
       "      <td>0.833 $\\pm$ 0.006</td>\n",
       "      <td>...</td>\n",
       "      <td>8.341 $\\pm$ 0.607</td>\n",
       "      <td>0.155773</td>\n",
       "      <td>0.011402</td>\n",
       "      <td>40.354045</td>\n",
       "      <td>0.058992</td>\n",
       "      <td>0.006236</td>\n",
       "      <td>0.015787</td>\n",
       "      <td>0.013150</td>\n",
       "      <td>0.606648</td>\n",
       "      <td>GPT-3.5 (Scaffold, #ICL examples: 10)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>molecule_design</td>\n",
       "      <td>GPT-4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>5</td>\n",
       "      <td>Scaffold</td>\n",
       "      <td>0.815 $\\pm$ 0.011</td>\n",
       "      <td>0.164 $\\pm$ 0.018</td>\n",
       "      <td>21.862 $\\pm$ 1.768</td>\n",
       "      <td>0.874 $\\pm$ 0.030</td>\n",
       "      <td>0.865 $\\pm$ 0.015</td>\n",
       "      <td>...</td>\n",
       "      <td>5.843 $\\pm$ 0.515</td>\n",
       "      <td>0.010730</td>\n",
       "      <td>0.018166</td>\n",
       "      <td>1.768239</td>\n",
       "      <td>0.030496</td>\n",
       "      <td>0.014595</td>\n",
       "      <td>0.022951</td>\n",
       "      <td>0.028266</td>\n",
       "      <td>0.514937</td>\n",
       "      <td>GPT-4 (Scaffold, #ICL examples: 5)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>molecule_design</td>\n",
       "      <td>GPT-4</td>\n",
       "      <td>SMILES</td>\n",
       "      <td>10</td>\n",
       "      <td>Random</td>\n",
       "      <td>0.602 $\\pm$ 0.016</td>\n",
       "      <td>0.060 $\\pm$ 0.007</td>\n",
       "      <td>42.390 $\\pm$ 1.008</td>\n",
       "      <td>0.770 $\\pm$ 0.030</td>\n",
       "      <td>0.762 $\\pm$ 0.013</td>\n",
       "      <td>...</td>\n",
       "      <td>10.594 $\\pm$ 0.414</td>\n",
       "      <td>0.016402</td>\n",
       "      <td>0.007071</td>\n",
       "      <td>1.007795</td>\n",
       "      <td>0.030000</td>\n",
       "      <td>0.013061</td>\n",
       "      <td>0.017267</td>\n",
       "      <td>0.014985</td>\n",
       "      <td>0.414004</td>\n",
       "      <td>GPT-4 (Random, #ICL examples: 10)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>zero-shot</td>\n",
       "      <td>Zero-shot (GPT-4)</td>\n",
       "      <td>zero-shot</td>\n",
       "      <td>zero-shot</td>\n",
       "      <td>Zero-shot (GPT-4)</td>\n",
       "      <td>0.490 $\\pm$ 0.017</td>\n",
       "      <td>0.046 $\\pm$ 0.009</td>\n",
       "      <td>47.418 $\\pm$ 1.668</td>\n",
       "      <td>0.758 $\\pm$ 0.015</td>\n",
       "      <td>0.733 $\\pm$ 0.020</td>\n",
       "      <td>...</td>\n",
       "      <td>11.913 $\\pm$ 0.972</td>\n",
       "      <td>0.016608</td>\n",
       "      <td>0.008944</td>\n",
       "      <td>1.668104</td>\n",
       "      <td>0.014832</td>\n",
       "      <td>0.019985</td>\n",
       "      <td>0.020665</td>\n",
       "      <td>0.014155</td>\n",
       "      <td>0.971867</td>\n",
       "      <td>Zero-shot (GPT-4) (Zero-shot (GPT-4), #ICL exa...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              task              model       name sample_num  \\\n",
       "0  molecule_design              GPT-4     SMILES         10   \n",
       "1  molecule_design        Davinci-003     SMILES         10   \n",
       "2  molecule_design            GPT-3.5     SMILES         10   \n",
       "3  molecule_design              GPT-4     SMILES          5   \n",
       "4  molecule_design              GPT-4     SMILES         10   \n",
       "5        zero-shot  Zero-shot (GPT-4)  zero-shot  zero-shot   \n",
       "\n",
       "       sample_method               bleu        exact_match  \\\n",
       "0           Scaffold  0.816 $\\pm$ 0.004  0.174 $\\pm$ 0.029   \n",
       "1           Scaffold  0.741 $\\pm$ 0.011  0.100 $\\pm$ 0.010   \n",
       "2           Scaffold  0.479 $\\pm$ 0.156  0.094 $\\pm$ 0.011   \n",
       "3           Scaffold  0.815 $\\pm$ 0.011  0.164 $\\pm$ 0.018   \n",
       "4             Random  0.602 $\\pm$ 0.016  0.060 $\\pm$ 0.007   \n",
       "5  Zero-shot (GPT-4)  0.490 $\\pm$ 0.017  0.046 $\\pm$ 0.009   \n",
       "\n",
       "           levenshtein           validity         maccs_sims  ...  \\\n",
       "0   21.160 $\\pm$ 0.600  0.888 $\\pm$ 0.023  0.867 $\\pm$ 0.005  ...   \n",
       "1   25.648 $\\pm$ 2.186  0.936 $\\pm$ 0.009  0.783 $\\pm$ 0.014  ...   \n",
       "2  82.008 $\\pm$ 40.354  0.854 $\\pm$ 0.059  0.833 $\\pm$ 0.006  ...   \n",
       "3   21.862 $\\pm$ 1.768  0.874 $\\pm$ 0.030  0.865 $\\pm$ 0.015  ...   \n",
       "4   42.390 $\\pm$ 1.008  0.770 $\\pm$ 0.030  0.762 $\\pm$ 0.013  ...   \n",
       "5   47.418 $\\pm$ 1.668  0.758 $\\pm$ 0.015  0.733 $\\pm$ 0.020  ...   \n",
       "\n",
       "                  fcd  bleu_std exact_match_std  levenshtein_std  \\\n",
       "0   6.224 $\\pm$ 0.449  0.003625        0.028810         0.599792   \n",
       "1   8.335 $\\pm$ 0.310  0.011309        0.010000         2.186211   \n",
       "2   8.341 $\\pm$ 0.607  0.155773        0.011402        40.354045   \n",
       "3   5.843 $\\pm$ 0.515  0.010730        0.018166         1.768239   \n",
       "4  10.594 $\\pm$ 0.414  0.016402        0.007071         1.007795   \n",
       "5  11.913 $\\pm$ 0.972  0.016608        0.008944         1.668104   \n",
       "\n",
       "   validity_std  maccs_sims_std  rdk_sims_std  morgan_sims_std   fcd_std  \\\n",
       "0      0.022804        0.005329      0.009698         0.013001  0.449094   \n",
       "1      0.008944        0.014157      0.004082         0.009790  0.310120   \n",
       "2      0.058992        0.006236      0.015787         0.013150  0.606648   \n",
       "3      0.030496        0.014595      0.022951         0.028266  0.514937   \n",
       "4      0.030000        0.013061      0.017267         0.014985  0.414004   \n",
       "5      0.014832        0.019985      0.020665         0.014155  0.971867   \n",
       "\n",
       "                                              Method  \n",
       "0                GPT-4 (Scaffold, #ICL examples: 10)  \n",
       "1          Davinci-003 (Scaffold, #ICL examples: 10)  \n",
       "2              GPT-3.5 (Scaffold, #ICL examples: 10)  \n",
       "3                 GPT-4 (Scaffold, #ICL examples: 5)  \n",
       "4                  GPT-4 (Random, #ICL examples: 10)  \n",
       "5  Zero-shot (GPT-4) (Zero-shot (GPT-4), #ICL exa...  \n",
       "\n",
       "[6 rows x 22 columns]"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "design_performance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# all data in one file "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## molecule captioning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T14:29:06.624880Z",
     "start_time": "2023-05-16T14:29:06.618610Z"
    }
   },
   "outputs": [],
   "source": [
    "params = [\n",
    "    ('Scaffold_SIM', 10, 'SMILES', 'gpt4'),\n",
    "    ('Scaffold_SIM', 10, 'SMILES', 'davinci003'),\n",
    "    ('Scaffold_SIM', 10, 'SMILES', 'gpt35'),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T14:31:27.524423Z",
     "start_time": "2023-05-16T14:31:27.507796Z"
    }
   },
   "outputs": [],
   "source": [
    "caption_detail_summary = pd.read_csv(detail_predict_file)  \n",
    "\n",
    "caption_detail_summary = caption_detail_summary[['index', 'SMILES', 'description']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T14:33:47.171530Z",
     "start_time": "2023-05-16T14:33:47.145953Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10185/269367765.py:14: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  caption_detail_summary[model] = tem['pred_1']\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10185/269367765.py:14: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  caption_detail_summary[model] = tem['pred_1']\n",
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10185/269367765.py:14: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  caption_detail_summary[model] = tem['pred_1']\n"
     ]
    }
   ],
   "source": [
    "detail_save_folder = folder + '/results_test/'\n",
    "TASK_Types = ['molecule_captioning']\n",
    "# scaffold search names \n",
    "scaffold_col_smiles_name = 'SMILES'\n",
    "\n",
    "for task in TASK_Types:\n",
    "    for sample_method, sample_num, name, model in params:\n",
    "        \n",
    "        details_results = []\n",
    "        performance_results = []\n",
    "\n",
    "        detail_predict_file = detail_save_folder + 'valid_{}_{}_{}_{}_{}.csv'.format(task, model, name, sample_num, sample_method)\n",
    "        tem = pd.read_csv(detail_predict_file)\n",
    "        caption_detail_summary[model] = tem['pred_1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T14:40:17.204971Z",
     "start_time": "2023-05-16T14:40:17.182124Z"
    }
   },
   "outputs": [],
   "source": [
    "tem = pd.read_csv(detail_save_folder + 'molecule_captioning_gpt4_zero_shot.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T14:40:50.052674Z",
     "start_time": "2023-05-16T14:40:50.046332Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/d8/4krhpv4d7130t6yv97svrwh80000gn/T/ipykernel_10185/1703562037.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  caption_detail_summary['zero_shot'] = tem['pred_1']\n"
     ]
    }
   ],
   "source": [
    "caption_detail_summary['zero_shot'] = tem['pred_1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T14:50:44.286701Z",
     "start_time": "2023-05-16T14:50:44.278608Z"
    }
   },
   "outputs": [],
   "source": [
    "tem = pd.read_csv(detail_save_folder + \"mol_captioning_test_pred.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T14:50:44.515611Z",
     "start_time": "2023-05-16T14:50:44.511289Z"
    }
   },
   "outputs": [],
   "source": [
    "caption_detail_summary['baseline'] =  tem['pred']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T14:51:49.523263Z",
     "start_time": "2023-05-16T14:51:49.517921Z"
    }
   },
   "outputs": [],
   "source": [
    "caption_detail_summary = caption_detail_summary.rename({\"ground_truth_description\": \"label\"}, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T14:54:48.592163Z",
     "start_time": "2023-05-16T14:54:48.575805Z"
    }
   },
   "outputs": [],
   "source": [
    "caption_detail_summary.to_csv(detail_save_folder + \"molecule_captioning_details_summary.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## molecule design"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T15:36:26.406740Z",
     "start_time": "2023-05-16T15:36:26.375136Z"
    }
   },
   "outputs": [],
   "source": [
    "detail_save_folder = folder + '/results_test/'\n",
    "TASK_Types = ['molecule_design']\n",
    "# scaffold search names \n",
    "scaffold_col_smiles_name = 'SMILES'\n",
    "\n",
    "for task in TASK_Types:\n",
    "    for sample_method, sample_num, name, model in params:\n",
    "        \n",
    "        details_results = []\n",
    "        performance_results = []\n",
    "\n",
    "        detail_predict_file = detail_save_folder + 'valid_{}_{}_{}_{}_{}.csv'.format(task, model, name, sample_num, sample_method)\n",
    "        tem = pd.read_csv(detail_predict_file)\n",
    "        caption_detail_summary[model] = tem['pred_1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T15:37:57.021388Z",
     "start_time": "2023-05-16T15:37:57.012526Z"
    }
   },
   "outputs": [],
   "source": [
    "tem = pd.read_csv(detail_save_folder + 'molecule_design_gpt4_zero_shot.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T15:37:57.390633Z",
     "start_time": "2023-05-16T15:37:57.386509Z"
    }
   },
   "outputs": [],
   "source": [
    "caption_detail_summary['zero_shot'] = tem['pred_1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T15:38:32.987850Z",
     "start_time": "2023-05-16T15:38:32.978641Z"
    }
   },
   "outputs": [],
   "source": [
    "tem = pd.read_csv(detail_save_folder + \"mol_design_test_pred.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T15:38:37.096136Z",
     "start_time": "2023-05-16T15:38:37.086775Z"
    }
   },
   "outputs": [],
   "source": [
    "caption_detail_summary['baseline'] =  tem['pred']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T15:38:59.912333Z",
     "start_time": "2023-05-16T15:38:59.902669Z"
    }
   },
   "outputs": [],
   "source": [
    "caption_detail_summary = caption_detail_summary.rename({\"SMILES\": \"label\"}, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T15:39:16.515187Z",
     "start_time": "2023-05-16T15:39:16.475261Z"
    }
   },
   "outputs": [],
   "source": [
    "caption_detail_summary.to_csv(detail_save_folder + \"molecule_design_details_summary.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-16T15:54:39.385023Z",
     "start_time": "2023-05-16T15:54:39.370724Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>label</th>\n",
       "      <th>description</th>\n",
       "      <th>gpt4</th>\n",
       "      <th>davinci003</th>\n",
       "      <th>gpt35</th>\n",
       "      <th>zero_shot</th>\n",
       "      <th>baseline</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>52</td>\n",
       "      <td>CC(C)C[C@H]1C(=O)N2CCC[C@H]2[C@]2(O)O[C@](NC(=...</td>\n",
       "      <td>The molecule is an indole alkaloid. It has a r...</td>\n",
       "      <td>CN1CC[C@@]23c4c5cccc1c1c(ccc3c1C2)CCN5C(=O)C(C...</td>\n",
       "      <td>Cc1ccc2c(c1)c3c(cc4ccccc4[nH]3)C2=C1C[C@@H]3C[...</td>\n",
       "      <td>CN1[C@@H]2CC[C@@H]1[C@H](c1c[nH]c3ccccc13)C(=O...</td>\n",
       "      <td>CN1CCC[C@H]1[C@H]2Cc3c[nH]c4cccc(C2)c34</td>\n",
       "      <td>CC(C)[C@@]1(C(=O)N2[C@H](C(=O)N3CCC[C@H]3[C@@]...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>679</td>\n",
       "      <td>C[C@@H]1O[C@@H](OC[C@H]2O[C@@H](Oc3cc(O)c4c(c3...</td>\n",
       "      <td>The molecule is a disaccharide derivative that...</td>\n",
       "      <td>C[C@@H]1O[C@@H](O[C@H]2[C@H](Oc3cc(O)c4c(c3)O[...</td>\n",
       "      <td>C[C@@H]1O[C@@H](O[C@H]2[C@H](Oc3cc(O)c4c(c3)O[...</td>\n",
       "      <td>C[C@@H]1O[C@@H](O[C@H]2[C@H](Oc3cc(O)c4c(c3)O[...</td>\n",
       "      <td>C[C@H]1[C@@H](O[C@H](CO)[C@@H](O)[C@H]1O)O[C@@...</td>\n",
       "      <td>C[C@H]1[C@@H]([C@H]([C@H]([C@@H](O1)OC[C@@H]2[...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1253</td>\n",
       "      <td>CCCCC[C@H]1O[C@H]1C[C@@H](/C=C/C=C\\C/C=C\\CCCC(...</td>\n",
       "      <td>The molecule is a hydroperoxy fatty acid that ...</td>\n",
       "      <td>CCCC[C@H]1O[C#@H]1C/C=C\\C/C=C\\C[C@@H](CCCC(=O)...</td>\n",
       "      <td>CCCCC[C@@H]1O[C@H]1C/C=C\\C/C=C\\C[C@H](C/C=C\\CC...</td>\n",
       "      <td>CCCCC[C@H]1O[C@H]1C/C=C\\C=C\\[C@@H](CCCC(=O)O)OO</td>\n",
       "      <td>CC\\C=C/C\\C=C\\C\\C=C/CC1OC1C[C@@](OO)([C@H]CCCCC)C</td>\n",
       "      <td>CCCCC[C@@H]1[C@@H](O1)C/C=CC/C=CC=C[C@H](C/C=C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2130</td>\n",
       "      <td>CC(=O)N[C@@H]1[C@@H](O)[C@H](O[C@@H]2O[C@H](CO...</td>\n",
       "      <td>The molecule is a linear amino tetrasaccharide...</td>\n",
       "      <td>CC(=O)N[C@H]1[C@H](O[C@@H]2[C@H](O)[C@@H](O)[C...</td>\n",
       "      <td>CC(=O)N[C@H]1[C@H](O[C@H]2[C@H](O)[C@@H](O[C@H...</td>\n",
       "      <td>CC(=O)N[C@H]1[C@H](O[C@@H]2[C@H](O)[C@@H](O)[C...</td>\n",
       "      <td>OC[C@H]1OC(O[C@H]2O[C@@](C(N)=O)(CO)C(O)C(O)C2...</td>\n",
       "      <td>CC(=O)N[C@@H]1[C@H]([C@H]([C@H](O[C@H]1O)CO)O[...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>203</td>\n",
       "      <td>COC[C@H]1OC(=O)c2coc3c2[C@@]1(C)C1=C(C3=O)[C@@...</td>\n",
       "      <td>The molecule is an organic heteropentacyclic c...</td>\n",
       "      <td>CC(=O)O[C@@H]1C[C@H](O)C[C@@H]2C(=O)C(=O)[C@@H...</td>\n",
       "      <td>COC(=O)C1C(=O)C2C3C4C5C(C(=O)C1C2C(=O)O)C(=O)C...</td>\n",
       "      <td>CC(=O)O[C@@H]1[C@@]2(CC[C@H]3[C@H]2[C@@H](C=C[...</td>\n",
       "      <td>CC(=O)OC1C(=O)O[C@@H]2[C@@]3([C@@H]2[C@@@@H]([...</td>\n",
       "      <td>CC(=O)O[C@@H]1C[C@H]2[C@](C=CC(=O)C2(C)C)([C@@...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>3144</td>\n",
       "      <td>COc1cc(C2Oc3c(OC)cc(/C=C/CO)cc3C2CO)ccc1O</td>\n",
       "      <td>The molecule is a guaiacyl lignin obtained by ...</td>\n",
       "      <td>OCC1C(O)c2cc(O)ccc2C(CO)c2ccc(O)cc21</td>\n",
       "      <td>CC1(C)OC(=O)C[C@H]2C[C@@H]3[C@H](C2)OC(C3)C1C(C)C</td>\n",
       "      <td>Oc1ccc(C[C@@H]2[C@H](O)c3ccc(O)cc3OC[C@@H]2O)c...</td>\n",
       "      <td>C1=CC(=C(C=C1OC)O)C2C(C(OC(=O)C(C2OC1=CC(=C(C=...</td>\n",
       "      <td>COC1=CC(=CC2=C1OC(C2CO)C3=CC(=C(C=C3)O)OC)/C=C/CO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>2846</td>\n",
       "      <td>CCCCC/C=C\\C/C=C\\CC1OC1C/C=C\\CCCC(=O)SCCNC(=O)C...</td>\n",
       "      <td>The molecule is an unsaturated fatty acyl-CoA ...</td>\n",
       "      <td>CCCCC/C=C\\C/C=C\\CC1OC1CCC/C=C\\C/C=C\\CCC(=O)SCC...</td>\n",
       "      <td>CCCCC/C=C\\CC1OC1C/C=C\\C/C=C\\CCCCC(=O)SCCNC(=O)...</td>\n",
       "      <td>CCCCC/C=C\\CC1OC1C/C=C\\C/C=C\\CCCC(=O)SCCNC(=O)C...</td>\n",
       "      <td>CC\\C=C(/C=C/C=C\\C=C\\C=C\\C1CO1)C(=O)SCCNC(=O)CC...</td>\n",
       "      <td>CCCCC/C=CC/C=CCC1C(O1)C/C=CCCCC(=O)SCCNC(=O)CC...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>642</td>\n",
       "      <td>CC(=O)N[C@@H]1[C@@H](O[C@@H]2O[C@H](CO)[C@H](O...</td>\n",
       "      <td>The molecule is an alpha-L-Fucp-(1-&gt;2)-beta-D-...</td>\n",
       "      <td>CC(=O)N[C@@H]1[C@@H](O[C@H]2O[C@@H](C)[C@@H](O...</td>\n",
       "      <td>CC(=O)N[C@H]1[C@H]([C@H](O)[C@H](O)CO)O[C@@](O...</td>\n",
       "      <td>CC(=O)N[C@@H]1[C@@H](O[C@@H]2O[C@H](CO)[C@H](O...</td>\n",
       "      <td>O[C@H]1[C@@H](O)[C@H](CO[C@@H]2O[C@H](CO)[C@@H...</td>\n",
       "      <td>C[C@H]1[C@H]([C@H]([C@@H]([C@@H](O1)O[C@@H]2[C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>93</td>\n",
       "      <td>N[C@@H](CCC(=O)N[C@@H](CSCCO)C(=O)NCC(=O)O)C(=O)O</td>\n",
       "      <td>The molecule is a Glu-Cys-Gly tripeptide deriv...</td>\n",
       "      <td>CCO[C@@H]1CSSC[C@@H](N)C(=O)N[C@@H](Cc2ccccc2)...</td>\n",
       "      <td>CC(C(=O)O)NCCC(C(=O)O)NCC(O)C(C(=O)O)N</td>\n",
       "      <td>C[C@@H](C(=O)N[C@@H](CSSC[C@H](C(=O)O)N)C(=O)N...</td>\n",
       "      <td>N[C@@H](CCC(=O)N[C@@H](CS[C@H](CO)O)C(=O)NCC(=...</td>\n",
       "      <td>C(CC(=O)N[C@@H](CS)C(=O)NCC(=O)O)[C@@H](C(=O)O)N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>1412</td>\n",
       "      <td>Cc1cccc(CC(=O)N2CCc3c2ccc(-c2cn(C)c4ncnc(N)c24...</td>\n",
       "      <td>The molecule is a pyrrolopyrimidine that is 7-...</td>\n",
       "      <td>CC(=O)N(c1cccnc1C)C(=O)Nc1cc2CCN(F)CCc2cc1c1cc...</td>\n",
       "      <td>Cc1c(C)c(C(=O)NCC2=NC=C(C3=CC=C(F)C=C3)N2C)c2c...</td>\n",
       "      <td>CC1=C(N=C2C(NC=C2C3=C(F)C=CC=C3NC(=O)C4=CN(C)C...</td>\n",
       "      <td>Cc1ccnc(C(=O)Nc2ccc3CC(N(Cc3c2)C(=O)Nc2cc(C)nc...</td>\n",
       "      <td>CC1=C(C=CC2=C1NC(=C2C(=O)N3CCC4=C(C3)C=NC=C4F)...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    index                                              label  \\\n",
       "0      52  CC(C)C[C@H]1C(=O)N2CCC[C@H]2[C@]2(O)O[C@](NC(=...   \n",
       "1     679  C[C@@H]1O[C@@H](OC[C@H]2O[C@@H](Oc3cc(O)c4c(c3...   \n",
       "2    1253  CCCCC[C@H]1O[C@H]1C[C@@H](/C=C/C=C\\C/C=C\\CCCC(...   \n",
       "3    2130  CC(=O)N[C@@H]1[C@@H](O)[C@H](O[C@@H]2O[C@H](CO...   \n",
       "4     203  COC[C@H]1OC(=O)c2coc3c2[C@@]1(C)C1=C(C3=O)[C@@...   \n",
       "..    ...                                                ...   \n",
       "95   3144          COc1cc(C2Oc3c(OC)cc(/C=C/CO)cc3C2CO)ccc1O   \n",
       "96   2846  CCCCC/C=C\\C/C=C\\CC1OC1C/C=C\\CCCC(=O)SCCNC(=O)C...   \n",
       "97    642  CC(=O)N[C@@H]1[C@@H](O[C@@H]2O[C@H](CO)[C@H](O...   \n",
       "98     93  N[C@@H](CCC(=O)N[C@@H](CSCCO)C(=O)NCC(=O)O)C(=O)O   \n",
       "99   1412  Cc1cccc(CC(=O)N2CCc3c2ccc(-c2cn(C)c4ncnc(N)c24...   \n",
       "\n",
       "                                          description  \\\n",
       "0   The molecule is an indole alkaloid. It has a r...   \n",
       "1   The molecule is a disaccharide derivative that...   \n",
       "2   The molecule is a hydroperoxy fatty acid that ...   \n",
       "3   The molecule is a linear amino tetrasaccharide...   \n",
       "4   The molecule is an organic heteropentacyclic c...   \n",
       "..                                                ...   \n",
       "95  The molecule is a guaiacyl lignin obtained by ...   \n",
       "96  The molecule is an unsaturated fatty acyl-CoA ...   \n",
       "97  The molecule is an alpha-L-Fucp-(1->2)-beta-D-...   \n",
       "98  The molecule is a Glu-Cys-Gly tripeptide deriv...   \n",
       "99  The molecule is a pyrrolopyrimidine that is 7-...   \n",
       "\n",
       "                                                 gpt4  \\\n",
       "0   CN1CC[C@@]23c4c5cccc1c1c(ccc3c1C2)CCN5C(=O)C(C...   \n",
       "1   C[C@@H]1O[C@@H](O[C@H]2[C@H](Oc3cc(O)c4c(c3)O[...   \n",
       "2   CCCC[C@H]1O[C#@H]1C/C=C\\C/C=C\\C[C@@H](CCCC(=O)...   \n",
       "3   CC(=O)N[C@H]1[C@H](O[C@@H]2[C@H](O)[C@@H](O)[C...   \n",
       "4   CC(=O)O[C@@H]1C[C@H](O)C[C@@H]2C(=O)C(=O)[C@@H...   \n",
       "..                                                ...   \n",
       "95               OCC1C(O)c2cc(O)ccc2C(CO)c2ccc(O)cc21   \n",
       "96  CCCCC/C=C\\C/C=C\\CC1OC1CCC/C=C\\C/C=C\\CCC(=O)SCC...   \n",
       "97  CC(=O)N[C@@H]1[C@@H](O[C@H]2O[C@@H](C)[C@@H](O...   \n",
       "98  CCO[C@@H]1CSSC[C@@H](N)C(=O)N[C@@H](Cc2ccccc2)...   \n",
       "99  CC(=O)N(c1cccnc1C)C(=O)Nc1cc2CCN(F)CCc2cc1c1cc...   \n",
       "\n",
       "                                           davinci003  \\\n",
       "0   Cc1ccc2c(c1)c3c(cc4ccccc4[nH]3)C2=C1C[C@@H]3C[...   \n",
       "1   C[C@@H]1O[C@@H](O[C@H]2[C@H](Oc3cc(O)c4c(c3)O[...   \n",
       "2   CCCCC[C@@H]1O[C@H]1C/C=C\\C/C=C\\C[C@H](C/C=C\\CC...   \n",
       "3   CC(=O)N[C@H]1[C@H](O[C@H]2[C@H](O)[C@@H](O[C@H...   \n",
       "4   COC(=O)C1C(=O)C2C3C4C5C(C(=O)C1C2C(=O)O)C(=O)C...   \n",
       "..                                                ...   \n",
       "95  CC1(C)OC(=O)C[C@H]2C[C@@H]3[C@H](C2)OC(C3)C1C(C)C   \n",
       "96  CCCCC/C=C\\CC1OC1C/C=C\\C/C=C\\CCCCC(=O)SCCNC(=O)...   \n",
       "97  CC(=O)N[C@H]1[C@H]([C@H](O)[C@H](O)CO)O[C@@](O...   \n",
       "98             CC(C(=O)O)NCCC(C(=O)O)NCC(O)C(C(=O)O)N   \n",
       "99  Cc1c(C)c(C(=O)NCC2=NC=C(C3=CC=C(F)C=C3)N2C)c2c...   \n",
       "\n",
       "                                                gpt35  \\\n",
       "0   CN1[C@@H]2CC[C@@H]1[C@H](c1c[nH]c3ccccc13)C(=O...   \n",
       "1   C[C@@H]1O[C@@H](O[C@H]2[C@H](Oc3cc(O)c4c(c3)O[...   \n",
       "2     CCCCC[C@H]1O[C@H]1C/C=C\\C=C\\[C@@H](CCCC(=O)O)OO   \n",
       "3   CC(=O)N[C@H]1[C@H](O[C@@H]2[C@H](O)[C@@H](O)[C...   \n",
       "4   CC(=O)O[C@@H]1[C@@]2(CC[C@H]3[C@H]2[C@@H](C=C[...   \n",
       "..                                                ...   \n",
       "95  Oc1ccc(C[C@@H]2[C@H](O)c3ccc(O)cc3OC[C@@H]2O)c...   \n",
       "96  CCCCC/C=C\\CC1OC1C/C=C\\C/C=C\\CCCC(=O)SCCNC(=O)C...   \n",
       "97  CC(=O)N[C@@H]1[C@@H](O[C@@H]2O[C@H](CO)[C@H](O...   \n",
       "98  C[C@@H](C(=O)N[C@@H](CSSC[C@H](C(=O)O)N)C(=O)N...   \n",
       "99  CC1=C(N=C2C(NC=C2C3=C(F)C=CC=C3NC(=O)C4=CN(C)C...   \n",
       "\n",
       "                                            zero_shot  \\\n",
       "0             CN1CCC[C@H]1[C@H]2Cc3c[nH]c4cccc(C2)c34   \n",
       "1   C[C@H]1[C@@H](O[C@H](CO)[C@@H](O)[C@H]1O)O[C@@...   \n",
       "2    CC\\C=C/C\\C=C\\C\\C=C/CC1OC1C[C@@](OO)([C@H]CCCCC)C   \n",
       "3   OC[C@H]1OC(O[C@H]2O[C@@](C(N)=O)(CO)C(O)C(O)C2...   \n",
       "4   CC(=O)OC1C(=O)O[C@@H]2[C@@]3([C@@H]2[C@@@@H]([...   \n",
       "..                                                ...   \n",
       "95  C1=CC(=C(C=C1OC)O)C2C(C(OC(=O)C(C2OC1=CC(=C(C=...   \n",
       "96  CC\\C=C(/C=C/C=C\\C=C\\C=C\\C1CO1)C(=O)SCCNC(=O)CC...   \n",
       "97  O[C@H]1[C@@H](O)[C@H](CO[C@@H]2O[C@H](CO)[C@@H...   \n",
       "98  N[C@@H](CCC(=O)N[C@@H](CS[C@H](CO)O)C(=O)NCC(=...   \n",
       "99  Cc1ccnc(C(=O)Nc2ccc3CC(N(Cc3c2)C(=O)Nc2cc(C)nc...   \n",
       "\n",
       "                                             baseline  \n",
       "0   CC(C)[C@@]1(C(=O)N2[C@H](C(=O)N3CCC[C@H]3[C@@]...  \n",
       "1   C[C@H]1[C@@H]([C@H]([C@H]([C@@H](O1)OC[C@@H]2[...  \n",
       "2   CCCCC[C@@H]1[C@@H](O1)C/C=CC/C=CC=C[C@H](C/C=C...  \n",
       "3   CC(=O)N[C@@H]1[C@H]([C@H]([C@H](O[C@H]1O)CO)O[...  \n",
       "4   CC(=O)O[C@@H]1C[C@H]2[C@](C=CC(=O)C2(C)C)([C@@...  \n",
       "..                                                ...  \n",
       "95  COC1=CC(=CC2=C1OC(C2CO)C3=CC(=C(C=C3)O)OC)/C=C/CO  \n",
       "96  CCCCC/C=CC/C=CCC1C(O1)C/C=CCCCC(=O)SCCNC(=O)CC...  \n",
       "97  C[C@H]1[C@H]([C@H]([C@@H]([C@@H](O1)O[C@@H]2[C...  \n",
       "98   C(CC(=O)N[C@@H](CS)C(=O)NCC(=O)O)[C@@H](C(=O)O)N  \n",
       "99  CC1=C(C=CC2=C1NC(=C2C(=O)N3CCC4=C(C3)C=NC=C4F)...  \n",
       "\n",
       "[100 rows x 8 columns]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caption_detail_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-20T23:32:43.745545Z",
     "start_time": "2023-05-20T23:32:43.729017Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcIAAACWCAIAAADCEh9HAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nO3dd1xT5/cH8E/CEIEqaF2IIChaRHCAtoIiKqJSV11YLeoPV+vAtu5V0Pp1tgpY66jbWgfUgduKCxAXAoKiKKgsFZSlQoAk5/dHImCHoklusD3vV//IteE5T3jB4d77nHseERGBMcbYuxJrewKMMfZ+4zTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq4TTKGGMq0dX2BN4TMhk2bsTly6hbF2PHwtpa2xNijFUVfDZaOWPGIDMTfn7w8EDv3khL0/aEGGNVhYiItD2HKi87G25uSEiASAQAa9YgKwsLFmh7WoyxKoHPRivh/n00aaLMoQCaN0dKilYnxBirQjiNvlZJCQDUqYPs7PJ/zMpC3bramhFjTCXPnyMxEUVFahyS0+g/IIK/P1xdUVgIS0sACA8HgMJCrF2LwYO1OzvG2LtYtgw9e2LtWri5YcMGAHj4EElJePJElVH53ujfKSnBmDHYsQN6ejh2DN26ITUVU6ciLw8yGUaPxvDh2p4iY+wtxcZi0iScPw+xGCUlcHTE4cN4+BBpaWjeHA4O7zwwFzz9xfPnGDwYx4/D2Bh79qBzZ+zZAy8vBAdre2aMMRVERMDTE2IxAOjrw90dFy/CywuffKLiwO9hGiXCzz/j0CHI5ejUCbNmQU9PbYNnZqJ3b8TEoH59HDmCZs3Qpw+OH8fjx/D1VVsUxpjwpFLoVsh4enooLVXLwO/hvdFffkFUFEJDceIEnj3Dd9+pa+CEhITdX36J2FjY2uLiRZiZwc0Nx4+jdm04OakrCmNMO1q3xoUL5YeRkWjTRi0Dv4f3Rjt1wubNsLEBgKIifPQRHjxQfdQzZ84MGDAgLy/vrI9P5x9+wNOn6NULd++iSRMcO6YMxxh7H+Xm4sULmJvj889hagpnZ5w8CRMTBAWpZfj3MI3a2CAmBsbGykMzM2RkIDkZTZu+85AhISHe3t4SiWTAgAG//vqrJDbWtHdv5OSgQweEhuLDD9Uzc8aY8DIz4emJ0lJERMDEBBERSEmBrS3at1dXhPfwor5RI9y/r3ydnw9DQ9y4ARsb2NnB3x+3br3teIGBgV5eXhKJxNfXNzg4+Pjx41YeHo9sbNC/P06d4hzKmKY8eIDgYPzxh3qrOF8RF4d27RAXBx0dFBZCJEKnThg5Uo05FADoPSKXExHt3UuenpSdTQUF5ONDK1bQvn1kakqA8r/Wrel//6M7dyoz5L59+wCIxeKAgAAiWrlypVgsBjDn669JKtXop2HsP23vXurYkTZsoAULyMmJsrPVH+LUKapZkwDq0oVyc9U//kvvTxrNyKBPPqGICCKi33+nQYPos8/ol1+UubW4mA4fppEjycREkUzlItGn7u5LlixJTk5+zagymWzkyJEhISFyuXzmzJkARCKRn5+fEJ+Isf8suZyaNKEnT5SHQUE0Z46aI2zfTnp6BNCwYVRcrN7B/0RNaVSRyzQnNpYaNiSAOnd+wzslEgoNJW/vyI8/LjvjbtGihZ+fX1JSkuItjx49WrFihZ+f34ULF65cufLHH39IJJKhQ4cC0NfX37lzp2Y/C2Ps4UNq06b8MCaGevVS4/ABAQHftWtHurrk60symRpH/lsqp9EdO8jZmTw8qFMnOndOHVP6i7CwdzgzLyoqOnDgwPDhwz/44ANFMhWJRO3bt79+/XqbNm127tx58uTJ48ePZ2Vlpaen5+bmtmjRwsTE5MyZMxr5CIyxinJzyc6u/PDiRfrsMzp3jkaPphMnqLT0nQcuLS0dO3YsAB0dnYtbtqg+08pQLY3euEFOTvTiBRFRRgbZ2FB+Pu3ZQ4sW0fnzapkfbd9O+voE0KBBVFT0DgMUFRWFhoZ6e3t/8MEHenp6WVlZdevWffr0qeL/HjlyZMuWLUR0//79hIQE9cyZMfZGrVrRywtEmjOHVq2i0aOVyxu1apG3N4WGUknJWw35/Pnz3r17AzAwMNi7d6/65/wPVEujK1fSDz+UH44cSSdPqjihioKWLi01NyeAZsxQ/b5BYWHhuXPniCggIMDa2nrSpEkZGRnbtm1bvHixOibLGKsEiYR27yYiioqitm3J15eGDKF+/aioiG7eJD8/srUtXy6uU6dw8uTTp09LK7He++jRIycnJwC1atWKUCyiCEW1NOrnR+vXlx9OnkzBwRQWRikpKk6rtLR03LhxAPra2Mh+/lnF0f6qqKho7dq17dq14zTKmHByc8nNjQBl3igpoRs3KC3tz29LSCA/P/roIwKCP/kEQO3atb29vUNDQ0tfvd4vLCwsLi4motzc3J9//hlA06ZN71SuSkeNVEujv/1GEyaUH3bqRNevk6UlAdSiBfn5lZ+0vw1Nn5mXlJQQUX5+voWFBadRxoiI7t6lJUtozhz64w9NhcjIoFatCKAGDSgmplJfEhu7Z8UKmwrPENavX3/ixInnzp2TSCT9+/f38PBwcXEJDQ0NCgq6fft2YGBgVlaWpub/z1RLo8XF1L49rVtH0dH03Xc0eDDl5dGwYfTBB8pzcpGIPv6YfvhB+uBBJYd88uSJs7Oz4sw8PDxcpen9nYKCgs6dO3t6erZt23bbtm0hISE//fST2qMw9j5JSKDWrenoUbp0iT77jAIC1B8iPp4aNSKA7Oyo0tmgwgQT/Pz8mjdvrkimJiYmYWFh/fv3L3vD5cuXyxY8hKfySv3z57R2Lc2cSb/+Wl6vXlSkqDoqy6djO3VSVB29sYqzVatWAJo0aZL0TmeylSSRSEpKSq5du3b9+nXNRalaoqLI05Pc3alvX/rvfGpWGT4+dOCA8nVBAVlYqHf4nDNnqEYNtZTBR0dHz5o1a+7cuZmZmQ0bNly7dm1eXh4RjRo16uLFi2qa71vTcPl9YSEFB8uHDLEwN1f8GRGLxS4uLoGBgRkZGUSUl5c3derUTz/9dMqUKRKJxMfH59ixY87Ozo8fP9bsxIg2bNgAYPjw4ZoOVCU8ekS2tsqzgMREat6cnj3T9pxYldGp0yv331q2VOMzRdu2bTMxMnrStu07F9v8k7t3786ZM8fa2josLOxfnUZfevHixZ49ewYOHFi9evWyfDpv3rxvvvlmxYoVz58/v3TpEhEVFRURkVzTxfxERBQdHQ2gefPmAsTSvs2bafbs8sPRo+nwYe3NhlUxffvSlSvlhxYWlJdHlpbKqiMVqjgXLVokEokALJ43T0MP6Rw9enTIkCHaTaMCtSYxNDQcMmRISEjI06dPFVWc1atXt7e3Ly4ufvbsWfXq1du3by+VShV3RUVle3Bqkr29vYGBQVJSUn5+vgDhtCw7+5Vt+OrWRVaW9mbDqpg+fbB+PRTN3kJD0aoVzp/HgwfYsQN9+8LCAr6+CA+HXF75IWUy2YQJE+bNmycWi3/66afZ338Ptf5ex8bGbtiw4ezZszt27GjXrp0aR34X2srfBQUFEonk4cOHn3/+uY2NTWBgYElJib29vZBzUHz3/81PLqWnU34+EdGePTRxYvm/DxxI/+JPzSpJLid/f0pOJqmUZs0iV1fy8KB+/Sgjg4goOppmzSJr6/IqzoYN1/n5RUREvPF6UYAy+Ly8vB07dnz//ff79++XyWSRkZGRkZGaCFQZVaI1SV5eXuvWrSMjIwVOo1999RWAFStWCBlUOAkJZGFBbm4kkVBREdnb05EjysfM2rcX4EFjVqVJJDR0KAHUsmX54vDf5kdFFaeNTW7Nmvr6+gDMzc19fX3Dw8Mr5tObN2+eP38+Pz8/Ozs7MjKyefPmmi6Dz8zM3Lx58++//15aWlqjRg2xWPxMS3f8tdxv9P79+3K53NDQ8IMPPtDR0RE4uqOjIwDFTdJ/m9On4eKC1FTI5ZBIYGCAU6dw/jxGj0ZCAk6eVG7sxaqgjAxMnIhevTBxItLTNRIiLw89emD3bhgbY8UKlP3q/e11t6KT7+3bJWFhkydPtrS0TE9PDwoK6tSpU+PGjadNm5acnPzNN9/Mnz//2LFjS5Ysyc/PLygoOHLkyKVLl1xcXDQyfwBAbGysj49PYGCgrq6ujY2NXC6PiYnRXLjX0UryLhMYGOju7u7m5rZs2TKpVDp58mQho8fGxtY1Mprr4SFk0HKFhZpqaRocTAYGBNCAAVRYSOHhxI8YvC8KC8nOjs6cIamUzpyhFi2osFDNId6hDP5ViirOJk2aKHLI+fPnzc3NC1/O886dO4cFWcDMzs4GYGxsLJPJFA89rly5UoC4f1UlLurT0tIKCwvPnz/fs2fP+fPnCxZXXlpKxsYkEmm0pevfSEwkNzfq25dcXWnSJFVWQv9KvnQpiUQE0NSpJJOVp9Rjx9QYhWnK/v00dmz54bhxtG+fGoePj42VNW9OANnb/81TmG9DLpdHRkbOnj1bJpNNmjTJxcVl165dxcXFJ06cmDJlirom/HoWFhYAbt68uX79emivflH7V3b9+/dv1KjRmTNniouLjx8/HhYWJlhoka4uWrYEEQS+rvfxwbJlOHgQ586hpASbN6tlVMVP884jRwBg+XL88ANWr4aXFyQSjBkDd3e1RGGalZEBC4vyw8aNkZaGnByoY8+0sLCwjp07TzcwkHfvjvBwvKzmfjcikcjZ2Xnx4sVisXj16tWrVq06depU3759VZ9n5Sl6kURHRyteXL16VcjoZbSfRps2bQrg6tWrjo6OIpEoNjZWKpUKF16xc7KQ3/38fOTllW8FM2IETp5UfdTCwsKBAweuWbNm3NWrDw4dwrRpmDULX38NIvj54ZdfXtmhm1VZii0ay6SloWFDjBgBCwtMmYKIiHfOp9u3b+/Vq1d+fn6qjU1JaChq1lTPhF9q167dxo0bY2NjZTKZekd+jbLlDe3WL2o/jZZ9I0xNTa2trQsLC2/evClkeACCno0+fw4jo/JDIyM8f45793DgACSSdxsyJyenR48eBw8erFWr1okTJ+p163b3m2+wbBmqVcPOnfD3V8vEmRA8PHDuHOLiAOD6dZw9C3d33L6N9HQEBaFTJ1hZYcYMXLnyVqMGBgaOGjWqtLTU19d3z549BgYGapyyTCYbOnToypUrv/zyyw4dOgi5Vlx2Eqqnp2dvb1+7evX7168LFr2cVm4lVJSUlATAzMyMiLy8vABs3rxZuPDx8QSQlZVwEWUysrSkggLl4caNNHUqzZtHABkaUu/etG2bshN25aSkpChaNjRu3DgxMTEnJ8fV1dVIT++xiwsXh75PfvmFJk0iIrp9m7y8yN2dvLzo9m0iIrmcIiPp669J0X4XIOC3/v1nzpwZHR39xoEVhX06Ojpr1qzR0NwfPXp09OjRyMhImUyWnJy8bdu2+/fvayhWRU+ePBndsuV+d3eSSgumTSNd3Vc6IAtF+2lULpebmJgAyMzMXL58OYCJFQvFNU0qJSMjEonKd9fSqIcPiYjWrKE+feiPP2j7dnJwoLQ02rKFHB3L65xr1qQRI+SHDhVXYiuuGTNmAHB0dHz48OG9e/dsbW0Vf5ZiY2M1/nGYWsjlNH26siPapUtvePPVq+TrS2ZmnRwcFGdClpaWf6riDA8PDwwM3LdvX3Fx8fr16zds2KDpbvClpaV+fn6DBw+Wy+U+Pj4AVq9erblwr7CyIoASEmjjRgJo6FCB4lag/TRKRF27dgUQGhqqWF/6+OOPBQ3v4kKmpq88U6whAQFkZESKJ38vXKAlS2jdOqrYhOX+fQoIIBcXxVJ7hL29iYmJolvta/KpVCpdtGjRs2fP4uLiGjZsCKBly5apqaka/zj/BU+e0IgR5O5Obm60ZIlGnlkoLqZhwwggfX3asaOyXyWTnT93buLEifXr1y+7srSxsQkKCtq1a5eHh8eBAwcWLVr04sWLsLAwIhLg58HMzAzAnTt31qxZA2DUqFGajqg0aBABtHUrxcYSQDY2AsWtoEqk0RkzZohFop8XL87Py7vi5pbr6Pi2e7CoJDubrl+nqCgNNj0qLaVx4wggHR2qzC2LpCRatGjJ0KFlvyG1atVStL9S9Jxev369h4eHp6fnhQsXFixYkJCQcP78eWNjYwDu7u75igdAmeoGDaJffyUikkrp889J7VuklXWDNzZ+t4o0mUwWHh7u6+vboEEDAHPmzJkxY0ZQUFDZG8aPH6++6b5Onz59AOzatevixYuKv+XCxKUlSwigyZOptJSqV9dC/WIVSaN5+/bJa9SgTz8lImrWjIB3qwp+Fzk55OpKEybQvHnUtq3aduKr6Plz6t2bADIwoLe8sLp169b3339vb29flk/r169/+fJlZ2fnwsLCx48fZ2dnFxcXy2Sy7OzsZs2ajRgxokTIv0D/bqWl1KhR+eHly9SnjxqHf/DgQcLgwYpn1SkuTsXRpFJpWFhYSkpKQkJCs2bNhgwZouh6bmtrq47Jvpm/vz+AadOmFRUV6enp6ejoPH/+XIjAf/xBAHXoQET0yScEUFiYEHErqBJplJKTCaB69YiIPv+cAPrlF4FCz51LgYHK10lJ1KqVmsd/8oScnZWbHarQzD8lJSUgIMDFxaVXr17Xr19v3rz5g5ctxEeOHKloEZaVlSVMj8H/ipwcatGi/DA5mZyd6elTqsSqzhvFxsaamZkZ6OqmDRhA6r7ilslkp0+fbt68+eXLlwVLo4cPHwbg5uZGRG3atAEg0L5yOTkkEpGBAZWU0KRJBNCyZULErUD7BU8AYGWF2rXx+DHS04WuQLp4Ed27K1/b2KCwEM+eqWvs5ORkr759ZffuwdoaFy6gY8d3HsrKymrKlCkRERHBwcH29vbTp0/v16+fp6dnRoUawzp16gjTY/C/wtQUz5+jsFB5mJiIpk3x229wdETjxsoqzncSFhbWuXPnzMzMDp06GW/ahEaN1DZnAIBYLO7SpUu3bt2Sk5PVO/JrKGqPrl27JpfLFVWMAhXDm5rC2hoSCRITldnjxg0h4lZQNdKoSIS2bQEgOloL9fB/qmcmwtdf49gxlJaqMmpUVNQnn3yy98KFhR06ICoKL7eRUZGRkRGA0aNHx8TEKJ6dVcuw7M8UNbxffYUJE5CSgsuX4eeHKVMgl6N+fTx4oKzibN4c8+fjbWoVy8rgBw0adPToUUWZihotWrSob9++Xl5esbGxvXr1Um+J6GvUq1fP3Ny8oKDgzp07To6O3Rs3NtdQX5W/6toVXbqguBj9+uHOHWzbJlDcMgKf/f6jWbMIoPnz6dkzEotJX58kEiHizp5NZVvapaRQq1YUEaGsOjI1VXb/rkTV0Z/s37/f0NAQQI8ePQrKSkTVJCsrKz09nYhWr149adKksot6pjb795O5OSUmEhEFB9O4cTR9OpUVkMlkFB5Ovr5Uv77iRyW5USMrKytF1dHrB96xY4fiimHmzJmauwNTUFCg2KQnIyMjKChIsELsnRMmXOzc+WlwMF25QgAJdT+BiKikhBYtoh49yNOTtm8XLi4RVZV7o0QUHEwA9epFRIr9qenqVSHiPnlCHTrQjBm0fDk5OdGZM5SWRgsWUMuW5VWctWvT2LE3T5+WVq4hU1BQkOJBDh8fn1K1th1RSEhI8PLy6tat2+jRo7Ozs9evX3/37l21R6miCgvp0SPNhli9mnR0CCB//ze8UyqlU6do3Ljlnp5l5yUtWrTw9/e/efOm4i2nT58eP378hAkToqOjN27cGBUV1aZNG01vRrtq1SoHB4cDBw5EREQAaNOmjUbDlfv+ewLo22+puJiqVSOxmNR9DvGPpk6lOXNIKqVnz6h3b9q1S6C4RFSF0ui9ewSQmRkR0S+/0M8/a/y3pUxxMUVF0YkT9KcNWpOTaelSatGCgBI9vdq1atWqVUtRxfma1fD79+9Xr15dJBIt1nxvuoiIiIULF964cUPTgaqEkhIaOZK6d6fhw+njjzVSyyGXk5+fsgzez6/yX1daWnry5MkxY8bUrl27LJ/a29snJiba2dklJCRcunQpKSkpJydHIpFU5pEKFc2bNw/A7NmzX7x4oaurq6enV6j2bnt/69gxAsjVlYiUj5OcOydEXCIyNy+/arx2jXr0ECguEVWhNEpEFy/Sixd09CgFBtLRoxraAOtdxMffXb78o48+KvsNqVu37ldffXXmzBmpVLpv374OHTo4Ozv7+vpGRUX9+OOPv//++47K11GrYOzYsQACNLGreBW0Zg19843ydVwctW2r5vHfrQz+VVKpVFHFWbdu3bZt2968edPOzq7srs6cOXOEWbzev38/gO7duxNRy5YtAQh02yc7W1kDK5XS+PEEkDANQIuLydKy/DAri4TdR6MqpVEiGjCApk6l0FCaOpX699f2bP4sLi5u7ty5NjY2Zfk0ODi4cePGubm5RCSVSnNzc+/duyfYfNatWwfA29tbsIjaNHAgRUWVH9rZ/fnqQQW5ublzBg+Wm5hQjRp06pTqA5aUlNy7d08uly9cuNDa2nrcuHE5OTkTJ04ME6SkMS0tDYCpqalcLh81ahQAzT1N/2eWlgTQjRu0YQMBJFgDUAuL8t2bL18mT0+B4hJRVSl4UggPR2kpfvgBffrghx8gk+H8eW3P6RUODg6LFi1KSkqKiYmZPXu2g4NDr169LCwsAgMDc3JydHR0Ll26FBAQINh8tNtjUWgSCfT1yw+rV0dREdauhcr9wFJTUzt27Lg4OHihoyPCw9Gtm4oDAtDT02vcuLFIJJo/f/6tW7fMzMy+/fZb1YetJHNz8wYNGuTm5qakpAi9U05ZpY3AJTfe3pg7FyUlyM3FvHkYN06guACqSsGTwo0byrIvhfbtkZCgvdm8TuvWrRcvXhwXF2dkZBQSEiKXy11cXBR9VYRkb29frVq127dvFxQUCBxaC+ztcfmy8nV+Pp4+hUSCiRNhZ6fcLCgx8R1GTUhI6Nix440bN+zs7P5v82a87PehFkVFRRKJRE9Pr127doVl9aeCKMuegpZwokLnyZYtYWSE6tVRUiJE3AULYG6Ozz7D6NEYPx79+gkR9KWqlEaNjVHxR+3FCxgba282lVWnTp0FCxZcu3Zt+fLlQjasBaCvr29vby+Xy2NjY4WMK7TUVMhk+PprbNiANWuwbx8GD8Z33wGAjw9q1cLNm1iwAC1aoFUr/O9/+XfuVHLg06dPd+zYMS0trUuXLhERERYV286rw927d3v37t29e/eVK1f6+/tbWVnVVHez5H9SlkZbt249zclpxYcfQpg83qULRo6Eqyv09JCTg5iYV64hNEdHB998gyNHsG8fBgwQImJFQt5BeIP798nBQXmDo6iIHBxIkJaFKtq6dWtiYuKhQ4fs7OyOHz8u2C40CuPHjwfw448/ChlUUBcvUp06yu2J8vJo1y5as4auXy9/g1SqrOKsU0dRoGbTuHGLFi38/Pxu3br1moEfPXqkqO0dNmyYRJNFynK5PDc3Vy6XT58+3d3dXYCVeiJKOn78gbPz80GDiEi5gd2FCwLEVVq+nFxdyd2dPD3pzh3h4mpJVUqjRLR1Kzk50YgR5OREW7dqezZvJpVKt2zZMnny5FmzZt27d+/evXvHjx8XqLiEiIj2b926/JNPrk6bJljEcnfu0OHDpNFyq337qHp1AqhPnzc/BFFcTIcPp06eXPGMr23btkuXLk1JSSGiq1ev9u7du1OnTnPnzo2Li1u0aNHGjRtnzJih0UYEZ8+eNTEx8fT0JCJFsUdlGi2rwcOHysa1MhmNHk0AVWj7pFkHD1K/fsqNGiMiqH37KlR1oxlVLI0SkVRKGRma2nlYA+Ry+ZUrV7Zu3UpE3377rZ6e3nYhH6K4do0AatZMuIgK8+ZR794UEEBeXjRmjEZCBAUpy+B9fN5q81SJRBIaGvrFF1/UqFGjLJ/++uuvbdq0SUhIkMlkmZmZJSUlwrQTTElJAVCnTh0iGj58OID169cLEJeIqGFDAigpiX7+mQAaOVKguGPG0JEj5Ydt21JmpkChtaQq3RtV0NGBmRkE3M5FRUTUtWvXUaNGZWVl1a5du7S0VLhVUQAtW6J6ddy5g7w84YLevo0TJ3DoEKZMwe7dyMxUc00FEaZPh68v5HIsXoxNm95qP75q1ar16dNnx44dWVlZoaGh3t7eJiYmXbp0MTIyCg8Pl8lkDRo0uHz5sjBL51ZWVnXq1MnOzk5NTdXaorli2UewVaa8PFT4AwYTE+TmChRaS6peGn3fiMViRVuwa9euaaECSU8PDg4gQkyMcEGvXUOnTuWHbm5q/BUtKSkZ/sUXcVeuQF8f27dj9ux3HkqRT7dv356VlWVmZrZt27ZLly7Z2dltVtOO1pXUtm1bAFevXhX6x6Ns0bxVK+jrIzFRjd3LXqdZs/IeS6WlSElB48ZCxNUeTqNqUPbr4eTkJBKJYmJi/uV7ROvooGJNgkwGXV1s2IBly5CSosrAeXl5PXr0+O2337rHxxecOIEvvlB1qgAAPT09ANbW1lu2bImKipozZ47kXTdhfQdle6k7Ojrq6OjEx8cLFL3sJLRaNbRsCbkcwlR0TJiAn37CoUOIj8ekSRg2DIaGQsTVHk6jalB2sVarVq3GjRsXFhbeunVLyPCAsHtEt2uHc+fKM+nJk3B2xqpVmDULTZooqzgrXXVUJjMz083N7ezZsw0aNDgZFlbDzU29s46IiCguLs7Nza1WrZoisQqjrHLT0NDwo48+Ki0tjY+PFyJwu3YAEB0NuRw9eqBPHwjzqRs2xLFjSEjAli3o0QP/+58QQbVL2zdn/w0USdPc3JyIBg8eDGCL2jfteY24OAKoSRMhYj1+rHwi88cfyc2NFiygnj1p9mySy+ngQRo+nGrUKO+M1b596lk1H0EAAAvQSURBVJo1ldxoNz4+vlGjRgDs7OzKGvurkeK5zE8//XTgwIHh4eHJycnC9D0gotTUVAAN69UjosMzZ8a7uuYKtoX40qV08CA9f043bwrXbOm/h9OoGpTtEf3w4cOlS5cC+NbXV7jwpaXUujV5e2u8vOH2bWrShExMlEVOjx7R+fOUlvbKeyQSCg0lb29FPvXt3BmAoorzNd38Xrx4MWDAAABdunTJ1eR+ZHl5eWfPniWigIAAa2vrTZs2aS5WRTmdOpGhIaWkUFAQATR6tDBxiYh+/ZWcnGjyZHJ1pblzhYv7X8JpVD0G9ew50cHhzokT2WfPFltZkbOzoOGjo2nZMlq9WoMPLCjK4AFq1+6VTaH/SWEh/f77lLFjFe36AYhEog4dOqxatSotLS09Pd3V1bVbt26Ojo6ZmZmjRo3Kz8+fO3eupkvTTU1NAWRkZCie3J0wYYJGw5Xz9CSA9u6lyEgC1L/l1z95/JhsbUlRyCyXk7u7KhuCsX/CaVRNpk1TNvrNyyvfYEsYv/1G3brRiRMUEkKtW5c3aVej/fvJ0JAA8vB422vDoqIiRdXRBx98oMinnp6e/v7+q1atKnvPG5vGq0u3bt0AhIaGnj59GkD79u2FiUvffUcAzZxJL16Qri7p6pIwz2gcPfpKVW9QEGm+De5/EC8xqUnZOk/NmmjSRLnBljAWLsTOnfDwwMCBWLIES5eqd/ii9esxaBAKCzF+PI4exctsWEkGBgaKqqOHDx/u2bNn4MCBX3zxha2t7Y4dO86dO0dEABSNUwVQcdFcLBZfv369VLUdtyqr7MfD0BC2tpBKERcnRNzS0ldKsPX0VNxhjP0tTqNqoqg6unIFgKDVzsXFKCpCvXrKwzZt1Ji+icjf3//TxYvltWrBzw/r1qnyWISRkdGQIUNCQkI+//zzIUOGfP/99z/99JOTk1N2dra6JvxGZYvmNWrUaNq0qUQiSRCmi1hZGiUStEDNwQGXL5dv2hgZqdw7kqkVp1E1adIEtWrh0SNkZgpagaSvj4o1qgUFqFkTkZHKqiMVqjiLi4uHDRu2YMGCyEePwlavhr+/6pOtyNPTMzg42NXVVdGqXRgVu8aVnZkKEbhhQzRogNxcpKQI+uPRuDF69sTQofj1V3zzDQoK8OmnQsT9j+E0qiYiEdq0AQRvWCsS4eOPcfCg8nDrVnh64vfflb3jbGzQsSOCglBhO/vKyMvL69mz5+7du42NjQ8ePNjdy0u9sz569OjJkycvXLhw4cIFxS4XwrCysqpdu/bjx4/T09O19mhm586YOBGffSZQ3MWLMXMmZDL064cDByASCRT3v0REf9qlnb2zWbOwbBm++w5Tp8LUFPr6KCgQouA5IwNjxkBXF0VFsLTEmjWQy3H4MPbuxdGjKCoCALG4tHPnjQMH9h8woEGDBq8fLzMz09PTMy4urkGDBkeOHFE86qpeiYmJBw4cePbsWbdu3bp167Z79+7OnTu/cWJq4eHh8TgubtPmzfVMTfW//tqkZs1qf/whQFxcuwaxGHXr4tIl6OvDxQXq3qGeaY2Wl7j+Tfbupdq1af58IqJjxyg1VdDoRUV/0wapsFBZxWlkdNTJCYBYLHZxcQkICMis0HRHJpPdvHnz1q1bcrn83r17Bw8e1NfXb9myZaomP8KdO3c2b94cHx8fFRVlYGDg4uKiuVgVFc+fTwDNm0fPnpFYTPr6pMlmo6+IiCBHR1qzhn78kVq3psREgeIyDeM0qj6K6veYGPryS/LyolWr3qq9m2YVFESFhPTr18/AwEDx51NHR6dLly5r167Nz8/v2rWrj4+Pt7f39u3bDx8+nJCQcPz4cY2WwRORosHSwoULnz59KhKJDA0NS4X5doWEEEA9exIR2doSQFevChGXiJydKSFB+frwYRo6VKC4TMM4jarVzZvk4EBxcfT0Kfn706hR2p7Qn7148UJRxalo/G5oaBgTE1PxTPDMmTN3BGlXvnPnTgD9+vUjImtrawDx8fECxKX79wmgDz8kIvriCwJo3Toh4hKRmVn567w8at5coLhMw3iJSa02bMD06XBwQK1a+O47hIejim02Z2hoqKjizMjI2LZtm5+fX4sWLZ4+fTpx4sS4uDgAx44du6nyXpuVUXGhXNBt1ywtUacOnjzBgwda6OpSthRRUoKXVwbsfcdpVK1SU2FtrXwtEsHKCqmpWp3QPzIxMRkxYsSMGTP09fWjo6OdnZ3HjBnz008/CTYBGxsbExOT9PT0hw8fCr1oXpY9Be4x6OSEsDDl69BQuLoKFJdpGKdRtapXD48elR9mZkKQ1WcVGRoaDh8+fNOmTSEhIYIFFYlEWmt3rcie0dFo2xa6ukhIgDANQH/8EXPnYsIEjB6NHTswf74QQZnmvcXeDOzNhg3D7Nnw8ICxMUJCYGGB2rW1Pac3iI+P37Rpk729fWhoaK9evXJycgQL7eTkdObMmatXr06ZMkUkEqUnJcmlUvHb7BfyjhRno+npMDTEmjVo0uStNil5d02b4sIFJCdDVxdWVlzC+a/BaVStOnbEpEkYMABEsLDAjh3antCbtWzZ0tvbOykpae7cue3bt4+Ojk5KSpLL5WKxxq9UnNu187Gz+zg728TEpMDNzTg8HDduoFUrTceFhweePoVUipUr8eQJatcWbu8vHR00ayZQLCYULr9nSE9P37Rpk4GBwcyZMy0sLNLS0hITExW7AWvW3buwsUH9+nj4EJ9/jt27sWkTfHw0HhdAdja6dsX8+bC2xpYt0NVFYKAQcdm/Ed8bZcjPz/f391+3bh0EftJcW40IAPzyC/7v/zBkCJycsHo1jh/H06cChWb/OpxGGWxtbY2Nje/fv5+dnS3oorm2GhEASEpC2bP8YjE++gjJyQKFZv86nEYZxGJx69atAVy7dk3QEk68umguFiMuDiUlQsQ1NUVeXvlhbi5q1RIiLvs34jTKgArX8ooXMTExsopbKGtOWW/WGjVgY4Pi4vItzjWqTx9s2qTsYRwXh+fPywt+GXtLvFLPgAp7RH/44YeWlpYPHjy4fft2ixYtNB74T+2ub9/GtWvQQE+pP+vaFfHx6NoVhoYgws6d0HxlAvu34pV6BgC3bt2ytbW1sLB48ODBoEGD8q5eXbJiRbvBgzUemAhNmsDSEocOISMDOjpo0kTQgkoirt9kKuKzUQYAzZo1+6Fz546APCtrb7t24t9/R0QEBEijIhFSUpCfj4ULER+PBg0wdSrs7DQet+IEGFMNX8gwABCLxVOBj8+dE0dHixXb9QhWe0SE3r3h6IhDh+Dri0GDkJkpUGjG1IHTKHupbLXHyQkiEWJiXtnlSXMSEqCvDy8v6OqidWv83//ht9+EiMuYmnAaZS+VFcCbmsLKCoWFAu0RnZ4OS8vyw8aNkZYmRFzG1ITTKHupYgF8WTmnAMzMkJ5efpiWhoYNhYjLmJpwGmUv2djAxAQZGXj4UNBHM+3tkZeHU6cAIDUVmzdj6FAh4jKmJpxG2Utlj2Yq+hnr6r7ynI/miMU4dAi7dqF7d/j6YvNmWFgIEZcxNeGCJ1ZB375o1Ah16qBtW+Tnw9BQoLj16mHTJoFiMaZuXH7PXlVUhFmzlE9ktm6NRYt4yyDGXo8v6tmr/P3x4Yc4dQqnTkFPD4sXa3tCjFV1fDbKXtWsGWJiYGQEADk5cHVFQoK258RYlcZno+xVz54pcygAExPk5mp1Noy9BziNsleZm+PePeXrpCRYWWl1Noy9BziNsldNm4avvkJiIm7cwMSJmD5d2xNirKrje6PsL8LCsH8/xGIMHIjOnbU9G8aqOk6jjDGmEr6oZ4wxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlXAaZYwxlfw/LjR5GfxReGgAAAINelRYdHJka2l0UEtMIHJka2l0IDIwMjIuMDkuMQAAeJx7v2/tPQYg4GWAAEYgVgJiRSBuYORQ0ALS/5kZ+RQ0gAxmFg4GMM0Eo0eMOCwguBkYFRiZNJgYmRWYWRRYWDWYWNgU2NgV2Dk0mNg5FTi5FLi4NZi4eBR4eBV4+TSYePkV+AUUBAQ1mASEFISEFYRFNJiERRVExRTExDWYxCQUJCQVJKU0mCSlFaRlFGRkNZhk5BTk5BXkFTSY5BUVRBjEFzFCYgUMlPKrAw4GNZ2xB3Ead1sevBnRD2Y3/TI5yJe6eR+IPXGP+sF9BuwOIPbiPwIHmS0ugtWk/f57IFhBAyy+xJrnoJG+hh2InfP91QHfvgVgvQWlDw9sCH2yH8Tu/XnsgMIpXbDejZUzD2y+4wtWv+h71gHLIzPA4lu+9x9oK1sEVr/+vcuB//Pvg9lbXxse4HvpewDEvv/85/6deo5g8dsX2PdfOzAXzL7InL+v8qcOWI2inIT99IOlYDc4sjM6RBw2B6ux5hZxeMLJA1bDwOzpcHt2D9gNMYyTHERFAsDqI5tnO8w3uQZWb/N/q4MjjxfYbRL1jx3mFM0HqxdlfOtQGjoFrKZmH7Oj19NlYDVZdVKOC8uDwewMe3nHGW9UwGqianUci4+8AIt72Do7/pq1Gsx2++fuKLSTBWyvhU2o4/lAUXB4igEA+PScK+/y3JAAAAMuelRYdE1PTCByZGtpdCAyMDIyLjA5LjEAAHicfZZLjhQxDIb3fYpcYCK/4jgLFvNAgIAZCYa5A3vuL34HdapHSlHVZXWlbdcXv6ovJY8fT19//ynrkKfLpRT6z2eMUd6UiC7fS34pDx8/fXkuj6/3D9eVx5dfz68/i1pRhQ3O97r3ry/frytcXsodayUl1la4ttG79EKV5gHrx8+fPtxxebv/9oGvZlJ+/oYdV6eQ3mHHPAJoN3ZpYFcDnc/hqszcRrmjqo006MbgqmpTlaqFNu1F4FqgvNFsk2JUcvY+AOFqTeQUwtNzr0N9aMCxi4+uG8c9FaN2t+aM39lNbKcYk6BXNemkc1sIY/gpwviHQNZaNGyz9tDwtnHNlKpehfvwfLb3ONHkSWEVikypIcSk/RSCJV0rPLK6ZfKGWsTO9cybVUNyoycvQiYmO1WbFCil3lnG3Jt16nSO0dK5IARE2ANMsVe1sXM+U8d1NNGY5dObOW9V++Sg2pSa8mRWMftPOCKdI2qqhg0nEcn7JliqmT+4HibIz7RBQ/pGUygxpBI7eUxi7iF2SiHZipL1w1kZKPsIJtu5zvRpZaOgTJp08y2taDJYbSh094lLKnTeIJK9ZzUwBMIyco58BO9cZ+5a9ehGuf0eKItdOsSToVemIZ67wQRA8M4Rsvd6hVcNSQRuEW1Xm5JpC2TAiHJyKeZJ31UmihEIozp6bnZSiJOdZ0Kz80bW2sDeETU3wwTaeNbMGefuydH+XIPDxlZzTkwWZEBQuImLLfp5UWp2HvRtjNZnxaOpnHf708waZriiaJFc1JAiyLch287wj89P714F/14ODy/PT8fLIU85hn6eeox0/FYWMOYIrnbMZsHlx+hl3PZjwDKuOMao4BrHkGTc8u0o5CkWO+ejIXjBITI8VxYfJ2CKo9EtrQC6KDGCeK4sUE7SFIsVA0VS8MJFbHmuLGJOZAi57X+Z4nhnzmBm+940MqeQxYyOlSluW5LnymKWZIaQxYwekykWs2SAc2UxSzJDyGJGW0gKPQop45wri1mTGUIXs84awOeog4xzrizmrKzbOsr76x8UfL/8BdUtsbesDcDgAAABeXpUWHRTTUlMRVMgcmRraXQgMjAyMi4wOS4xAAB4nK1Sy23lQAxrZY8JYAv6jT4wUsM75BikgPSQ4kOphj15zKEoiprXx9fnz/fb1+v+fn/t8eP1/w8j/+/37RYhE9HrZrLDVtczUHDZJSTSPYARGwuA05k2CJOXnUtBUXyfu4lDosAJ86NAktqiHZzQ6ARSlOEnLiYJV7clmWtud7FTsRD7OX3dQlm2UJBK9hRGVsX0cwIgQJSFbdQNl2Ixttu8l+NdvUpwoL4kTrBW3JOnTtGGMR3u0GWHEeqj1mMrj4eMGNMxPiarZupnMTXzkYMI60QDWrvKtXfdMP0osQTvkClZ40NnXMiiTqrkeozEuXae9EAij9NBblulbMo6UGEBNQ4izbGrQ1HpU5aFAeEzSbh1osJaxwgQ57CtklOTZ8GaIyfDvtGpKc5Ez1Qa7D4I5u+a3uG+C5ZpwZNuSc2/wpaeUYF8LuAzLmpEsRr4xbsxTcULMDMA779/ngGaxTHKeoYAAAAASUVORK5CYII=",
      "text/plain": [
       "<rdkit.Chem.rdchem.Mol at 0x1a790bf20>"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Chem.MolFromSmiles(\"[O-][Si](=O)O[Si](=O)O[Si](=O)O[Si](=O)O[Si](=O)O[Si](=O)O[Si](=O)O[Si](=O)O[Si](=O)O[Si](=O)O[Si](=O)[O-]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rdkit",
   "language": "python",
   "name": "my-rdkit-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "205.479px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}