AlphaZero_from_scratch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMRlTUSAe4+wccaqy8NBqNi",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "f339370d10c24547ae7b48d7c64b4770": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_721729af097743828cbbe5c8fe3cbec9",
              "IPY_MODEL_ed6ed5a9e2f346d29f0b91e1caaa6432",
              "IPY_MODEL_d424f5a2f4654239a208360efec49ed6"
            ],
            "layout": "IPY_MODEL_9ad40cc2518a4cf19f31dfba601b5c47"
          }
        },
        "721729af097743828cbbe5c8fe3cbec9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cf1bfc5677f145189a9312b358a36e55",
            "placeholder": "​",
            "style": "IPY_MODEL_36dbae7e8fcf41f498cfedbe43915855",
            "value": "100%"
          }
        },
        "ed6ed5a9e2f346d29f0b91e1caaa6432": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b989ee665f904cdab11b8d38294ae21d",
            "max": 4,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_06f26a63b64e4afd9b954178b7d4e32f",
            "value": 4
          }
        },
        "d424f5a2f4654239a208360efec49ed6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8a42fc0e854048ae855e51fcbaa21146",
            "placeholder": "​",
            "style": "IPY_MODEL_4f66ebfca97147dba6aadabf25bd963d",
            "value": " 4/4 [25:45&lt;00:00, 388.38s/it]"
          }
        },
        "9ad40cc2518a4cf19f31dfba601b5c47": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cf1bfc5677f145189a9312b358a36e55": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "36dbae7e8fcf41f498cfedbe43915855": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b989ee665f904cdab11b8d38294ae21d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "06f26a63b64e4afd9b954178b7d4e32f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8a42fc0e854048ae855e51fcbaa21146": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4f66ebfca97147dba6aadabf25bd963d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c77bd5fedd4144efa0cc0bf40d83cc43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1efdbed767c64f87a70e2b9fb3206827",
              "IPY_MODEL_69f4c603fa874f9cb7fc53ccd02b1070",
              "IPY_MODEL_866e2890710643948a87b1d73a4a0e6a"
            ],
            "layout": "IPY_MODEL_d4faf057554e4f12bc24f832507d013b"
          }
        },
        "1efdbed767c64f87a70e2b9fb3206827": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d64c6d8938c742809941a430f3543d04",
            "placeholder": "​",
            "style": "IPY_MODEL_71c3a716513944099dd1e23eb4ed35eb",
            "value": "100%"
          }
        },
        "69f4c603fa874f9cb7fc53ccd02b1070": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7008e96889a147cd9315b497a644db90",
            "max": 4,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b1e39f60ab654a1dba28e66dc6d1d0d8",
            "value": 4
          }
        },
        "866e2890710643948a87b1d73a4a0e6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_78768ad755fa44a0a843f16551cba90e",
            "placeholder": "​",
            "style": "IPY_MODEL_b4984c3fd55c48ce98b0c7a67cd02438",
            "value": " 4/4 [00:04&lt;00:00,  1.10s/it]"
          }
        },
        "d4faf057554e4f12bc24f832507d013b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d64c6d8938c742809941a430f3543d04": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "71c3a716513944099dd1e23eb4ed35eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7008e96889a147cd9315b497a644db90": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b1e39f60ab654a1dba28e66dc6d1d0d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "78768ad755fa44a0a843f16551cba90e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b4984c3fd55c48ce98b0c7a67cd02438": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9d2fba8fda324f7fb96be7f376cd36c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_bcf9a7bdef4944a8b4b553b162ee8449",
              "IPY_MODEL_7fa757a00fc74013b70b0137f5160894",
              "IPY_MODEL_a90e3f85cebc4bdc88a8e2cabcf7246a"
            ],
            "layout": "IPY_MODEL_35d2c7f272dc46d4a2cd1ddfc00d9455"
          }
        },
        "bcf9a7bdef4944a8b4b553b162ee8449": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b0d42f07ac944288b57258b3232fbb34",
            "placeholder": "​",
            "style": "IPY_MODEL_483692bc8ce846da9b7523c65d7d51bd",
            "value": " 75%"
          }
        },
        "7fa757a00fc74013b70b0137f5160894": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bbb8af06c7c24fd1982fae7082697825",
            "max": 4,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e6f86aa408f943b59fe852003809adc5",
            "value": 3
          }
        },
        "a90e3f85cebc4bdc88a8e2cabcf7246a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a459fd5e10254e9c8f47116572877f84",
            "placeholder": "​",
            "style": "IPY_MODEL_55b57684c23c4dbc96d1a5d96a162716",
            "value": " 3/4 [18:09&lt;06:04, 364.23s/it]"
          }
        },
        "35d2c7f272dc46d4a2cd1ddfc00d9455": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b0d42f07ac944288b57258b3232fbb34": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "483692bc8ce846da9b7523c65d7d51bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bbb8af06c7c24fd1982fae7082697825": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e6f86aa408f943b59fe852003809adc5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a459fd5e10254e9c8f47116572877f84": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "55b57684c23c4dbc96d1a5d96a162716": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/JeanMichelDeveloper/Training/blob/main/AlphaZero_from_scratch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Create a tic-tic-toe game"
      ],
      "metadata": {
        "id": "z2OzFkBvi0Op"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install kaggle-environments"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vinzDyr_PdPf",
        "outputId": "17f37c6e-7801-43c8-e869-da9d1cbbe932"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting kaggle-environments\n",
            "  Downloading kaggle_environments-1.13.0-py2.py3-none-any.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m18.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting gym==0.21.0\n",
            "  Downloading gym-0.21.0.tar.gz (1.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m41.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: requests>=2.25.1 in /usr/local/lib/python3.9/dist-packages (from kaggle-environments) (2.25.1)\n",
            "Requirement already satisfied: Flask>=1.1.2 in /usr/local/lib/python3.9/dist-packages (from kaggle-environments) (2.2.3)\n",
            "Collecting stable-baselines3==1.7.0\n",
            "  Downloading stable_baselines3-1.7.0-py3-none-any.whl (171 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m171.8/171.8 KB\u001b[0m \u001b[31m17.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.19.5 in /usr/local/lib/python3.9/dist-packages (from kaggle-environments) (1.22.4)\n",
            "Collecting kaggle-environments\n",
            "  Downloading kaggle_environments-1.12.0-py2.py3-none-any.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m37.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: jsonschema>=3.0.1 in /usr/local/lib/python3.9/dist-packages (from kaggle-environments) (4.3.3)\n",
            "Requirement already satisfied: Werkzeug>=2.2.2 in /usr/local/lib/python3.9/dist-packages (from Flask>=1.1.2->kaggle-environments) (2.2.3)\n",
            "Requirement already satisfied: click>=8.0 in /usr/local/lib/python3.9/dist-packages (from Flask>=1.1.2->kaggle-environments) (8.1.3)\n",
            "Requirement already satisfied: Jinja2>=3.0 in /usr/local/lib/python3.9/dist-packages (from Flask>=1.1.2->kaggle-environments) (3.1.2)\n",
            "Requirement already satisfied: importlib-metadata>=3.6.0 in /usr/local/lib/python3.9/dist-packages (from Flask>=1.1.2->kaggle-environments) (6.0.0)\n",
            "Requirement already satisfied: itsdangerous>=2.0 in /usr/local/lib/python3.9/dist-packages (from Flask>=1.1.2->kaggle-environments) (2.1.2)\n",
            "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.9/dist-packages (from jsonschema>=3.0.1->kaggle-environments) (0.19.3)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.9/dist-packages (from jsonschema>=3.0.1->kaggle-environments) (22.2.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.25.1->kaggle-environments) (1.26.14)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests>=2.25.1->kaggle-environments) (2.10)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.9/dist-packages (from requests>=2.25.1->kaggle-environments) (4.0.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests>=2.25.1->kaggle-environments) (2022.12.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.9/dist-packages (from importlib-metadata>=3.6.0->Flask>=1.1.2->kaggle-environments) (3.15.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.9/dist-packages (from Jinja2>=3.0->Flask>=1.1.2->kaggle-environments) (2.1.2)\n",
            "Installing collected packages: kaggle-environments\n",
            "Successfully installed kaggle-environments-1.12.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fxy1l-jShVGA",
        "outputId": "bb8ec507-9e9c-4565-bd88-9bae4f515793"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading environment lux_ai_s2 failed: No module named 'vec_noise'\n"
          ]
        }
      ],
      "source": [
        "#import necessary libraries\n",
        "import numpy as np\n",
        "import math\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "torch.manual_seed(0)\n",
        "\n",
        "from tqdm.notebook import trange\n",
        "import random\n",
        "import kaggle_environments\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#define game and rules\n",
        "class TicTacToe:\n",
        "  #init game\n",
        "  def __init__(self):\n",
        "    self.row_count = 3\n",
        "    self.column_count = 3\n",
        "    self.action_size = self.row_count * self.column_count\n",
        "\n",
        "  def __repr__(self):\n",
        "    return \"TicTacToe\"\n",
        "\n",
        "  def get_initial_state(self):\n",
        "    return np.zeros((self.row_count, self.column_count))\n",
        "\n",
        "  #update game with next move\n",
        "  def get_next_state(self, state, action, player):\n",
        "    row = action // self.column_count\n",
        "    column = action % self.column_count\n",
        "    state[row, column] = player\n",
        "    return state\n",
        "\n",
        "  #list valid moves\n",
        "  def get_valid_moves(self, state):\n",
        "    return (state.reshape(-1) == 0).astype(np.uint8)\n",
        "\n",
        "  #check if game is won\n",
        "  def check_win(self, state, action):\n",
        "    if action == None:\n",
        "      return False\n",
        "\n",
        "    row = action // self.column_count\n",
        "    column = action % self.column_count\n",
        "    player = state[row, column]\n",
        "\n",
        "    return (\n",
        "        np.sum(state[row, :]) == player * self.column_count\n",
        "        or np.sum(state[:, column]) == player * self.row_count\n",
        "        or np.sum(np.diag(state)) == player * self.row_count\n",
        "        or np.sum((np.diag(np.flip(state, axis = 0)))) == player * self.row_count)\n",
        "\n",
        "  #check if game is terminated\n",
        "  def get_value_and_terminated(self, state, action):\n",
        "    if self.check_win(state, action):\n",
        "      return 1, True\n",
        "    if np.sum(self.get_valid_moves(state)) == 0:\n",
        "      return 0, True\n",
        "    return 0, False\n",
        "\n",
        "  #change player\n",
        "  def get_opponent(self,player):\n",
        "    return -player\n",
        "  \n",
        "  #get opponent last play value (negative because it's the opponent play) \n",
        "  def get_opponent_value(self, value):\n",
        "    return -value\n",
        "  \n",
        "  #simulate move from player perspective\n",
        "  def change_perspective(self, state, player):\n",
        "    return state * player\n",
        "\n",
        "  #encode state for model\n",
        "  def get_encoded_state(self, state):\n",
        "    encoded_state = np.stack(\n",
        "        (state == -1, state == 0, state == 1)\n",
        "    ).astype(np.float32)\n",
        "\n",
        "    if len(state.shape) == 3:\n",
        "      encoded_state = np.swapaxes(encoded_state, 0, 1)\n",
        "\n",
        "    return encoded_state"
      ],
      "metadata": {
        "id": "gG1uP5zGjDsH"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#define game and rules\n",
        "class ConnectFour:\n",
        "  #init game\n",
        "  def __init__(self):\n",
        "    self.row_count = 6\n",
        "    self.column_count = 7\n",
        "    self.action_size = self.column_count\n",
        "    self.in_a_row = 4\n",
        "\n",
        "  def __repr__(self):\n",
        "    return \"ConnectFour\"\n",
        "\n",
        "  def get_initial_state(self):\n",
        "    return np.zeros((self.row_count, self.column_count))\n",
        "\n",
        "  #update game with next move\n",
        "  def get_next_state(self, state, action, player):\n",
        "    row = np.max(np.where(state[:, action] == 0))\n",
        "    state[row, action] = player\n",
        "    return state\n",
        "\n",
        "  #list valid moves\n",
        "  def get_valid_moves(self, state):\n",
        "    return (state[0] == 0).astype(np.uint8)\n",
        "\n",
        "  #check if game is won\n",
        "  def check_win(self, state, action):\n",
        "    if action == None:\n",
        "      return False\n",
        "\n",
        "    row = np.min(np.where(state[:, action] != 0))\n",
        "    column = action\n",
        "    player = state[row][column]\n",
        "\n",
        "    def count(offset_row, offset_column):\n",
        "      for i in range(1, self.in_a_row):\n",
        "        r = row + offset_row * i\n",
        "        c = action + offset_column * i\n",
        "        \n",
        "        if (\n",
        "            r < 0\n",
        "            or r >= self.row_count\n",
        "            or c < 0\n",
        "            or c >= self.column_count\n",
        "            or state[r][c] != player\n",
        "         ):\n",
        "\n",
        "          return i -1\n",
        "\n",
        "      return self.in_a_row - 1\n",
        "\n",
        "    return (\n",
        "        count(1, 0) >= self.in_a_row - 1 #vertical\n",
        "        or (count(0, 1) + count(0, -1)) >= self.in_a_row - 1 #horizontal\n",
        "        or (count(1, 1) + count(-1, -1)) >= self.in_a_row - 1 #top left diagonal\n",
        "        or (count(1, -1) + count(-1, 1)) >= self.in_a_row - 1 # top right diagonal\n",
        "        )  \n",
        "\n",
        "\n",
        "  #check if game is terminated\n",
        "  def get_value_and_terminated(self, state, action):\n",
        "    if self.check_win(state, action):\n",
        "      return 1, True\n",
        "    if np.sum(self.get_valid_moves(state)) == 0:\n",
        "      return 0, True\n",
        "    return 0, False\n",
        "\n",
        "  #change player\n",
        "  def get_opponent(self,player):\n",
        "    return -player\n",
        "  \n",
        "  #get opponent last play value (negative because it's the opponent play) \n",
        "  def get_opponent_value(self, value):\n",
        "    return -value\n",
        "  \n",
        "  #simulate move from player perspective\n",
        "  def change_perspective(self, state, player):\n",
        "    return state * player\n",
        "\n",
        "  #encode state for model\n",
        "  def get_encoded_state(self, state):\n",
        "    encoded_state = np.stack(\n",
        "        (state == -1, state == 0, state == 1)\n",
        "    ).astype(np.float32)\n",
        "\n",
        "    if len(state.shape) == 3:\n",
        "      encoded_state = np.swapaxes(encoded_state, 0, 1)\n",
        "    \n",
        "    return encoded_state"
      ],
      "metadata": {
        "id": "cve9peNtRfEC"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#create a tictactoe game\n",
        "tictactoe = TicTacToe()\n",
        "player = 1\n",
        "\n",
        "#initialise game\n",
        "state = tictactoe.get_initial_state()\n",
        "\n",
        "while True:\n",
        "  #get list of valid moves\n",
        "  print(state)\n",
        "  valid_moves = tictactoe.get_valid_moves(state)\n",
        "  print(\"valid_moves\", [i for i in range(tictactoe.action_size) if valid_moves[i] == 1])\n",
        "\n",
        "  #player choose next move\n",
        "  action = int(input(f\"{player}:\"))\n",
        "\n",
        "  #check if move is valid\n",
        "  if valid_moves[action] == 0:\n",
        "    print(\"action not valid\")\n",
        "    continue\n",
        "  \n",
        "  #update game\n",
        "  state = tictactoe.get_next_state(state, action, player)\n",
        "\n",
        "  value, is_terminal = tictactoe.get_value_and_terminated(state, action)\n",
        "\n",
        "  #check if game is terminated\n",
        "  if is_terminal:\n",
        "    print(state)\n",
        "    if value == 1:\n",
        "      print(\"player\", player, \"won\")\n",
        "    else:\n",
        "      print(\"draw\")\n",
        "    break\n",
        "  \n",
        "  #change player\n",
        "  player = tictactoe.get_opponent(player)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 450
        },
        "id": "3CeviJVhmG5Z",
        "outputId": "6acaacb3-5570-4494-ba77-79c395bb1902"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[0. 0. 0.]\n",
            " [0. 0. 0.]\n",
            " [0. 0. 0.]]\n",
            "valid_moves [0, 1, 2, 3, 4, 5, 6, 7, 8]\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-9-9f086174b20a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m   \u001b[0;31m#player choose next move\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m   \u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{player}:\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m   \u001b[0;31m#check if move is valid\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/ipykernel/kernelbase.py\u001b[0m in \u001b[0;36mraw_input\u001b[0;34m(self, prompt)\u001b[0m\n\u001b[1;32m    858\u001b[0m                 \u001b[0;34m\"raw_input was called, but this frontend does not support input requests.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    859\u001b[0m             )\n\u001b[0;32m--> 860\u001b[0;31m         return self._input_request(str(prompt),\n\u001b[0m\u001b[1;32m    861\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent_ident\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    862\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent_header\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/ipykernel/kernelbase.py\u001b[0m in \u001b[0;36m_input_request\u001b[0;34m(self, prompt, ident, parent, password)\u001b[0m\n\u001b[1;32m    902\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    903\u001b[0m                 \u001b[0;31m# re-raise KeyboardInterrupt, to truncate traceback\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 904\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Interrupted by user\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    905\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    906\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwarning\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Invalid Message:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_info\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: Interrupted by user"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Create Monte Carlo Tree Search"
      ],
      "metadata": {
        "id": "NI09MTXdpVq8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#create model\n",
        "class ResNet(nn.Module):\n",
        "  def __init__(self, game, num_resBlocks, num_hidden, device):\n",
        "    super().__init__()\n",
        "\n",
        "    self.device = device\n",
        "\n",
        "    self.startBlock = nn.Sequential(\n",
        "        nn.Conv2d(3, num_hidden, kernel_size=3, padding=1),\n",
        "        nn.BatchNorm2d(num_hidden),\n",
        "        nn.ReLU()\n",
        "    )\n",
        "\n",
        "    self.backBone = nn.ModuleList(\n",
        "        [ResBlock(num_hidden) for i in range(num_resBlocks)])\n",
        "    \n",
        "    self.policyHead = nn.Sequential(\n",
        "        nn.Conv2d(num_hidden, 32, kernel_size=3, padding=1),\n",
        "        nn.BatchNorm2d(32),\n",
        "        nn.ReLU(),\n",
        "        nn.Flatten(),\n",
        "        nn.Linear(32 * game.row_count * game.column_count, game.action_size)\n",
        "    )\n",
        "\n",
        "    self.valueHead = nn.Sequential(\n",
        "        nn.Conv2d(num_hidden, 3, kernel_size=3, padding=1),\n",
        "        nn.BatchNorm2d(3),\n",
        "        nn.ReLU(),\n",
        "        nn.Flatten(),\n",
        "        nn.Linear(3 * game.row_count * game.column_count, 1),\n",
        "        nn.Tanh()\n",
        "    )\n",
        "\n",
        "    self.to(device)\n",
        "\n",
        "  def forward(self, x):\n",
        "    x = self.startBlock(x)\n",
        "\n",
        "    for resBlock in self.backBone:\n",
        "      x = resBlock(x)\n",
        "\n",
        "    policy = self.policyHead(x)\n",
        "    value = self.valueHead(x)\n",
        "\n",
        "    return policy, value\n",
        "\n",
        "\n",
        "\n",
        "class ResBlock(nn.Module):\n",
        "  def __init__(self, num_hidden):\n",
        "    super().__init__()\n",
        "    self.conv1 = nn.Conv2d(num_hidden, num_hidden, kernel_size=3, padding=1)\n",
        "    self.bn1 = nn.BatchNorm2d(num_hidden)\n",
        "    self.conv2 = nn.Conv2d(num_hidden, num_hidden, kernel_size=3, padding=1)\n",
        "    self.bn2 = nn.BatchNorm2d(num_hidden)\n",
        "\n",
        "  def forward(self, x):\n",
        "    residual = x\n",
        "    x = F.relu(self.bn1(self.conv1(x)))\n",
        "    x = self.bn2(self.conv2(x))\n",
        "    x += residual\n",
        "    x = F.relu(x)\n",
        "    return x"
      ],
      "metadata": {
        "id": "xrX2Ant9btgc"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#test\n",
        "tictactoe = TicTacToe()\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "state = tictactoe.get_initial_state()\n",
        "state = tictactoe.get_next_state(state, 2, -1)\n",
        "state = tictactoe.get_next_state(state, 4, -1)\n",
        "state = tictactoe.get_next_state(state, 8, 1)\n",
        "state = tictactoe.get_next_state(state, 6, 1)\n",
        "\n",
        "encoded_state = tictactoe.get_encoded_state(state)\n",
        "\n",
        "tensor_state = torch.tensor(encoded_state, device = device).unsqueeze(0)\n",
        "\n",
        "model = ResNet(tictactoe, 4, 64, device = device)\n",
        "model.load_state_dict(torch.load('model_2.pt', map_location = device))\n",
        "model.eval()\n",
        "\n",
        "policy, value = model(tensor_state)\n",
        "\n",
        "value = value.item()\n",
        "policy = torch.softmax(policy, axis=1).squeeze(0).detach().cpu().numpy()\n",
        "\n",
        "print(value)\n",
        "print(state)\n",
        "print(tensor_state)\n",
        "\n",
        "plt.bar(range(tictactoe.action_size), policy)\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 380
        },
        "id": "RDd_RMWrf3Cj",
        "outputId": "d9ef3eb5-f1d3-4f90-e7f4-df3bcfe77f87"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "error",
          "ename": "FileNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-11-758d83b6bb3a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mResNet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtictactoe\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m64\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_state_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'model_2.pt'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmap_location\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(f, map_location, pickle_module, weights_only, **pickle_load_args)\u001b[0m\n\u001b[1;32m    769\u001b[0m         \u001b[0mpickle_load_args\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'encoding'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'utf-8'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    770\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 771\u001b[0;31m     \u001b[0;32mwith\u001b[0m \u001b[0m_open_file_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mopened_file\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    772\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0m_is_zipfile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopened_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    773\u001b[0m             \u001b[0;31m# The zipfile reader is going to advance the current file position.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36m_open_file_like\u001b[0;34m(name_or_buffer, mode)\u001b[0m\n\u001b[1;32m    268\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_open_file_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    269\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0m_is_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 270\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_open_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    271\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    272\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m'w'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, name, mode)\u001b[0m\n\u001b[1;32m    249\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0m_open_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_opener\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    250\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 251\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_open_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    252\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    253\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__exit__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'model_2.pt'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#create a class to define nodes\n",
        "class Node:\n",
        "  def __init__(self, game, args, state, parent = None, action_taken = None, prior=0, visit_count = 0):\n",
        "    self.game = game\n",
        "    self.args = args\n",
        "    self.state = state\n",
        "    self.parent = parent\n",
        "    self.action_taken = action_taken\n",
        "    self.prior = prior\n",
        "\n",
        "    self.children = []\n",
        "    #self.expandable_moves = game.get_valid_moves(state)\n",
        "\n",
        "    self.visit_count = visit_count\n",
        "    self.value_sum = 0\n",
        "  \n",
        "  #check if node is fully expanded\n",
        "  def is_fully_expanded(self):\n",
        "    return len(self.children) > 0\n",
        "\n",
        "  #select best child of node\n",
        "  def select(self):\n",
        "    best_child = None\n",
        "    best_ucb = -np.inf\n",
        "\n",
        "    for child in self.children:\n",
        "      ucb = self.get_ucb(child)\n",
        "      \n",
        "      if ucb > best_ucb:\n",
        "        best_child = child\n",
        "        best_ucb = ucb\n",
        "    \n",
        "    return best_child\n",
        "\n",
        "  #calculate ucb\n",
        "  def get_ucb(self, child):\n",
        "    if child.visit_count == 0:\n",
        "      q_value = 0\n",
        "    else:  \n",
        "      q_value = 1 - ((child.value_sum / child.visit_count) + 1) / 2\n",
        "    return q_value + self.args['C'] * (math.sqrt(self.visit_count) / (child.visit_count + 1)) * child.prior\n",
        "\n",
        "  #expand node\n",
        "  def expand(self, policy):\n",
        "    for action, prob in enumerate(policy):\n",
        "      if prob > 0:\n",
        "\n",
        "        child_state = self.state.copy()\n",
        "        child_state = self.game.get_next_state(child_state, action, 1)\n",
        "        child_state = self.game.change_perspective(child_state, player = -1)\n",
        "\n",
        "        child = Node(self.game, self.args, child_state, self, action, prob)\n",
        "        self.children.append(child)\n",
        "\n",
        "    #return child\n",
        "\n",
        "  #simulate game\n",
        "  #def simulate(self):\n",
        "   # value, is_terminal = self.game.get_value_and_terminated(self.state, self.action_taken)\n",
        "    #value = self.game.get_opponent_value(value)\n",
        "\n",
        "    #if is_terminal:\n",
        "     # return value\n",
        "\n",
        "    #rollout_state = self.state.copy()\n",
        "    #rollout_player = 1\n",
        "\n",
        "    #while True:\n",
        "     # valid_moves = self.game.get_valid_moves(rollout_state)\n",
        "     # action = np.random.choice(np.where(valid_moves == 1)[0])\n",
        "     # rollout_state = self.game.get_next_state(rollout_state, action, rollout_player)\n",
        "     # value, is_terminal = self.game.get_value_and_terminated(rollout_state, action)\n",
        "\n",
        "     # if is_terminal:\n",
        "      #  if rollout_player == -1:\n",
        "       #   value = self.game.get_opponent_value(value)\n",
        "       # return value\n",
        "      \n",
        "     # rollout_player = self.game.get_opponent(rollout_player)\n",
        "\n",
        "  #backpropagate info up the tree\n",
        "  def backpropagate(self, value):\n",
        "    self.value_sum += value\n",
        "    self.visit_count += 1\n",
        "\n",
        "    value = self.game.get_opponent_value(value)\n",
        "\n",
        "    if self.parent is not None:\n",
        "      self.parent.backpropagate(value)"
      ],
      "metadata": {
        "id": "YgjWXGAopRZ5"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#create a class for MCTS\n",
        "class MCTS:\n",
        "  #initialize MCTS\n",
        "  def __init__(self, game, args, model):\n",
        "    self.game = game\n",
        "    self.args = args\n",
        "    self.model = model\n",
        "\n",
        "  @torch.no_grad()\n",
        "\n",
        "  def search(self, state):\n",
        "    #define root\n",
        "    root = Node(self.game, self.args, state, visit_count = 1)\n",
        "\n",
        "    policy, _ = self.model(torch.tensor(self.game.get_encoded_state(state), device = self.model.device).unsqueeze(0))\n",
        "\n",
        "    policy = torch.softmax(policy, axis=1).squeeze(0).cpu().numpy()\n",
        "\n",
        "    policy = (1 - self.args['dirichlet_epsilon']) * policy + self.args['dirichlet_epsilon'] \\\n",
        "      * np.random.dirichlet([self.args['dirichlet_alpha']] * self.game.action_size)\n",
        "\n",
        "    valid_moves = self.game.get_valid_moves(state)\n",
        "\n",
        "    policy *= valid_moves\n",
        "\n",
        "    policy /= np.sum(policy)\n",
        "\n",
        "    root.expand(policy)\n",
        "\n",
        "    for search in range(self.args['num_searches']):\n",
        "      node = root\n",
        "      #selection\n",
        "      while node.is_fully_expanded():\n",
        "        node = node.select()\n",
        "\n",
        "      value, is_terminal = self.game.get_value_and_terminated(node.state, node.action_taken)\n",
        "\n",
        "      value = self.game.get_opponent_value(value)\n",
        "\n",
        "      if not is_terminal:\n",
        "\n",
        "        policy, value = self.model(\n",
        "            torch.tensor(self.game.get_encoded_state(node.state), device = self.model.device).unsqueeze(0)\n",
        "        )\n",
        "        policy = torch.softmax(policy, axis=1).squeeze(0).cpu().numpy()\n",
        "\n",
        "        valid_moves = self.game.get_valid_moves(node.state)\n",
        "\n",
        "        policy *= valid_moves\n",
        "        policy /= np.sum(policy)\n",
        "\n",
        "        value = value.item()\n",
        "\n",
        "        #expansion\n",
        "        node.expand(policy)\n",
        "\n",
        "        #simulation\n",
        "        #value = node.simulate()\n",
        "\n",
        "      #backpropagation\n",
        "      node.backpropagate(value)\n",
        "    \n",
        "    #return visit_counts\n",
        "    action_probs = np.zeros(self.game.action_size)\n",
        "\n",
        "    for child in root.children:\n",
        "      action_probs[child.action_taken] = child.visit_count\n",
        "\n",
        "    action_probs /= np.sum(action_probs)\n",
        "\n",
        "    return action_probs"
      ],
      "metadata": {
        "id": "Z3kTJ6sWAC0B"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Create AlphaZero"
      ],
      "metadata": {
        "id": "CfZHA099A0nW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class AlphaZero:\n",
        "  def __init__(self, model, optimizer, game, args):\n",
        "    self.model = model\n",
        "    self.optimizer = optimizer\n",
        "    self.game = game\n",
        "    self.args =args\n",
        "    self.mcts = MCTS(game, args, model)\n",
        "  \n",
        "  #Make the model play against itself\n",
        "  def selfPlay(self):\n",
        "    memory = []\n",
        "    player = 1\n",
        "    state = self.game.get_initial_state()\n",
        "\n",
        "    while True:\n",
        "      neutral_state = self.game.change_perspective(state, player)\n",
        "      action_probs = self.mcts.search(neutral_state)\n",
        "\n",
        "      memory.append((neutral_state, action_probs, player))\n",
        "\n",
        "      temperature_action_probs = action_probs ** (1 / self.args['temperature'])\n",
        "\n",
        "      action = np.random.choice(self.game.action_size, p = temperature_action_probs)\n",
        "\n",
        "      state = self.game.get_next_state(state, action, player)\n",
        "\n",
        "      value, is_terminal = self.game.get_value_and_terminated(state, action)\n",
        "\n",
        "      if is_terminal:\n",
        "        returnMemory = []\n",
        "\n",
        "        for hist_neutral_state, hist_action_probs, hist_player in memory:\n",
        "          hist_outcome = value if hist_player == player else self.game.get_opponent_value(value)\n",
        "\n",
        "          returnMemory.append((\n",
        "              self.game.get_encoded_state(hist_neutral_state),\n",
        "              hist_action_probs,\n",
        "              hist_outcome\n",
        "          ))\n",
        "\n",
        "        return returnMemory\n",
        "\n",
        "    player = self.game.get_opponent(player)\n",
        "\n",
        "  #train model\n",
        "  def train(self, memory):\n",
        "    random.shuffle(memory)\n",
        "\n",
        "    for batchIdx in range(0, len(memory), self.args['batch_size']):\n",
        "      sample = memory[batchIdx:min(len(memory) - 1, batchIdx + self.args['batch_size'])]\n",
        "      state, policy_targets, value_targets = zip(*sample)\n",
        "\n",
        "      state, policy_targets, value_targets = np.array(state), np.array(policy_targets), np.array(value_targets).reshape(-1, 1)\n",
        "\n",
        "      state = torch.tensor(state, dtype = torch.float32, device = self.model.device)\n",
        "      policy_targets = torch.tensor(policy_targets, dtype = torch.float32, device = self.model.device)\n",
        "      value_targets = torch.tensor(value_targets, dtype = torch.float32, device = self.model.device)\n",
        "\n",
        "      out_policy, out_value = self.model(state)\n",
        "\n",
        "      policy_loss = F.cross_entropy(out_policy, policy_targets)\n",
        "      value_loss = F.mse_loss(out_value, value_targets)\n",
        "      loss = policy_loss + value_loss\n",
        "\n",
        "      self.optimizer.zero_grad()\n",
        "      loss.backward()\n",
        "      self.optimizer.step()\n",
        "\n",
        "  #make model learn continuously\n",
        "  def learn(self):\n",
        "    for iteration in range(self.args['num_iterations']):\n",
        "      memory = []\n",
        "\n",
        "      self.model.eval()\n",
        "\n",
        "      for selfPlay_iteration in trange(self.args['num_selfPlay_iterations']):\n",
        "        memory += self.selfPlay()\n",
        "\n",
        "      self.model.train()\n",
        "\n",
        "      for epoch in trange(self.args['num_epochs']):\n",
        "        self.train(memory)\n",
        "\n",
        "      torch.save(self.model.state_dict(), f\"model_{iteration}_{self.game}.pt\")\n",
        "      torch.save(self.optimizer.state_dict(), f\"optimizer_{iteration}_{self.game}.pt\")"
      ],
      "metadata": {
        "id": "NagWXCRfAwvS"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#MCTS with parallelization\n",
        "class MCTSParallel:\n",
        "  #initialize MCTS\n",
        "  def __init__(self, game, args, model):\n",
        "    self.game = game\n",
        "    self.args = args\n",
        "    self.model = model\n",
        "\n",
        "  @torch.no_grad()\n",
        "\n",
        "  def search(self, states, spGames):\n",
        "    #define root\n",
        "    policy, _ = self.model(torch.tensor(self.game.get_encoded_state(states), device = self.model.device))\n",
        "\n",
        "    policy = torch.softmax(policy, axis=1).cpu().numpy()\n",
        "\n",
        "    policy = (1 - self.args['dirichlet_epsilon']) * policy + self.args['dirichlet_epsilon'] \\\n",
        "      * np.random.dirichlet([self.args['dirichlet_alpha']] * self.game.action_size, size = policy.shape[0])\n",
        "\n",
        "    for i, spg in enumerate(spGames):\n",
        "      spg_policy = policy[i]\n",
        "\n",
        "      valid_moves = self.game.get_valid_moves(states[i])\n",
        "\n",
        "      spg_policy *= valid_moves\n",
        "\n",
        "      spg_policy /= np.sum(spg_policy)\n",
        "\n",
        "      spg.root = Node(self.game, self.args, states[i], visit_count = 1)\n",
        "\n",
        "      spg.root.expand(spg_policy)\n",
        "\n",
        "\n",
        "    for search in range(self.args['num_searches']):\n",
        "      for spg in spGames:\n",
        "        spg.node = None\n",
        "        node = spg.root\n",
        "        #selection\n",
        "        while node.is_fully_expanded():\n",
        "          node = node.select()\n",
        "\n",
        "        value, is_terminal = self.game.get_value_and_terminated(node.state, node.action_taken)\n",
        "\n",
        "        value = self.game.get_opponent_value(value)\n",
        "\n",
        "        if is_terminal:\n",
        "          node.backpropagate(value)\n",
        "\n",
        "        else:\n",
        "          spg.node = node\n",
        "\n",
        "      expandable_spGames = [mappingIdx for mappingIdx in range(len(spGames)) if spGames[mappingIdx].node is not None]\n",
        "\n",
        "      if len(expandable_spGames) > 0:\n",
        "        states = np.stack([spGames[mappingIdx].node.state for mappingIdx in expandable_spGames])\n",
        "\n",
        "        policy, value = self.model(\n",
        "                torch.tensor(self.game.get_encoded_state(states), device = self.model.device)\n",
        "            )\n",
        "        policy = torch.softmax(policy, axis=1).cpu().numpy()\n",
        "\n",
        "        value = value.cpu().numpy()\n",
        "      \n",
        "      for i, mappingIdx in enumerate(expandable_spGames):\n",
        "        node = spGames[mappingIdx].node\n",
        "        spg_policy, spg_value = policy[i], value[i]\n",
        "\n",
        "        valid_moves = self.game.get_valid_moves(node.state)\n",
        "\n",
        "        spg_policy *= valid_moves\n",
        "        spg_policy /= np.sum(spg_policy)\n",
        "\n",
        "        #expansion\n",
        "        node.expand(spg_policy)\n",
        "\n",
        "        #backpropagation\n",
        "        node.backpropagate(spg_value)"
      ],
      "metadata": {
        "id": "3YeIHcYqLaFy"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#AlphaZero with parallelization\n",
        "class AlphaZeroParallel:\n",
        "  def __init__(self, model, optimizer, game, args):\n",
        "    self.model = model\n",
        "    self.optimizer = optimizer\n",
        "    self.game = game\n",
        "    self.args =args\n",
        "    self.mcts = MCTSParallel(game, args, model)\n",
        "  \n",
        "  #Make the model play against itself\n",
        "  def selfPlay(self):\n",
        "    return_memory = []\n",
        "    player = 1\n",
        "    spGames = [SPG(self.game) for spg in range(self.args['num_parallel_games'])]\n",
        "\n",
        "    while len(spGames) > 0:\n",
        "      states = np.stack([spg.state for spg in spGames])\n",
        "\n",
        "      neutral_states = self.game.change_perspective(states, player)\n",
        "\n",
        "      self.mcts.search(neutral_states, spGames)\n",
        "\n",
        "      for i in range(len(spGames))[::-1]:\n",
        "        spg = spGames[i]\n",
        "\n",
        "        action_probs = np.zeros(self.game.action_size)\n",
        "\n",
        "        for child in spg.root.children:\n",
        "          action_probs[child.action_taken] = child.visit_count\n",
        "\n",
        "        action_probs /= np.sum(action_probs)\n",
        "\n",
        "\n",
        "        spg.memory.append((spg.root.state, action_probs, player))\n",
        "\n",
        "        temperature_action_probs = action_probs ** (1 / self.args['temperature'])\n",
        "\n",
        "        action = np.random.choice(self.game.action_size, p = action_probs)\n",
        "\n",
        "        spg.state = self.game.get_next_state(spg.state, action, player)\n",
        "\n",
        "        value, is_terminal = self.game.get_value_and_terminated(spg.state, action)\n",
        "\n",
        "        if is_terminal:\n",
        "\n",
        "          for hist_neutral_state, hist_action_probs, hist_player in spg.memory:\n",
        "            hist_outcome = value if hist_player == player else self.game.get_opponent_value(value)\n",
        "\n",
        "            return_memory.append((\n",
        "                self.game.get_encoded_state(hist_neutral_state),\n",
        "                hist_action_probs,\n",
        "                hist_outcome\n",
        "            ))\n",
        "          \n",
        "          del spGames[i]\n",
        "\n",
        "      player = self.game.get_opponent(player)\n",
        "\n",
        "    return return_memory\n",
        "\n",
        "  #train model\n",
        "  def train(self, memory):\n",
        "    random.shuffle(memory)\n",
        "\n",
        "    for batchIdx in range(0, len(memory), self.args['batch_size']):\n",
        "      sample = memory[batchIdx:min(len(memory) - 1, batchIdx + self.args['batch_size'])]\n",
        "      state, policy_targets, value_targets = zip(*sample)\n",
        "\n",
        "      state, policy_targets, value_targets = np.array(state), np.array(policy_targets), np.array(value_targets).reshape(-1, 1)\n",
        "\n",
        "      state = torch.tensor(state, dtype = torch.float32, device = self.model.device)\n",
        "      policy_targets = torch.tensor(policy_targets, dtype = torch.float32, device = self.model.device)\n",
        "      value_targets = torch.tensor(value_targets, dtype = torch.float32, device = self.model.device)\n",
        "\n",
        "      out_policy, out_value = self.model(state)\n",
        "\n",
        "      policy_loss = F.cross_entropy(out_policy, policy_targets)\n",
        "      value_loss = F.mse_loss(out_value, value_targets)\n",
        "      loss = policy_loss + value_loss\n",
        "\n",
        "      self.optimizer.zero_grad()\n",
        "      loss.backward()\n",
        "      self.optimizer.step()\n",
        "\n",
        "  #make model learn continuously\n",
        "  def learn(self):\n",
        "    for iteration in range(self.args['num_iterations']):\n",
        "      memory = []\n",
        "\n",
        "      self.model.eval()\n",
        "\n",
        "      for selfPlay_iteration in trange(self.args['num_selfPlay_iterations'] // self.args['num_parallel_games']):\n",
        "        memory += self.selfPlay()\n",
        "\n",
        "      self.model.train()\n",
        "\n",
        "      for epoch in trange(self.args['num_epochs']):\n",
        "        self.train(memory)\n",
        "\n",
        "      torch.save(self.model.state_dict(), f\"model_{iteration}_{self.game}.pt\")\n",
        "      torch.save(self.optimizer.state_dict(), f\"optimizer_{iteration}_{self.game}.pt\")\n",
        "\n",
        "\n",
        "class SPG:\n",
        "  def __init__(self, game):\n",
        "    self.state = game.get_initial_state()\n",
        "    self.memory = []\n",
        "    self.root = None\n",
        "    self.node = None"
      ],
      "metadata": {
        "id": "xapgDFOMLIA6"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#define arguments for the AlphaZero instance\n",
        "#game = TicTacToe()\n",
        "game = ConnectFour()\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "model = ResNet(game, 9, 128, device)\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr = 0.001, weight_decay = 0.0001)\n",
        "args = {\n",
        "     'C': 2,\n",
        "     'num_searches': 600,\n",
        "     'num_iterations': 5,\n",
        "     'num_selfPlay_iterations': 200,\n",
        "     'num_parallel_games': 50,\n",
        "     'num_epochs': 4,\n",
        "     'batch_size': 128,\n",
        "     'temperature': 1.25,\n",
        "     'dirichlet_epsilon': 0.25,\n",
        "     'dirichlet_alpha': 0.3\n",
        "}\n",
        "\n",
        "#Define AlphaZero instance\n",
        "alphaZero = AlphaZeroParallel(model, optimizer, game, args)\n",
        "alphaZero.learn()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "f339370d10c24547ae7b48d7c64b4770",
            "721729af097743828cbbe5c8fe3cbec9",
            "ed6ed5a9e2f346d29f0b91e1caaa6432",
            "d424f5a2f4654239a208360efec49ed6",
            "9ad40cc2518a4cf19f31dfba601b5c47",
            "cf1bfc5677f145189a9312b358a36e55",
            "36dbae7e8fcf41f498cfedbe43915855",
            "b989ee665f904cdab11b8d38294ae21d",
            "06f26a63b64e4afd9b954178b7d4e32f",
            "8a42fc0e854048ae855e51fcbaa21146",
            "4f66ebfca97147dba6aadabf25bd963d",
            "c77bd5fedd4144efa0cc0bf40d83cc43",
            "1efdbed767c64f87a70e2b9fb3206827",
            "69f4c603fa874f9cb7fc53ccd02b1070",
            "866e2890710643948a87b1d73a4a0e6a",
            "d4faf057554e4f12bc24f832507d013b",
            "d64c6d8938c742809941a430f3543d04",
            "71c3a716513944099dd1e23eb4ed35eb",
            "7008e96889a147cd9315b497a644db90",
            "b1e39f60ab654a1dba28e66dc6d1d0d8",
            "78768ad755fa44a0a843f16551cba90e",
            "b4984c3fd55c48ce98b0c7a67cd02438",
            "9d2fba8fda324f7fb96be7f376cd36c2",
            "bcf9a7bdef4944a8b4b553b162ee8449",
            "7fa757a00fc74013b70b0137f5160894",
            "a90e3f85cebc4bdc88a8e2cabcf7246a",
            "35d2c7f272dc46d4a2cd1ddfc00d9455",
            "b0d42f07ac944288b57258b3232fbb34",
            "483692bc8ce846da9b7523c65d7d51bd",
            "bbb8af06c7c24fd1982fae7082697825",
            "e6f86aa408f943b59fe852003809adc5",
            "a459fd5e10254e9c8f47116572877f84",
            "55b57684c23c4dbc96d1a5d96a162716"
          ]
        },
        "id": "mbE0xxPHGjOM",
        "outputId": "47a79192-32fa-477f-9788-2e65fdb9daad"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/4 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f339370d10c24547ae7b48d7c64b4770"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/4 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c77bd5fedd4144efa0cc0bf40d83cc43"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/4 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9d2fba8fda324f7fb96be7f376cd36c2"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "TicTacToe game with MCTS"
      ],
      "metadata": {
        "id": "sdu8fnuma343"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#create a game with MCTS\n",
        "#game = TicTacToe()\n",
        "game = ConnectFour()\n",
        "player = 1\n",
        "\n",
        "#define MCTS arguments\n",
        "args = {\n",
        "    'C': 2,\n",
        "    'num_searches': 100,\n",
        "    'dirichlet_epsilon': 0.0,\n",
        "    'dirichlet_alpha': 0.3\n",
        "}\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "#initialise model\n",
        "model = ResNet(game, 9, 128, device)\n",
        "model.load_state_dict(torch.load(\"model_4_ConnectFour.pt\", map_location = device))\n",
        "model.eval()\n",
        "\n",
        "#initialise MCTS player\n",
        "mcts = MCTS(game, args, model)\n",
        "\n",
        "#initialise game\n",
        "state = game.get_initial_state()\n",
        "\n",
        "while True:\n",
        "  print(state)\n",
        "\n",
        "  #player 1 move\n",
        "  if player == 1:\n",
        "    #get list of valid moves\n",
        "    valid_moves = game.get_valid_moves(state)\n",
        "    print(\"valid_moves\", [i for i in range(game.action_size) if valid_moves[i] == 1])\n",
        "\n",
        "    #player choose next move\n",
        "    action = int(input(f\"{player}:\"))\n",
        "\n",
        "    #check if move is valid\n",
        "    if valid_moves[action] == 0:\n",
        "      print(\"action not valid\")\n",
        "      continue\n",
        "  \n",
        "  #MCTS move\n",
        "  else:\n",
        "    neutral_state = game.change_perspective(state,player)\n",
        "    mcts_probs = mcts.search(neutral_state)\n",
        "    action = np.argmax(mcts_probs)\n",
        "\n",
        "  #update game\n",
        "  state = game.get_next_state(state, action, player)\n",
        "\n",
        "  value, is_terminal = game.get_value_and_terminated(state, action)\n",
        "\n",
        "  #check if game is terminated\n",
        "  if is_terminal:\n",
        "    print(state)\n",
        "    if value == 1:\n",
        "      print(\"player\", player, \"won\")\n",
        "    else:\n",
        "      print(\"draw\")\n",
        "    break\n",
        "  \n",
        "  #change player\n",
        "  player = game.get_opponent(player)"
      ],
      "metadata": {
        "id": "TS-B00FwAS-s"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class KaggleAgent:\n",
        "  def __init__(self, model, gmae, args):\n",
        "    self.model = model\n",
        "    self.game = game\n",
        "    self.args = args\n",
        "    if self.args['search']:\n",
        "      self.mcts = MCTS(self.ggmae, self.args, self.model)\n",
        "\n",
        "  def run(self, obs, conf):\n",
        "    player = obs['mark'] if obs['mark'] == 1 else -1\n",
        "    state = np.array(obs['board']).reshape(self.game.row_count, self.game.column_count)\n",
        "    state[state == 2] = -1\n",
        "\n",
        "    state = self.game.change_perspective(state, player)\n",
        "\n",
        "    if self.args['search']:\n",
        "      policy = self.mcts.search(state)\n",
        "\n",
        "    else:\n",
        "      policy, _ = self.model.predict(state, augment = self.args['augment'])\n",
        "\n",
        "    valid_moves = self.game.get_valid_moves(state)\n",
        "    policy *= valid_moves\n",
        "    policy /= np.sum(policy)\n",
        "\n",
        "    if self.args['temperature'] == 0:\n",
        "      action = int(np.argmax(policy))\n",
        "\n",
        "    elif self.args['temperature'] == float('inf'):\n",
        "      action = np.random.choice([r for r in range(self.game.action_size) if policy[r] > 0])\n",
        "    \n",
        "    else:\n",
        "      policy = policy **(1 / self.args['temperature'])\n",
        "      policy /= np.sum(policy)\n",
        "      action = np.random.choice(self.game.action_size, p = policy)\n",
        "\n",
        "    return action\n",
        "\n",
        "\n",
        "game = ConnectFour()\n",
        "\n",
        "#define MCTS arguments\n",
        "args = {\n",
        "    'C': 2,\n",
        "    'num_searches': 600,\n",
        "    'dirichlet_epsilon': 0.1,\n",
        "    'dirichlet_alpha': 0.3,\n",
        "    'search': True,\n",
        "    'temperature': 0\n",
        "}\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "#initialise model\n",
        "model = ResNet(game, 9, 128, device)\n",
        "model.load_state_dict(torch.load(\"model_4_ConnectFour.pt\", map_location = device))\n",
        "model.eval()\n",
        "\n",
        "env = kaggle_environments.make(\"connectx\")\n",
        "\n",
        "player1 = KaggleAgent(model, game, args)\n",
        "player2 = KaggleAgent(model, game, args)\n",
        "\n",
        "players = [player1.run, player2.run]\n",
        "\n",
        "env.run(players)\n",
        "\n",
        "env.render(mode = \"ipython\")"
      ],
      "metadata": {
        "id": "vwhKVqqrHTLw"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}