Fix & temporarily disable raytune tutorial, print tutorial execution …

…errors & memory usage (#1974)

Summary:
The tutorials script has been updated to mimic the setup in the BoTorch script to print out the errors encountered while running the tutorials.

The raytune tutorial mysteriously fails even after fixing the deprecation errors, so we'll temporarily remove it from the CI. The early stopping tutorial fails trial execution both when run locally and with this diff, so that's also disabled temporarily.

The tutorial changes address the deprecation warning / error:
```
DeprecationWarning: `tune.report` is deprecated.
Use `ray.train.report` instead -- see the example below:

from ray import tune     ->     from ray import train
tune.report(metric=1)    ->     train.report({'metric': 1})
```
Also reduces # of trials from 30 to 10.

Pull Request resolved: #1974

Test Plan: Ran `python scripts/make_tutorials.py -w $(pwd) -e -s` locally and checked the outputs (including with added errors in the tutorials).

Reviewed By: esantorella

Differential Revision: D51187010

Pulled By: saitcakmak

fbshipit-source-id: f5d90bd20710dc497c7fb937a0f3ef68031ee538

scripts/make_tutorials.py

@@ -15,8 +15,14 @@
 
 import nbformat
 from bs4 import BeautifulSoup
+from memory_profiler import memory_usage
 from nbconvert import HTMLExporter, ScriptExporter
 
+TUTORIALS_TO_SKIP = [
+    "raytune_pytorch_cnn",  # TODO: Times out CI but passes locally. Investigate.
+    "early_stopping",  # TODO: The trials fail. Investigate.
+]
+
 
 TEMPLATE = """const CWD = process.cwd();
 
@@ -147,8 +153,7 @@ def gen_tutorials(
     # prepare paths for converted tutorials & files
     os.makedirs(os.path.join(repo_dir, "website", "_tutorials"), exist_ok=True)
     os.makedirs(os.path.join(repo_dir, "website", "static", "files"), exist_ok=True)
-    if smoke_test:
-        os.environ["SMOKE_TEST"] = str(smoke_test)
+    env = {"SMOKE_TEST": "True"} if smoke_test else None
 
     for config in tutorial_configs:
         tid = config["id"]
@@ -162,32 +167,45 @@ def gen_tutorials(
             nb_str = infile.read()
             nb = nbformat.reads(nb_str, nbformat.NO_CONVERT)
 
+        total_time = None
         if exec_tutorials and exec_on_build:
+            if tid in TUTORIALS_TO_SKIP:
+                print(f"Skipping {tid}")
+                continue
             tutorial_path = Path(paths["tutorial_path"])
             print("Executing tutorial {}".format(tid))
-            start_time = time.time()
+            start_time = time.monotonic()
 
-            # try / catch failures for now
-            # will re-raise at the end
+            # Try / catch failures for now. We will re-raise at the end.
+            timeout_minutes = 15 if smoke_test else 150
             try:
                 # Execute notebook.
-                # TODO: [T163244135] Speed up tutorials and reduce timeout limits.
-                timeout_minutes = 15 if smoke_test else 150
-                run_script(tutorial=tutorial_path, timeout_minutes=timeout_minutes)
-                total_time = time.time() - start_time
+                mem_usage, run_out = memory_usage(
+                    (run_script, (tutorial_path, timeout_minutes), {"env": env}),
+                    retval=True,
+                    include_children=True,
+                )
+                total_time = time.monotonic() - start_time
                 print(
-                    "Done executing tutorial {}. Took {:.2f} seconds.".format(
-                        tid, total_time
-                    )
+                    f"Finished executing tutorial {tid} in {total_time:.2f} seconds. "
+                    f"Starting memory usage was {mem_usage[0]} MB & "
+                    f"the peak memory usage was {max(mem_usage)} MB."
                 )
-            except Exception as exc:
+            except subprocess.TimeoutExpired:
                 has_errors = True
-                print("Couldn't execute tutorial {}!".format(tid))
-                print(exc)
-                total_time = None
-        else:
-            total_time = None
-
+                print(
+                    f"Tutorial {tid} exceeded the maximum runtime of "
+                    f"{timeout_minutes} minutes."
+                )
+            try:
+                run_out.check_returncode()
+            except subprocess.CalledProcessError:
+                has_errors = True
+                print(
+                    f"Encountered error running tutorial {tid}: \n"
+                    f"stdout: \n {run_out.stdout} \n"
+                    f"stderr: \n {run_out.stderr} \n"
+                )
         # convert notebook to HTML
         exporter = HTMLExporter(template_name="classic")
         html, _ = exporter.from_notebook_node(nb)

setup.py

@@ -73,6 +73,7 @@
     "pyro-ppl",  # Required for to call run_inference.
     "pytorch-lightning",  # For the early stopping tutorial.
     "papermill",  # For executing the tutorials.
+    "memory_profiler",  # For measuring memory usage of the tutorials.
 ]