Loosen Mixtral loss

modal-labs · Feb 15, 2024 · b03e00c · b03e00c
1 parent aaa3b68
commit b03e00c
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 1 deletion.
diff --git a/ci/check_inference.py b/ci/check_inference.py
@@ -12,4 +12,6 @@
 
     output = subprocess.check_output(["modal", "run", "src.inference", "--run-folder", f"/runs/{run_name}", "--prompt", prompt])
 
+    print("Asserting that output contains [SQ] SELECT ... [/SQL]")
+
     assert b"[SQL] SELECT" in output and b"[/SQL]" in output
diff --git a/ci/check_loss.py b/ci/check_loss.py
@@ -27,5 +27,8 @@
     train_loss = float(results["TrainingLoss"].iloc[-1])
     val_loss = float(results["ValidationLoss"].iloc[-1])
 
+    # Arbitrary threshold
+    max_loss = 0.8 if b"Mixtral" in contents else 0.25
+
     print(f"Loss: {train_loss:.2f} (training), {val_loss:.2f} (validation)")
-    sys.exit(val_loss > 0.4)  # Arbitrary threshold
+    sys.exit(val_loss > max_loss)