Add infrastructure for running experiments connected to classes.

src/gened/base.py

@@ -22,6 +22,7 @@
     db,
     demo,
     docs,
+    experiments,  # noqa: F401 (import registers routes even though unused here)
     filters,
     instructor,
     lti,

src/gened/experiments.py

@@ -0,0 +1,82 @@
+# SPDX-FileCopyrightText: 2024 Mark Liffiton <[email protected]>
+#
+# SPDX-License-Identifier: AGPL-3.0-only
+
+from flask import (
+    flash,
+    redirect,
+    render_template,
+    request,
+    url_for,
+)
+from werkzeug.wrappers.response import Response
+
+from .admin import bp as bp_admin
+from .admin import register_admin_link
+from .db import get_db
+
+# ### Admin routes ###
+# Auth requirements covered by admin.before_request()
+
+@register_admin_link("Experiments")
+@bp_admin.route("/experiments/")
+def experiments_view() -> str:
+    db = get_db()
+    experiments = db.execute("SELECT *, (SELECT COUNT(*) FROM experiment_class WHERE experiment_id=id) AS count FROM experiments").fetchall()
+    return render_template("admin_experiments.html", experiments=experiments)
+
+@bp_admin.route("/experiment/new")
+def experiment_new() -> str:
+    return render_template("experiment_form.html")
+
+@bp_admin.route("/experiment/<int:id>")
+def experiment_form(id: int) -> str:
+    db = get_db()
+    experiment = db.execute("SELECT * FROM experiments WHERE id=?", [id]).fetchone()
+    classes = db.execute("SELECT id, name FROM classes ORDER BY name").fetchall()
+    classes = [dict(row) for row in classes]  # so we can tojson it in the template
+    assigned_classes = db.execute("SELECT class_id AS id, classes.name FROM experiment_class JOIN classes ON experiment_class.class_id=classes.id WHERE experiment_id=? ORDER BY name", [id]).fetchall()
+    assigned_classes = [dict(row) for row in assigned_classes]
+    return render_template("experiment_form.html", experiment=experiment, classes=classes, assigned_classes=assigned_classes)
+
+@bp_admin.route("/experiment/update", methods=['POST'])
+def experiment_update() -> Response:
+    db = get_db()
+
+    exp_id = request.form.get("exp_id", type=int)
+
+    if exp_id is None:
+        # Adding a new experiment
+        cur = db.execute("INSERT INTO experiments (name, description) VALUES (?, ?)",
+                         [request.form['name'], request.form['description']])
+        exp_id = cur.lastrowid
+        db.commit()
+        flash(f"Experiment {request.form['name']} created.")
+    else:
+        # Updating
+        db.execute("UPDATE experiments SET name=?, description=? WHERE id=?",
+                   [request.form['name'], request.form['description'], exp_id])
+        db.commit()
+        flash("Experiment updated.")
+
+    # Update assigned classes
+    db.execute("DELETE FROM experiment_class WHERE experiment_id=?", [exp_id])
+    for class_id in request.form.getlist('assigned_classes'):
+        db.execute("INSERT INTO experiment_class (experiment_id, class_id) VALUES (?, ?)",
+                   [exp_id, class_id])
+    db.commit()
+
+    return redirect(url_for(".experiment_form", id=exp_id))
+
+@bp_admin.route("/experiment/delete/<int:exp_id>", methods=['POST'])
+def experiment_delete(exp_id: int) -> Response:
+    db = get_db()
+
+    # Delete the experiment and its class assignments
+    db.execute("DELETE FROM experiment_class WHERE experiment_id=?", [exp_id])
+    db.execute("DELETE FROM experiments WHERE id=?", [exp_id])
+    db.commit()
+
+    flash("Experiment deleted.")
+
+    return redirect(url_for(".experiments_view"))

src/gened/migrations/20240624--add_experiment_tables.sql

@@ -0,0 +1,27 @@
+-- SPDX-FileCopyrightText: 2024 Mark Liffiton <[email protected]>
+--
+-- SPDX-License-Identifier: AGPL-3.0-only
+
+BEGIN;
+
+-- Create experiments table
+CREATE TABLE experiments (
+    id          INTEGER PRIMARY KEY AUTOINCREMENT,
+    name        TEXT NOT NULL UNIQUE,
+    description TEXT
+);
+
+-- Create experiment_class table
+CREATE TABLE experiment_class (
+    experiment_id   INTEGER NOT NULL,
+    class_id        INTEGER NOT NULL,
+    PRIMARY KEY (experiment_id, class_id),
+    FOREIGN KEY (experiment_id) REFERENCES experiments (id),
+    FOREIGN KEY (class_id) REFERENCES classes (id)
+);
+
+-- Create indexes for experiment_class table
+CREATE INDEX exp_crs_experiment_idx ON experiment_class(experiment_id);
+CREATE INDEX exp_crs_class_idx ON experiment_class(class_id);
+
+COMMIT;

src/gened/schema_common.sql

@@ -157,10 +157,29 @@ CREATE TABLE models (
     id          INTEGER PRIMARY KEY AUTOINCREMENT,
     name        TEXT NOT NULL UNIQUE,
     shortname   TEXT NOT NULL UNIQUE,
-    model       TEXT
+    model       TEXT NOT NULL
 );
 INSERT INTO models(name, shortname, model) VALUES
     ('OpenAI GPT-3.5 Turbo', 'GPT-3.5', 'gpt-3.5-turbo-0125'),
     ('OpenAI GPT-4o', 'GPT-4', 'gpt-4o')
 ;
 
+
+-- Experiments (like feature flags)
+CREATE TABLE experiments (
+    id      INTEGER PRIMARY KEY AUTOINCREMENT,
+    name    TEXT NOT NULL UNIQUE,
+    description  TEXT
+);
+
+CREATE TABLE experiment_class (
+    experiment_id   INTEGER NOT NULL,
+    class_id        INTEGER NOT NULL,
+    PRIMARY KEY (experiment_id, class_id),
+    FOREIGN KEY (experiment_id) REFERENCES experiments (id),
+    FOREIGN KEY (class_id) REFERENCES classes (id)
+);
+DROP INDEX IF EXISTS exp_crs_experiment_idx;
+CREATE INDEX exp_crs_experiment_idx ON experiment_class(experiment_id);
+DROP INDEX IF EXISTS exp_crs_class_idx;
+CREATE INDEX exp_crs_class_idx ON experiment_class(class_id);

src/gened/templates/admin_experiments.html

@@ -0,0 +1,20 @@
+{#
+SPDX-FileCopyrightText: 2024 Mark Liffiton <liffiton@gmail.com>
+
+SPDX-License-Identifier: AGPL-3.0-only
+#}
+
+{% extends "admin_main.html" %}
+{% from "tables.html" import datatable %}
+
+{% block admin_body %}
+  <h1 class="is-size-3">Experiments <a class="button is-light is-link is-small mt-2" href="{{url_for('admin.experiment_new')}}">Create New</a></h1>
+  <div style="max-width: 50em;">
+    {{ datatable(
+      'experiments',
+      [('id', 'id'), ('name', 'name'), ('description', 'description'), ('classes', 'count', 'r')],
+      experiments,
+      edit_handler="admin.experiment_form",
+    ) }}
+  </div>
+{% endblock %}

src/gened/templates/experiment_form.html

@@ -0,0 +1,130 @@
+{#
+SPDX-FileCopyrightText: 2024 Mark Liffiton <liffiton@gmail.com>
+
+SPDX-License-Identifier: AGPL-3.0-only
+#}
+
+{% extends "admin_main.html" %}
+
+{% block admin_body %}
+  <div class="container">
+
+    <form action="{{url_for('admin.experiment_update')}}" method="post">
+      {% if experiment %}
+        {# We're editing an existing experiment.  Provide its ID. #}
+        <h1 class="title">Edit Experiment</h1>
+        <input type="hidden" name="exp_id" value="{{experiment.id}}">
+      {% else %}
+        <h1 class="title">New Experiment</h1>
+      {% endif %}
+
+      <div class="field is-horizontal">
+        <div class="field-label is-normal">
+          <label class="label" for="name">Name:</label>
+        </div>
+        <div class="field-body">
+          <div class="field">
+            <div class="control">
+              <input class="input" required name="name" id="name" value="{{ experiment.name if experiment else '' }}">
+            </div>
+          </div>
+        </div>
+      </div>
+
+      <div class="field is-horizontal">
+        <div class="field-label is-normal">
+          <label class="label" for="description">Description:</label>
+        </div>
+        <div class="field-body">
+          <div class="field">
+            <div class="control">
+              <textarea class="textarea" name="description" id="description">{{ experiment.description if experiment else '' }}</textarea>
+            </div>
+          </div>
+        </div>
+      </div>
+
+      {% if experiment %}
+      <div class="field is-horizontal" x-data="classSelector()">
+        <div class="field-label is-normal">
+          <label class="label">Assigned Classes:</label>
+        </div>
+        <div class="field-body" style="display: flex; flex-wrap: wrap;">
+          <div class="field" style="flex: 20em 0 0;">
+            <div class="dropdown" :class="{'is-active': isOpen}" style="width: 100%">
+              <div class="control dropdown-trigger" style="width: 100%">
+                <input class="input" type="text" style="width: 100%" placeholder="Search classes..." 
+                       x-model="search" @input="filterClasses" @focus="filterClasses" @click.away="isOpen=false">
+              </div>
+              <div class="dropdown-menu" style="width: 100%; max-width: 100%; max-height: 80vh; overflow-x: clip; overflow-y: auto;">
+                <div class="dropdown-content">
+                  <template x-for="cls in filteredClasses" :key="cls.id">
+                    <a class="dropdown-item" @click="addClass(cls)">
+                      <span x-text="cls.name"></span>
+                    </a>
+                  </template>
+                </div>
+              </div>
+            </div>
+          </div>
+          <div style="flex: 20em 1 0; overflow-x: auto;">
+            <template x-for="cls in selectedClasses" :key="cls.id">
+              <button @click="removeClass(cls)" class="button is-info is-ounded tag is-medium m-1">
+                <span x-text="cls.name"></span>
+                <span class="delete"></span>
+                <input type="hidden" name="assigned_classes" :value="cls.id">
+              </button>
+            </template>
+          </div>
+        </div>
+      </div>
+
+      <script>
+        function classSelector() {
+          return {
+            search: '',
+            isOpen: false,
+            classes: {{ classes | tojson }},
+            selectedClasses: {{ assigned_classes | tojson }},
+            filteredClasses: [],
+            filterClasses() {
+              this.filteredClasses = this.classes.filter(c => 
+                c.name.toLowerCase().includes(this.search.toLowerCase()) &&
+                !this.selectedClasses.some(sc => sc.id === c.id)
+              );
+              this.isOpen = this.filteredClasses.length > 0;
+            },
+            addClass(cls) {
+              this.selectedClasses.push(cls);
+              this.filterClasses();
+            },
+            removeClass(cls) {
+              const index = this.selectedClasses.findIndex(c => c.id === cls.id);
+              if (index !== -1) {
+                this.selectedClasses.splice(index, 1);
+              }
+            }
+          }
+        }
+      </script>
+      {% endif %}
+
+      <div class="field is-horizontal">
+        <div class="field-label is-normal"><!-- spacing --></div>
+        <div class="field-body">
+          <div class="field">
+            <div class="control">
+              <button class="button is-link" type="submit">Submit</button>
+              {% if experiment %}
+                <button class="button is-danger" type="submit" formaction="{{ url_for('admin.experiment_delete', exp_id=experiment.id) }}" onclick="return confirm('Are you sure you want to delete this experiment?');">
+                   <span class="delete mr-2"></span>
+                   Delete {{ experiment.name }}
+                </button>
+              {% endif %}
+            </div>
+          </div>
+        </div>
+      </div>
+
+  </div>
+{% endblock %}